From 40013fb01d60d85262f53002db587fbe73819695 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Tue, 16 Mar 2021 11:58:46 +0800 Subject: [PATCH] update --- projects/fangjia-script/pom.xml | 5 ++ .../script/wenyan/temp/ClickHouseDemo.groovy | 69 +++++++++++++++++++ .../fangjia/script/wenyan/temp/Price.groovy | 21 +++--- 3 files changed, 84 insertions(+), 11 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ClickHouseDemo.groovy diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index 759a77240..ee00efea2 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -115,6 +115,11 @@ fangjia-datastore-driver 3.9.1 + + cc.blynk.clickhouse + clickhouse4j + 1.4.4 + redis.clients jedis diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ClickHouseDemo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ClickHouseDemo.groovy new file mode 100644 index 000000000..71df8ece9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ClickHouseDemo.groovy @@ -0,0 +1,69 @@ +package com.fangjia.script.wenyan.temp + +import cc.blynk.clickhouse.copy.CopyManager +import cc.blynk.clickhouse.copy.CopyManagerFactory +import com.fangjia.core.util.DateUtil +import net.sf.json.JSONNull +import net.sf.json.JSONObject +import ru.yandex.clickhouse.ClickHouseConnection +import ru.yandex.clickhouse.ClickHouseDriver +import ru.yandex.clickhouse.settings.ClickHouseProperties + +import java.sql.PreparedStatement +import java.sql.Timestamp + +ClickHouseDriver clickHouseDriver = new ClickHouseDriver() +ClickHouseProperties clickHouseProperties = new ClickHouseProperties() +clickHouseProperties.connectionTimeout = 1000 * 3600 * 24 +clickHouseProperties.keepAliveTimeout = 1000 * 3600 * 24 +clickHouseProperties.dataTransferTimeout = 1000 * 3600 * 24 +clickHouseProperties.user = "default" +clickHouseProperties.password = "goojia123456" +ClickHouseConnection connection = clickHouseDriver.connect("jdbc:clickhouse://192.168.0.93:8123/fangjia", clickHouseProperties) +CopyManager copyManager = CopyManagerFactory.create(connection) +copyManager.copyFromDb("select * from fangjia.SEAWATER", "D://123.csv"); + + +//PreparedStatement sth = connection.prepareStatement("INSERT INTO SEAWATER(id,city,region,name,district_name,address,room,hall,toilet,area,avg_price,total_price ,floor_info,height,floor,direction,fitment,p_date,publish_date,c_date,m_date) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") +//def cols = ["id", "city", "region", "name", "district_name", "address", "room", "hall", "toilet", "area", "avg_price", "total_price ", "floor_info", "height", "floor", "direction", "fitment", "p_date", "publish_date", "c_date", "m_date"] +//int count = 0 +//new File("D:\\建行测试数据").eachFile { def file -> +// file.eachLine("UTF-8", { def line -> +// String id = line.substring(0, line.indexOf("\t")) +// String jsonStr = line.substring(line.indexOf("\t") + 1) +// count++ +// JSONObject json = JSONObject.fromObject(jsonStr) +// json.put("id", id) +// for (int i = 0; i < cols.size(); i++) { +// if (cols[i].toString().endsWith("date") && "p_date" != cols[i]) { +// if ("c_date" == cols[i]) { +// if (json.get("m_date") instanceof Long) { +// sth.setTimestamp(i + 1, new Timestamp(json.get("m_date"))) +// } else { +// sth.setTimestamp(i + 1, new Timestamp(((Date) DateUtil.parseISODate(json.get("m_date"))).getTime())) +// +// } +// } else { +// if (json.get(cols[i]) instanceof String) { +// sth.setTimestamp(i + 1, new Timestamp(((Date) DateUtil.parseISODate(json.get(cols[i]))).getTime())) +// } else { +// sth.setTimestamp(i + 1, new Timestamp(json.get(cols[i]))) +// +// } +// } +// } else { +// if (json.get(cols[i]) instanceof JSONNull) { +// sth.setObject(i + 1, null) +// } else { +// sth.setObject(i + 1, json.get(cols[i])) +// } +// } +// } +// sth.addBatch() +// if (count % 30000 == 0) { +// sth.executeBatch() +// println(new Date().format("HH:mm:ss " + count)) +// } +// }) +//} +//sth.execute() \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Price.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Price.groovy index d314114dc..b245cfec6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Price.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Price.groovy @@ -1,17 +1,16 @@ package com.fangjia.script.wenyan.temp -import com.fangjia.core.ProvinceData -import com.fangjia.core.RegionData -import com.fangjia.script.util.CommonUtils + import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject -ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\建行深开数据统计.xlsx",{List item-> - String city = RegionData.formatCity(item["A"]) - if(city){ - item["G"] = DbUtils.seaweed.count(DbUtils.buildSeaweedQuery(RegionData.formatCity(item["A"]))) - }else{ - item["G"] = null +def data = [["城市", "区域", "小区名", "别名", "地址", "地址别名"]] +["ID", "北京", "重庆", "哈尔滨"].each {def city-> + DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(city)).each {BasicDBObject district-> + def item = [district._id.toString(), district.city, district.region, district.name, district.alias?.join(","), district.address, district.addr_alias?.join(",")] + println(item) + data << item } - -}) \ No newline at end of file +} +ExcelUtils.write("D:\\待抓取小区.xlsx", data, "数据") \ No newline at end of file