From c217142d9ca01b35f6800407cb4dba4999f74310 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 13 Sep 2021 15:21:04 +0800 Subject: [PATCH] update --- .../ccbevaluation/UpdateHouseRegion.groovy | 150 ++++++++++++++++++ .../script/commData/CommonDatautils.groovy | 23 ++- 2 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy new file mode 100644 index 000000000..082d07eb9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy @@ -0,0 +1,150 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.commData.CommonDatautils +import net.sf.json.JSONObject +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class UpdateHouseRegion { + + + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + static main(args) { + def task = new UpdateHouseRegion() + task.run() + } + def run(){ + def city = "杭州" +// findCount(city, 20210101); + def rets = findHouseSource(city, 20200701, 20201231) + batchUpdate(city, rets); + } + + def findCount(def city, def startDate, def endDate){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT count(*) as count FROM \"seawater-2020\" WHERE 1=1") + sqlBuffer.append(" and city = '"+city+"'") + sqlBuffer.append(" and cat = 'sell'") + sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <="+endDate) + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + + while (rs.next()) { + def count = rs.getInt("count") + println count + } + } + + def batchUpdate(def city, def houses){ + def updateCount = new AtomicLong() + def curCount = new AtomicLong() + def totalCount = houses.size(); + final CountDownLatch latch = new CountDownLatch(totalCount) + println city +" 房源总量:"+totalCount + for(def i=0;i="+startDate).append(" and p_date <"+endDate); + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + + def houses = [] + while (rs.next()){ + houses << ["region":rs.getString("region"), "name":rs.getString("district_name"),"url":rs.getString("url")] + } + return houses; + } + + + + def updateHouseRegion2(def url, def region){ + def header = [:] + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("developer:5e8edd851d2fdfbd7415232c67367cc3".getBytes("UTF-8"))}".toString() + String seawaterUrl = "http://192.168.0.206:9200/seawater-2020/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false" + def body = """ + { + "script": { + "source": "ctx._source.region='${region}'", + "lang": "painless" + }, + "query": { + "term": { + "url": "${url}" + } + } + } + """ + HttpRequest request = new HttpRequest() + String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") +// def response = JSONObject.fromObject(res) +// println(response) + } + + + def updateHouseRegion(def url, def region){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("update seawater-2021") + sqlBuffer.append(" set region = '"+region+"'") + sqlBuffer.append(" where url='"+url+"'") + def result = dataSource.connection.createStatement().executeUpdate(sqlBuffer.toString()) + println region +" "+result + } + + def getAllDistricts(def city){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias"]) + def map = [:] + for(def d : dists){ + + map.put() + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 2a1d89a8a..42d3641f3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -536,8 +536,8 @@ class CommonDatautils { if(name){ alias << name; } - - + + DBObject q = new BasicDBObject(); q.put("city", city); q.put("region", region); @@ -890,6 +890,25 @@ class CommonDatautils { } return null; } + + public static def findDistrictsByName(String city, String name, String cat){ + + DBObject q = new BasicDBObject(); + q.put("city", city); + q.put("cat", cat); + q.put("name", name); + q.put("visible", 0); + + def rets = [] + DSCursor cursor = getDataStoreConnection().find(q); + if(cursor){ + for(def obj : cursor){ + rets << obj; + } + } + return rets; + } + public static def getSimpleDistrictMap(def cat, def city){ DBObject q = new BasicDBObject(); q.put("visible", 0);