+150
@@ -0,0 +1,150 @@
|
|||||||
|
package com.fangjia.script.ccbevaluation
|
||||||
|
|
||||||
|
import com.fangjia.core.util.HttpRequest
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import net.sf.json.JSONObject
|
||||||
|
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||||
|
|
||||||
|
import java.sql.ResultSet
|
||||||
|
import java.util.concurrent.ConcurrentHashMap
|
||||||
|
import java.util.concurrent.CountDownLatch
|
||||||
|
import java.util.concurrent.ExecutorService
|
||||||
|
import java.util.concurrent.Executors
|
||||||
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
|
||||||
|
class UpdateHouseRegion {
|
||||||
|
|
||||||
|
|
||||||
|
public static EsDataSource dataSource = new EsDataSource()
|
||||||
|
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||||
|
static {
|
||||||
|
dataSource.setUrl(address)
|
||||||
|
Properties connectionProperties = new Properties()
|
||||||
|
connectionProperties.put("user", "developer")
|
||||||
|
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||||
|
dataSource.setProperties(connectionProperties)
|
||||||
|
}
|
||||||
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new UpdateHouseRegion()
|
||||||
|
task.run()
|
||||||
|
}
|
||||||
|
def run(){
|
||||||
|
def city = "º¼ÖÝ"
|
||||||
|
// findCount(city, 20210101);
|
||||||
|
def rets = findHouseSource(city, 20200701, 20201231)
|
||||||
|
batchUpdate(city, rets);
|
||||||
|
}
|
||||||
|
|
||||||
|
def findCount(def city, def startDate, def endDate){
|
||||||
|
final StringBuffer sqlBuffer = new StringBuffer()
|
||||||
|
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-2020\" WHERE 1=1")
|
||||||
|
sqlBuffer.append(" and city = '"+city+"'")
|
||||||
|
sqlBuffer.append(" and cat = 'sell'")
|
||||||
|
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <="+endDate)
|
||||||
|
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||||
|
|
||||||
|
while (rs.next()) {
|
||||||
|
def count = rs.getInt("count")
|
||||||
|
println count
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def batchUpdate(def city, def houses){
|
||||||
|
def updateCount = new AtomicLong()
|
||||||
|
def curCount = new AtomicLong()
|
||||||
|
def totalCount = houses.size();
|
||||||
|
final CountDownLatch latch = new CountDownLatch(totalCount)
|
||||||
|
println city +" ·¿Ô´×ÜÁ¿£º"+totalCount
|
||||||
|
for(def i=0;i<totalCount;i++) {
|
||||||
|
final def house = houses[i];
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
curCount.addAndGet(1)
|
||||||
|
try {
|
||||||
|
def region = house.region
|
||||||
|
def name = house.name
|
||||||
|
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
|
||||||
|
if (!rets) {
|
||||||
|
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
|
||||||
|
}
|
||||||
|
|
||||||
|
if (rets.size() == 1) {
|
||||||
|
def dist = rets[0]
|
||||||
|
if (region != dist.region) {
|
||||||
|
updateCount.addAndGet(1)
|
||||||
|
updateHouseRegion2(house.url, dist.region);
|
||||||
|
println updateCount.get() + "/${curCount.get()}/" + totalCount + " " + region + " " + dist.region + " " + dist.name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
def findHouseSource(def city, def startDate, def endDate){
|
||||||
|
final StringBuffer sqlBuffer = new StringBuffer()
|
||||||
|
sqlBuffer.append("SELECT * FROM \"seawater-2020\" WHERE 1=1")
|
||||||
|
sqlBuffer.append(" and city = '"+city+"'")
|
||||||
|
sqlBuffer.append(" and cat = 'sell'")
|
||||||
|
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
|
||||||
|
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||||
|
|
||||||
|
def houses = []
|
||||||
|
while (rs.next()){
|
||||||
|
houses << ["region":rs.getString("region"), "name":rs.getString("district_name"),"url":rs.getString("url")]
|
||||||
|
}
|
||||||
|
return houses;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def updateHouseRegion2(def url, def region){
|
||||||
|
def header = [:]
|
||||||
|
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("developer:5e8edd851d2fdfbd7415232c67367cc3".getBytes("UTF-8"))}".toString()
|
||||||
|
String seawaterUrl = "http://192.168.0.206:9200/seawater-2020/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||||
|
def body = """
|
||||||
|
{
|
||||||
|
"script": {
|
||||||
|
"source": "ctx._source.region='${region}'",
|
||||||
|
"lang": "painless"
|
||||||
|
},
|
||||||
|
"query": {
|
||||||
|
"term": {
|
||||||
|
"url": "${url}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
HttpRequest request = new HttpRequest()
|
||||||
|
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||||
|
// def response = JSONObject.fromObject(res)
|
||||||
|
// println(response)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def updateHouseRegion(def url, def region){
|
||||||
|
final StringBuffer sqlBuffer = new StringBuffer()
|
||||||
|
sqlBuffer.append("update seawater-2021")
|
||||||
|
sqlBuffer.append(" set region = '"+region+"'")
|
||||||
|
sqlBuffer.append(" where url='"+url+"'")
|
||||||
|
def result = dataSource.connection.createStatement().executeUpdate(sqlBuffer.toString())
|
||||||
|
println region +" "+result
|
||||||
|
}
|
||||||
|
|
||||||
|
def getAllDistricts(def city){
|
||||||
|
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias"])
|
||||||
|
def map = [:]
|
||||||
|
for(def d : dists){
|
||||||
|
|
||||||
|
map.put()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+19
@@ -890,6 +890,25 @@ class CommonDatautils {
|
|||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static def findDistrictsByName(String city, String name, String cat){
|
||||||
|
|
||||||
|
DBObject q = new BasicDBObject();
|
||||||
|
q.put("city", city);
|
||||||
|
q.put("cat", cat);
|
||||||
|
q.put("name", name);
|
||||||
|
q.put("visible", 0);
|
||||||
|
|
||||||
|
def rets = []
|
||||||
|
DSCursor cursor = getDataStoreConnection().find(q);
|
||||||
|
if(cursor){
|
||||||
|
for(def obj : cursor){
|
||||||
|
rets << obj;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rets;
|
||||||
|
}
|
||||||
|
|
||||||
public static def getSimpleDistrictMap(def cat, def city){
|
public static def getSimpleDistrictMap(def cat, def city){
|
||||||
DBObject q = new BasicDBObject();
|
DBObject q = new BasicDBObject();
|
||||||
q.put("visible", 0);
|
q.put("visible", 0);
|
||||||
|
|||||||
Reference in New Issue
Block a user