Merge branch 'script_gj' into 'master'
Script gj See merge request !1102
This commit is contained in:
+2
-2
@@ -189,13 +189,13 @@ class AddDevShapeData {
|
||||
|
||||
|
||||
def run(){
|
||||
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\218\\'
|
||||
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\临时-没付费\\'
|
||||
def dir = new File(directoryPath)
|
||||
if (dir.isDirectory()) {
|
||||
dir.eachFileRecurse(FileType.FILES) { file ->
|
||||
process(file.absolutePath)
|
||||
}
|
||||
// addDevInfo("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx")
|
||||
addDevInfo("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx")
|
||||
} else {
|
||||
println "The specified path is not a directory."
|
||||
}
|
||||
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
/**
|
||||
* 将43上的开发区同步到mongo-100, es-89, es-52
|
||||
*/
|
||||
class SynchDevDataBatch extends BaseBatch{
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new SynchDevDataBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def devMap = synchDevTo100()
|
||||
// synchDevToEs(devMap)
|
||||
// synchDevEs(devMap, "192.168.0.89")
|
||||
synchDevEs(devMap, "192.168.0.52")
|
||||
}
|
||||
|
||||
def synchDevTo100(){
|
||||
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find()
|
||||
def rets = []
|
||||
def devMap = [:]
|
||||
for(def r : dbcursor){
|
||||
rets << r
|
||||
devMap.put(r._id.toString(), r)
|
||||
}
|
||||
//删除旧的,再同步新的
|
||||
development_zone_CONN.remove(new BasicDBObject())
|
||||
development_zone_CONN.insert(rets)
|
||||
return devMap;
|
||||
}
|
||||
|
||||
def synchDevToEs(devMap){
|
||||
// def esDevs = findEsDevDatas()
|
||||
// def ids = []
|
||||
// for(def dev : devDatas){
|
||||
// ids << dev._id.toString()
|
||||
// def esObj = esDevs.get(dev._id.toString())
|
||||
// def updateValMap = [:]
|
||||
// if(esObj){
|
||||
// dev.each{k,v->
|
||||
// if(k!="id" && k!="_id"){
|
||||
// def val = dev.get(k)
|
||||
// if(val!=v){
|
||||
// updateValMap.put(k,val)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// if(!esObj || (esObj && updateValMap)){
|
||||
// ElasticUtils.modify(dev._id.toString(), dev, "192.168.0.89", "development_zone")
|
||||
// ElasticUtils.modify(dev._id.toString(), dev, "192.168.0.52", "development_zone")
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// def esDelIds = []
|
||||
// esDevs.each{id,map->
|
||||
// if(!ids.contains(id)){
|
||||
// ElasticUtils.delete(id, "192.168.0.89", "development_zone")
|
||||
// ElasticUtils.delete(id, "192.168.0.52", "development_zone")
|
||||
// esDelIds << id
|
||||
// }
|
||||
// }
|
||||
// println "Es删除开发区数量:"+esDelIds.size()
|
||||
}
|
||||
|
||||
|
||||
def synchDevEs(devMap, host){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
||||
|
||||
ElasticUtils89.executeQuery(host, "development_zone", queryBuilder, { String id, doc->
|
||||
def mongoObj = devMap.get(id)
|
||||
def updateFieldCount = 0
|
||||
mongoObj.each{k,v->
|
||||
if(!["id","_id" ,"build_date" ,"create_time","bd_shape","img_url","operator","operate_result","m_date","township","block"].contains(k)){
|
||||
def val = doc.get(k)
|
||||
if(val!=v){
|
||||
println doc.name +" 修改 "+k+" "+ v+" "+id
|
||||
// updateValMap.put(k,val)
|
||||
updateFieldCount++
|
||||
doc.put(k,v)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(mongoObj && updateFieldCount){
|
||||
//进行修改
|
||||
return true
|
||||
}else{
|
||||
if(!mongoObj){
|
||||
println "删除:"+id +" "+doc.name
|
||||
ElasticUtils.delete(id, host, "development_zone")
|
||||
return false
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+82
-15
@@ -5,6 +5,7 @@ import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.math.RandomUtils
|
||||
@@ -17,8 +18,9 @@ import java.util.regex.Pattern
|
||||
*/
|
||||
class RegionPointCrawl {
|
||||
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline_2025")
|
||||
static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code")
|
||||
static DBCollection OLD_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
|
||||
static main(args) {
|
||||
def task = new RegionPointCrawl();
|
||||
@@ -26,16 +28,19 @@ class RegionPointCrawl {
|
||||
//4个直辖市 北京市、上海市、天津市、重庆市
|
||||
//5个自治区 "内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"
|
||||
// def provinces = ["河北省", "山西省","吉林省","辽宁省", "黑龙江省", "陕西省", "甘肃省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省", "云南省"]
|
||||
def provinces = ["甘肃省"]
|
||||
def index = 0
|
||||
for(def p : provinces){
|
||||
index ++
|
||||
println index +"/"+provinces.size() +" "+p
|
||||
task.crawlCityAndDistrictCode(p);
|
||||
println "城市 区域 code 完成"
|
||||
task.run(p);
|
||||
println "区域围栏 完成"
|
||||
}
|
||||
//已跑:甘肃省,河北省,山西省,"吉林省","辽宁省", "黑龙江省", "陕西省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省","云南省"
|
||||
// def provinces = ["北京市","上海市","天津市","重庆市" ]
|
||||
// def index = 0
|
||||
// for(def p : provinces){
|
||||
// index ++
|
||||
// println index +"/"+provinces.size() +" "+p
|
||||
// task.crawlCityAndDistrictCode(p);
|
||||
// println "城市 区域 code 完成"
|
||||
// task.run(p);
|
||||
// println "区域围栏 完成"
|
||||
// }
|
||||
|
||||
task.statSame()
|
||||
|
||||
}
|
||||
|
||||
@@ -47,7 +52,10 @@ class RegionPointCrawl {
|
||||
def cityCodes = findCityCode(pro);
|
||||
if(cityCodes){
|
||||
for(def cityMap : cityCodes){
|
||||
def codeMap = findAllDistrictCode(cityMap.city_code);
|
||||
def codeMap = findAllDistrictCode(cityMap.city_code, "district");
|
||||
// if(!codeMap){
|
||||
// codeMap = findAllDistrictCode(cityMap.city_code, "street");
|
||||
// }
|
||||
def index = 0
|
||||
for(def m : codeMap){
|
||||
index ++
|
||||
@@ -93,7 +101,7 @@ class RegionPointCrawl {
|
||||
def saveGaoDeRegionCode(province, cityName,districts){
|
||||
if(districts){
|
||||
for(def dist : districts){
|
||||
if(dist.level=="district" || cityName=="东莞市"){
|
||||
if(dist.level=="district" || dist.level=="street" ){
|
||||
saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level);
|
||||
}
|
||||
}
|
||||
@@ -125,10 +133,10 @@ class RegionPointCrawl {
|
||||
return new BasicDBObject("$regex", pattern);
|
||||
}
|
||||
|
||||
def findAllDistrictCode(def citycode){
|
||||
def findAllDistrictCode(def citycode, def level){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_code", citycode)
|
||||
q.put("level", "district")
|
||||
q.put("level", level)
|
||||
def dbcursor = GAODECITYCODECOLL.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
@@ -149,6 +157,7 @@ class RegionPointCrawl {
|
||||
|
||||
def o = GAODECITYCODECOLL.findOne(q)
|
||||
if(!o){
|
||||
println name +" "+adcode+" 新增"
|
||||
q.put("c_date", new Date())
|
||||
q.put("center", center)
|
||||
GAODECITYCODECOLL.save(q);
|
||||
@@ -233,4 +242,62 @@ class RegionPointCrawl {
|
||||
def count = RandomUtils.nextInt(KeyList.size())
|
||||
return KeyList.get(count);
|
||||
}
|
||||
|
||||
def statSame(){
|
||||
def dbcursor = POLYLINECOLL.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",d.city)
|
||||
q.put("region", d.region)
|
||||
def obj = OLD_POLYLINECOLL.findOne(q)
|
||||
if(obj){
|
||||
if(obj.polyline!=d.polyline){
|
||||
println index +" "+ d.city +" "+d.region+" 2次区域不同"
|
||||
}
|
||||
}else{
|
||||
println index +" "+ d.city +" "+d.region+" 没有此区域"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def regionPolyline(){
|
||||
def dbcursor = POLYLINECOLL.find(new BasicDBObject("level","district"))
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
if(rets.size()>1) {
|
||||
resultMap.type = "MultiPolygon"
|
||||
}
|
||||
println d.city+" "+d.region+" "+rets.size()
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
def center = d.center
|
||||
if(center){
|
||||
def pointArrs = center.split(",")
|
||||
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
|
||||
}
|
||||
|
||||
def fullCity = fullCityMap.get(d.city)
|
||||
map.put(fullCity.city+"|@|"+d.region, udata)
|
||||
}
|
||||
return map;
|
||||
}
|
||||
}
|
||||
|
||||
+35
-6
@@ -99,6 +99,29 @@ class ElasticUtils89 {
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
static RestHighLevelClient getClientOther(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 180);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));//
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
static RestHighLevelClient getClient89(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
@@ -174,7 +197,12 @@ class ElasticUtils89 {
|
||||
|
||||
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
|
||||
|
||||
RestHighLevelClient client = getClient(host)
|
||||
RestHighLevelClient client = null
|
||||
if(host == "192.168.0.89"){
|
||||
client = getClient(host);
|
||||
}else if(host == "192.168.0.52"){
|
||||
client = getClientOther(host);
|
||||
}
|
||||
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
@@ -190,10 +218,11 @@ class ElasticUtils89 {
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
for (int k=0; k < searchHits.size(); k++) {
|
||||
final SearchHit hit = searchHits[k];
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
// executorService.submit(new Runnable() {
|
||||
// @Override
|
||||
// void run() {
|
||||
try{
|
||||
|
||||
if(closure.getMaximumNumberOfParameters() == 1){
|
||||
def update = closure.call(hit.getSourceAsMap())
|
||||
if(update){
|
||||
@@ -208,8 +237,8 @@ class ElasticUtils89 {
|
||||
}catch (Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
})
|
||||
// }
|
||||
// })
|
||||
}
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
|
||||
+182
-26
@@ -4,6 +4,7 @@ import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
@@ -15,7 +16,19 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.*
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
@@ -24,15 +37,17 @@ import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
|
||||
/**
|
||||
* 同步小区,商铺,写字楼到89,52上
|
||||
*/
|
||||
class SeaweedDataSynch extends BaseBatch{
|
||||
|
||||
static def targetUrl = "http://192.168.0.89:9200/seaweed/"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_dishu", MONGO100_URL)
|
||||
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
|
||||
DBCollection administrative_division_2022_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
||||
|
||||
@@ -136,43 +151,103 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
|
||||
// priceMap = findAllDistrictSellPrice()
|
||||
def citys = ["潮州"]//CommonDatautils.findAllCity()
|
||||
priceMap = findAllDistrictSellPrice()
|
||||
def citys = ["福州"]//CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
for(def c : citys){
|
||||
index++
|
||||
|
||||
def dists = updating(c, "district")
|
||||
updateEs(c, dists)
|
||||
// updateEs(c, dists, "district", "192.168.0.89")
|
||||
updateEs(c, dists, "district", "192.168.0.52")
|
||||
|
||||
// def offices = updating(c, "office")
|
||||
// updateEs(c, offices, "office", "192.168.0.89")
|
||||
// updateEs(c, offices, "office", "192.168.0.52")
|
||||
|
||||
updating(c, "office")
|
||||
updating(c, "shop")
|
||||
// def shops = updating(c, "shop")
|
||||
// updateEs(c, shops, "shop", "192.168.0.89")
|
||||
// updateEs(c, offices, "shop", "192.168.0.52")
|
||||
|
||||
println index +" "+ c + " 完成"
|
||||
}
|
||||
}
|
||||
|
||||
def updateEs(city, dists){
|
||||
def updateEs(city, dists, cat, host){
|
||||
def data = fullCityMap.get(city)
|
||||
def distEsDatas = findEsSeaweedDatas(data.city)
|
||||
def resultMap = [:]
|
||||
for(def d : dists){
|
||||
def esObj = distEsDatas.get(d._id.toString())
|
||||
def map = [:]
|
||||
d.each {k,v->
|
||||
if(k!="_id"){
|
||||
def esVal = esObj.get(k)
|
||||
if(v!=esVal){
|
||||
map.put(k, v)
|
||||
def distEsDatas = findEsSeaweedDatas(data.city, cat, host)
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
def updateDatas = new CopyOnWriteArrayList()
|
||||
def ids = new CopyOnWriteArrayList()
|
||||
//更新89Es
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def d = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
ids << d._id.toString()
|
||||
def esObj = distEsDatas.get(d._id.toString())
|
||||
def map = updatingEs(d, esObj, host)
|
||||
if(map){
|
||||
updateDatas << map
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
if(map){
|
||||
resultMap.put(d._id.toString(), map)
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
println "需更改:"+updateDatas.size()
|
||||
|
||||
|
||||
updateSeaweedEs(updateDatas, host, "seaweed")
|
||||
|
||||
//将89Es中非正常小区删除
|
||||
def esDelIds = []
|
||||
distEsDatas.each{id,map->
|
||||
if(!ids.contains(id)){
|
||||
delEsData(id, host, "seaweed")
|
||||
esDelIds << id
|
||||
}
|
||||
}
|
||||
return resultMap
|
||||
println "Es 删除 ${cat}类型:"+esDelIds.size()
|
||||
//将100 mongodb中非正常小区删除
|
||||
def mongoIds = findMongoDbDatas(cat, data.city)
|
||||
|
||||
mongoIds.each{id->
|
||||
if(!ids.contains(id)){
|
||||
SEAWEED_CONN.remove(new BasicDBObject("_id", new ObjectId(id)))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def updatingEs(d, esObj, host){
|
||||
def map = [:]
|
||||
def diffMap = [:]
|
||||
d.each {k,v->
|
||||
if(v && k!="_id"){
|
||||
map.put(k, v)
|
||||
}
|
||||
if(esObj && k!="_id"){
|
||||
def esVal = esObj.get(k)
|
||||
if(esVal!=v){
|
||||
diffMap.put(k,v)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!esObj || (esObj && diffMap)){
|
||||
map.put("id", d._id.toString())
|
||||
return map
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def updating(city, cat){
|
||||
@@ -392,7 +467,7 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
map.avg_price = dataMap.avg_price
|
||||
}
|
||||
}
|
||||
|
||||
map.status = 0
|
||||
// map.update_date = org.apache.commons.httpclient.util.DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
|
||||
return map
|
||||
}
|
||||
@@ -448,12 +523,93 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
})
|
||||
}
|
||||
|
||||
def findEsSeaweedDatas(city){
|
||||
def findEsSeaweedDatas(city, cat, host){
|
||||
def map = new HashMap()
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
ElasticUtils89.executeQuery("192.168.0.89", "seaweed", queryBuilder, { String id, doc->
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("category", cat))
|
||||
ElasticUtils89.executeQuery(host, "seaweed", queryBuilder, { String id, doc->
|
||||
map.put(id, doc)
|
||||
})
|
||||
return map;
|
||||
}
|
||||
|
||||
def updateSeaweedEs(updateDatas, host, indexName){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int processedCount = 0;
|
||||
def client = null
|
||||
if(host && host == "192.168.0.89"){
|
||||
client = ElasticUtils89.getClient(host)
|
||||
}else if(host && host == "192.168.0.52"){
|
||||
client = ElasticUtils.getClient(host)
|
||||
}
|
||||
|
||||
updateDatas.each{map->
|
||||
request.add(new IndexRequest(indexName, "_doc", map.id).source(map, XContentType.JSON));
|
||||
if (request.numberOfActions() >= 100) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents "+host
|
||||
}
|
||||
request = new BulkRequest();
|
||||
}
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents "+host
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def delEsData(id, host, indexName){
|
||||
def header = [:]
|
||||
if(host && host == "192.168.0.89"){
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
}else if(host && host == "192.168.0.52"){
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
|
||||
}
|
||||
String seawaterUrl = "http://${host}:9200/${indexName}/_delete_by_query"
|
||||
def body = """
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"match": {
|
||||
"id": "${id}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
println(res)
|
||||
}
|
||||
|
||||
def findMongoDbDatas(cat, city){
|
||||
def q = new BasicDBObject("category", cat).append("city", city)
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id",1)
|
||||
def dbcursor = SEAWEED_CONN.find(q, keys)
|
||||
def mongoIds = []
|
||||
for(def d : dbcursor){
|
||||
mongoIds << d._id.toString()
|
||||
}
|
||||
return mongoIds
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.http.HttpHost
|
||||
import org.apache.http.auth.AuthScope
|
||||
import org.apache.http.auth.UsernamePasswordCredentials
|
||||
@@ -17,11 +20,13 @@ import org.elasticsearch.action.search.ClearScrollRequest
|
||||
import org.elasticsearch.action.search.SearchRequest
|
||||
import org.elasticsearch.action.search.SearchResponse
|
||||
import org.elasticsearch.action.search.SearchScrollRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestClient
|
||||
import org.elasticsearch.client.RestClientBuilder
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.search.Scroll
|
||||
import org.elasticsearch.search.SearchHit
|
||||
@@ -262,4 +267,68 @@ class ElasticUtils {
|
||||
client.close()
|
||||
return results;
|
||||
}
|
||||
|
||||
static def modify(id, map, host, indexName){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int processedCount = 0;
|
||||
def client = null
|
||||
if(host && host == "192.168.0.89"){
|
||||
client = ElasticUtils89.getClient(host)
|
||||
}else if(host && host == "192.168.0.52"){
|
||||
client = ElasticUtils.getClient(host)
|
||||
}
|
||||
|
||||
request.add(new IndexRequest(indexName, "_doc", id).source(map, XContentType.JSON));
|
||||
if (request.numberOfActions() > 100) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static def delete(id, host, indexName){
|
||||
def header = [:]
|
||||
if(host && host == "192.168.0.89"){
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
}else if(host && host == "192.168.0.52"){
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
|
||||
}
|
||||
String seawaterUrl = "http://${host}:9200/${indexName}/_delete_by_query"
|
||||
def body = """
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"match": {
|
||||
"id": "${id}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
println(res)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user