From a6ec66ceea0ec0fed77f228211fe3841bff45cf2 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 8 Apr 2025 14:43:15 +0800 Subject: [PATCH 1/6] =?UTF-8?q?es=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/util/ElasticUtils.groovy | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy index 6aba12207..a611fedc8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy @@ -1,5 +1,8 @@ package com.fangjia.script.util +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.gj.xiamen.ElasticUtils89 +import net.sf.json.JSONObject import org.apache.http.HttpHost import org.apache.http.auth.AuthScope import org.apache.http.auth.UsernamePasswordCredentials @@ -17,11 +20,13 @@ import org.elasticsearch.action.search.ClearScrollRequest import org.elasticsearch.action.search.SearchRequest import org.elasticsearch.action.search.SearchResponse import org.elasticsearch.action.search.SearchScrollRequest +import org.elasticsearch.action.support.WriteRequest import org.elasticsearch.client.RequestOptions import org.elasticsearch.client.RestClient import org.elasticsearch.client.RestClientBuilder import org.elasticsearch.client.RestHighLevelClient import org.elasticsearch.common.unit.TimeValue +import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.search.Scroll import org.elasticsearch.search.SearchHit @@ -262,4 +267,68 @@ class ElasticUtils { client.close() return results; } + + static def modify(id, map, host, indexName){ + BulkRequest request = new BulkRequest(); + int processedCount = 0; + def client = null + if(host && host == "192.168.0.89"){ + client = ElasticUtils89.getClient(host) + }else if(host && host == "192.168.0.52"){ + client = ElasticUtils.getClient(host) + } + + request.add(new IndexRequest(indexName, "_doc", id).source(map, XContentType.JSON)); + if (request.numberOfActions() > 100) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + } + if (request.numberOfActions() > 0) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + } + } + + static def delete(id, host, indexName){ + def header = [:] + if(host && host == "192.168.0.89"){ + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString() + }else if(host && host == "192.168.0.52"){ + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString() + } + String seawaterUrl = "http://${host}:9200/${indexName}/_delete_by_query" + def body = """ + { + "query": { + "bool": { + "must": [ + { + "match": { + "id": "${id}" + } + } + ] + } + } + } + """ + HttpRequest request = new HttpRequest() + String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + println(res) + } + } From d9a4f8de07bd1f6b1058e7280915eb74e2ff4f0f Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 8 Apr 2025 14:43:51 +0800 Subject: [PATCH 2/6] =?UTF-8?q?=E5=90=8C=E6=AD=A5=E5=BC=80=E5=8F=91?= =?UTF-8?q?=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/SynchDevDataBatch.groovy | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchDevDataBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchDevDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchDevDataBatch.groovy new file mode 100644 index 000000000..dc96e2246 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchDevDataBatch.groovy @@ -0,0 +1,119 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.ConcurrentHashMap + +/** + * 将43上的开发区同步到mongo-100, es-89, es-52 + */ +class SynchDevDataBatch extends BaseBatch{ + + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL) + + DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com") + + static main(args) { + runBatch(new SynchDevDataBatch(), args); + } + + @Override + public void process(Object args) { + def devMap = synchDevTo100() +// synchDevToEs(devMap) +// synchDevEs(devMap, "192.168.0.89") + synchDevEs(devMap, "192.168.0.52") + } + + def synchDevTo100(){ + def dbcursor = DEVELOPMENT_ZONE_43_COLL.find() + def rets = [] + def devMap = [:] + for(def r : dbcursor){ + rets << r + devMap.put(r._id.toString(), r) + } + //删除旧的,再同步新的 + development_zone_CONN.remove(new BasicDBObject()) + development_zone_CONN.insert(rets) + return devMap; + } + + def synchDevToEs(devMap){ +// def esDevs = findEsDevDatas() +// def ids = [] +// for(def dev : devDatas){ +// ids << dev._id.toString() +// def esObj = esDevs.get(dev._id.toString()) +// def updateValMap = [:] +// if(esObj){ +// dev.each{k,v-> +// if(k!="id" && k!="_id"){ +// def val = dev.get(k) +// if(val!=v){ +// updateValMap.put(k,val) +// } +// } +// } +// } +// if(!esObj || (esObj && updateValMap)){ +// ElasticUtils.modify(dev._id.toString(), dev, "192.168.0.89", "development_zone") +// ElasticUtils.modify(dev._id.toString(), dev, "192.168.0.52", "development_zone") +// } +// } +// +// def esDelIds = [] +// esDevs.each{id,map-> +// if(!ids.contains(id)){ +// ElasticUtils.delete(id, "192.168.0.89", "development_zone") +// ElasticUtils.delete(id, "192.168.0.52", "development_zone") +// esDelIds << id +// } +// } +// println "Es删除开发区数量:"+esDelIds.size() + } + + + def synchDevEs(devMap, host){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery() + + ElasticUtils89.executeQuery(host, "development_zone", queryBuilder, { String id, doc-> + def mongoObj = devMap.get(id) + def updateFieldCount = 0 + mongoObj.each{k,v-> + if(!["id","_id" ,"build_date" ,"create_time","bd_shape","img_url","operator","operate_result","m_date","township","block"].contains(k)){ + def val = doc.get(k) + if(val!=v){ + println doc.name +" 修改 "+k+" "+ v+" "+id +// updateValMap.put(k,val) + updateFieldCount++ + doc.put(k,v) + } + } + } + if(mongoObj && updateFieldCount){ + //进行修改 + return true + }else{ + if(!mongoObj){ + println "删除:"+id +" "+doc.name + ElasticUtils.delete(id, host, "development_zone") + return false + } + } + }) + } + + + + + +} From 076e72419925a658c4781bf9c636d6c602efb84f Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 8 Apr 2025 14:45:15 +0800 Subject: [PATCH 3/6] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=90=8C=E6=AD=A5100=2089=2052?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/xiamen/SeaweedDataSynch.groovy | 208 +++++++++++++++--- 1 file changed, 182 insertions(+), 26 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy index b5f10fe8d..65bbe11f6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy @@ -4,6 +4,7 @@ import com.fangjia.core.util.GeoUtils import com.fangjia.core.util.HttpRequest import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils + import com.fangjia.script.util.DateUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils @@ -15,7 +16,19 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.AMQP import net.sf.json.JSONObject + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.* +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + import org.bson.types.ObjectId +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.support.WriteRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders @@ -24,15 +37,17 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong -import com.fangjia.script.common.DataHandleUtils +/** + * 同步小区,商铺,写字楼到89,52上 + */ class SeaweedDataSynch extends BaseBatch{ static def targetUrl = "http://192.168.0.89:9200/seaweed/" public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; - DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL) + DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_dishu", MONGO100_URL) DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL) DBCollection administrative_division_2022_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL) @@ -136,43 +151,103 @@ class SeaweedDataSynch extends BaseBatch{ fullCityMap = CommonDatautils.queryFullNameCity() mcaDistMap = CommonDatautils.queryDistNameCityByMapping() -// priceMap = findAllDistrictSellPrice() - def citys = ["潮州"]//CommonDatautils.findAllCity() + priceMap = findAllDistrictSellPrice() + def citys = ["福州"]//CommonDatautils.findAllCity() def index = 0 for(def c : citys){ index++ def dists = updating(c, "district") - updateEs(c, dists) +// updateEs(c, dists, "district", "192.168.0.89") + updateEs(c, dists, "district", "192.168.0.52") +// def offices = updating(c, "office") +// updateEs(c, offices, "office", "192.168.0.89") +// updateEs(c, offices, "office", "192.168.0.52") - updating(c, "office") - updating(c, "shop") +// def shops = updating(c, "shop") +// updateEs(c, shops, "shop", "192.168.0.89") +// updateEs(c, offices, "shop", "192.168.0.52") println index +" "+ c + " 完成" } } - def updateEs(city, dists){ + def updateEs(city, dists, cat, host){ def data = fullCityMap.get(city) - def distEsDatas = findEsSeaweedDatas(data.city) - def resultMap = [:] - for(def d : dists){ - def esObj = distEsDatas.get(d._id.toString()) - def map = [:] - d.each {k,v-> - if(k!="_id"){ - def esVal = esObj.get(k) - if(v!=esVal){ - map.put(k, v) + def distEsDatas = findEsSeaweedDatas(data.city, cat, host) + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + def updateDatas = new CopyOnWriteArrayList() + def ids = new CopyOnWriteArrayList() + //更新89Es + for(def i=0;i + if(!ids.contains(id)){ + delEsData(id, host, "seaweed") + esDelIds << id } } - return resultMap + println "Es 删除 ${cat}类型:"+esDelIds.size() + //将100 mongodb中非正常小区删除 + def mongoIds = findMongoDbDatas(cat, data.city) + + mongoIds.each{id-> + if(!ids.contains(id)){ + SEAWEED_CONN.remove(new BasicDBObject("_id", new ObjectId(id))) + } + } + + } + + def updatingEs(d, esObj, host){ + def map = [:] + def diffMap = [:] + d.each {k,v-> + if(v && k!="_id"){ + map.put(k, v) + } + if(esObj && k!="_id"){ + def esVal = esObj.get(k) + if(esVal!=v){ + diffMap.put(k,v) + } + } + } + if(!esObj || (esObj && diffMap)){ + map.put("id", d._id.toString()) + return map + } + return null } def updating(city, cat){ @@ -392,7 +467,7 @@ class SeaweedDataSynch extends BaseBatch{ map.avg_price = dataMap.avg_price } } - + map.status = 0 // map.update_date = org.apache.commons.httpclient.util.DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T") return map } @@ -448,12 +523,93 @@ class SeaweedDataSynch extends BaseBatch{ }) } - def findEsSeaweedDatas(city){ + def findEsSeaweedDatas(city, cat, host){ def map = new HashMap() - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) - ElasticUtils89.executeQuery("192.168.0.89", "seaweed", queryBuilder, { String id, doc-> + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("category", cat)) + ElasticUtils89.executeQuery(host, "seaweed", queryBuilder, { String id, doc-> map.put(id, doc) }) return map; } + + def updateSeaweedEs(updateDatas, host, indexName){ + BulkRequest request = new BulkRequest(); + int processedCount = 0; + def client = null + if(host && host == "192.168.0.89"){ + client = ElasticUtils89.getClient(host) + }else if(host && host == "192.168.0.52"){ + client = ElasticUtils.getClient(host) + } + + updateDatas.each{map-> + request.add(new IndexRequest(indexName, "_doc", map.id).source(map, XContentType.JSON)); + if (request.numberOfActions() >= 100) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents "+host + } + request = new BulkRequest(); + } + } + + if (request.numberOfActions() > 0) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents "+host + } + } + } + + def delEsData(id, host, indexName){ + def header = [:] + if(host && host == "192.168.0.89"){ + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString() + }else if(host && host == "192.168.0.52"){ + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString() + } + String seawaterUrl = "http://${host}:9200/${indexName}/_delete_by_query" + def body = """ + { + "query": { + "bool": { + "must": [ + { + "match": { + "id": "${id}" + } + } + ] + } + } + } + """ + HttpRequest request = new HttpRequest() + String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + println(res) + } + + def findMongoDbDatas(cat, city){ + def q = new BasicDBObject("category", cat).append("city", city) + def keys = new BasicDBObject() + keys.put("_id",1) + def dbcursor = SEAWEED_CONN.find(q, keys) + def mongoIds = [] + for(def d : dbcursor){ + mongoIds << d._id.toString() + } + return mongoIds + } + + } From 961887256046b35527533450bb7d6376497eaffd Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 8 Apr 2025 14:45:55 +0800 Subject: [PATCH 4/6] =?UTF-8?q?=E5=BC=80=E5=8F=91=E5=8C=BA=E5=9B=B4?= =?UTF-8?q?=E6=A0=8F=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/data_handle/AddDevShapeData.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy index 19934afd6..a2d1c56b8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy @@ -189,13 +189,13 @@ class AddDevShapeData { def run(){ - def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\218\\' + def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\临时-没付费\\' def dir = new File(directoryPath) if (dir.isDirectory()) { dir.eachFileRecurse(FileType.FILES) { file -> process(file.absolutePath) } -// addDevInfo("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx") + addDevInfo("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx") } else { println "The specified path is not a directory." } From 780f1dccb135797f93d5560b14d2d98fe1b9507c Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 8 Apr 2025 14:47:28 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=E6=8C=87=E5=AE=9Ahost?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/xiamen/ElasticUtils89.groovy | 41 ++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy index a401bf480..b1b772b92 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy @@ -99,6 +99,29 @@ class ElasticUtils89 { return new RestHighLevelClient(builder) } + static RestHighLevelClient getClientOther(String host) { + RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http")); + builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { + @Override + RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) { + requestConfigBuilder.setConnectionRequestTimeout(1000 * 60); + requestConfigBuilder.setSocketTimeout(1000 * 180); + return requestConfigBuilder; + } + }) + builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + @Override + HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) { + httpAsyncClientBuilder.disableAuthCaching(); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));// + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return httpAsyncClientBuilder; + } + }) + return new RestHighLevelClient(builder) + } + static RestHighLevelClient getClient89(String host) { RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http")); @@ -174,7 +197,12 @@ class ElasticUtils89 { static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){ - RestHighLevelClient client = getClient(host) + RestHighLevelClient client = null + if(host == "192.168.0.89"){ + client = getClient(host); + }else if(host == "192.168.0.52"){ + client = getClientOther(host); + } BulkProcessor bulkProcessor = createBulkProcessor(host, client) final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L)) SearchRequest searchRequest = new SearchRequest(indexName); @@ -190,10 +218,11 @@ class ElasticUtils89 { while (searchHits != null && searchHits.length > 0) { for (int k=0; k < searchHits.size(); k++) { final SearchHit hit = searchHits[k]; - executorService.submit(new Runnable() { - @Override - void run() { +// executorService.submit(new Runnable() { +// @Override +// void run() { try{ + if(closure.getMaximumNumberOfParameters() == 1){ def update = closure.call(hit.getSourceAsMap()) if(update){ @@ -208,8 +237,8 @@ class ElasticUtils89 { }catch (Exception e){ e.printStackTrace() } - } - }) +// } +// }) } SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); scrollRequest.scroll(scroll); From 0a99a465d0d3861e9316775af143060f04d7ca2c Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 8 Apr 2025 14:48:29 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=E5=8C=BA=E5=9F=9F=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/location/RegionPoint.groovy | 97 ++++++++++++++++--- 1 file changed, 82 insertions(+), 15 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy index e80dd9c35..030658e63 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy @@ -5,6 +5,7 @@ import com.fangjia.core.RegionData import com.fangjia.core.util.HttpRequest import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection import net.sf.json.JSONObject import org.apache.commons.lang.math.RandomUtils @@ -17,8 +18,9 @@ import java.util.regex.Pattern */ class RegionPointCrawl { - static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") + static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline_2025") static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code") + static DBCollection OLD_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") static main(args) { def task = new RegionPointCrawl(); @@ -26,16 +28,19 @@ class RegionPointCrawl { //4个直辖市 北京市、上海市、天津市、重庆市 //5个自治区 "内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区" // def provinces = ["河北省", "山西省","吉林省","辽宁省", "黑龙江省", "陕西省", "甘肃省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省", "云南省"] - def provinces = ["甘肃省"] - def index = 0 - for(def p : provinces){ - index ++ - println index +"/"+provinces.size() +" "+p - task.crawlCityAndDistrictCode(p); - println "城市 区域 code 完成" - task.run(p); - println "区域围栏 完成" - } + //已跑:甘肃省,河北省,山西省,"吉林省","辽宁省", "黑龙江省", "陕西省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省","云南省" +// def provinces = ["北京市","上海市","天津市","重庆市" ] +// def index = 0 +// for(def p : provinces){ +// index ++ +// println index +"/"+provinces.size() +" "+p +// task.crawlCityAndDistrictCode(p); +// println "城市 区域 code 完成" +// task.run(p); +// println "区域围栏 完成" +// } + + task.statSame() } @@ -47,7 +52,10 @@ class RegionPointCrawl { def cityCodes = findCityCode(pro); if(cityCodes){ for(def cityMap : cityCodes){ - def codeMap = findAllDistrictCode(cityMap.city_code); + def codeMap = findAllDistrictCode(cityMap.city_code, "district"); +// if(!codeMap){ +// codeMap = findAllDistrictCode(cityMap.city_code, "street"); +// } def index = 0 for(def m : codeMap){ index ++ @@ -93,7 +101,7 @@ class RegionPointCrawl { def saveGaoDeRegionCode(province, cityName,districts){ if(districts){ for(def dist : districts){ - if(dist.level=="district" || cityName=="东莞市"){ + if(dist.level=="district" || dist.level=="street" ){ saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level); } } @@ -125,10 +133,10 @@ class RegionPointCrawl { return new BasicDBObject("$regex", pattern); } - def findAllDistrictCode(def citycode){ + def findAllDistrictCode(def citycode, def level){ def q = new BasicDBObject() q.put("city_code", citycode) - q.put("level", "district") + q.put("level", level) def dbcursor = GAODECITYCODECOLL.find(q) def rets = [] for(def d : dbcursor){ @@ -149,6 +157,7 @@ class RegionPointCrawl { def o = GAODECITYCODECOLL.findOne(q) if(!o){ + println name +" "+adcode+" 新增" q.put("c_date", new Date()) q.put("center", center) GAODECITYCODECOLL.save(q); @@ -233,4 +242,62 @@ class RegionPointCrawl { def count = RandomUtils.nextInt(KeyList.size()) return KeyList.get(count); } + + def statSame(){ + def dbcursor = POLYLINECOLL.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + + def index = 0 + for(def d : dbcursor){ + index++ + def q = new BasicDBObject() + q.put("city",d.city) + q.put("region", d.region) + def obj = OLD_POLYLINECOLL.findOne(q) + if(obj){ + if(obj.polyline!=d.polyline){ + println index +" "+ d.city +" "+d.region+" 2次区域不同" + } + }else{ + println index +" "+ d.city +" "+d.region+" 没有此区域" + } + } + } + + + def regionPolyline(){ + def dbcursor = POLYLINECOLL.find(new BasicDBObject("level","district")) + def map = [:] + for(def d : dbcursor){ + def resultMap = [:] + resultMap.type = "polygon" + def arrs = d.polyline.split("\\|") + def rets = [] + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + for(def pointStr : points){ + def pointArr = pointStr.split(",") + pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()] + } + rets << pointList + } + resultMap.coordinates = rets + if(rets.size()>1) { + resultMap.type = "MultiPolygon" + } + println d.city+" "+d.region+" "+rets.size() + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + def center = d.center + if(center){ + def pointArrs = center.split(",") + udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()]) + } + + def fullCity = fullCityMap.get(d.city) + map.put(fullCity.city+"|@|"+d.region, udata) + } + return map; + } }