From 2be600c1400147de86032d524d6b21483fe8e69f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 10 Nov 2024 10:29:53 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E6=94=B9=E4=B8=BA=E6=AF=8F=E6=97=A5?= =?UTF-8?q?=E8=BF=90=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/xiamen/RelatedTaiWanBatch.groovy | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy index e0324502c..4efd79e7f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy @@ -12,15 +12,13 @@ import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import com.rabbitmq.client.AMQP - /** * 厦门涉台园区数据同步 */ @BatchOwner(Watcher.GaoJuan) class RelatedTaiWanBatch extends BaseBatch{ - def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMM")); + def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")); DBCollection yuanqu_qiye_coll = MongoUtils.getCollectionByName("crawl", "qzd_shetai_company_detail", "192.168.0.206", 27020) def relateTaiwn_yuanqu_datas = ["福州台商投资区","杏林台商投资区","集美台商投资区","海沧台商投资区","漳州台商投资区","泉州台商投资区","漳浦台湾农民创业园","漳平台湾农民创业园","仙游台湾农民创业园","清流台湾农民创业园","福清台湾农民创业园","惠安台湾农民创业园","海峡两岸集成电路产业合作试验区(厦门、泉州产业园)","海峡两岸生技和医疗健康产业合作区","海峡两岸农业合作试验区","海峡两岸(三明)现代林业合作实验区","漳州古雷石化基地","永泰闽台农业融合发展产业园","上杭闽台农业融合发展产业园","同安闽台农业融合发展产业园","南靖闽台农业融合发展产业园","漳浦闽台农业融合发展产业园","永春闽台农业融合发展产业园","永安闽台农业融合发展产业园","邵武闽台农业融合发展产业园","平潭闽台农业融合发展产业园","南靖闽台精密机械产业园"] @@ -36,9 +34,17 @@ class RelatedTaiWanBatch extends BaseBatch{ @Override public void process(Object args) { -// createDatas() //企业库获取涉台园区企业 - updateRangeByPoint() //更改企业所在自贸区 - export() //导出数据 + + def day = new Date().getDate() + if(day == 10){ + //createDatas() //企业库获取涉台园区企业 + updateRangeByPoint() //更改企业所在自贸区 + export() //导出数据 + }else{ + exportEmptyFile() + } + + } @@ -158,4 +164,21 @@ class RelatedTaiWanBatch extends BaseBatch{ FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|"); XmBankFtpUtils.ftpUpload(fileName2, curDate) } + + def exportEmptyFile(){ + def fileName = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat" + } + FileUtils.batchCreateFileByCustomSuffix([], fileName, null, "utf-8", false, "|@|","|+|"); + XmBankFtpUtils.ftpUpload(fileName, curDate) + + def fileName2 = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat.rdy" + if(CommonDatautils.isWindow()){ + fileName2 = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat.rdy" + } + def rdy_datas = ["0"] + FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|"); + XmBankFtpUtils.ftpUpload(fileName2, curDate) + } } From 8deebf69c5d938b7a000478ea2c593d943c2417a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 10 Nov 2024 10:32:24 +0800 Subject: [PATCH 2/2] updadte --- .../gj/data_handle/RegionStandardBatch.groovy | 76 ++++++++++++++++++- 1 file changed, 72 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index aeac02b63..fc55cbd77 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -12,7 +12,6 @@ import org.elasticsearch.client.RequestOptions import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders - /** * 区域标准化 * 1.小区库区域标准化,包含小区,商铺,写字楼 @@ -163,7 +162,7 @@ class RegionStandardBatch extends BaseBatch{ } def findEsPoiByCity(city){ - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("district", "蕉岭县")) def rets = [] def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder) @@ -200,7 +199,7 @@ class RegionStandardBatch extends BaseBatch{ def m = [:] m.put(d.id, ["district":updateRegion]) results << m - if(results.size()%1==0){ + if(results.size()%100==0){ updateRegionByEs(results) results.clear() } @@ -220,7 +219,7 @@ class RegionStandardBatch extends BaseBatch{ } } - if(results.size()%1>0){ + if(results.size()>0){ updateRegionByEs(results) results.clear() } @@ -265,6 +264,75 @@ class RegionStandardBatch extends BaseBatch{ } +// def queryPoiDatasByCity(){ +// def esUrl = "http://localhost:9200/your_index/_search" +// def pageSize = 1000 // 每页返回的记录数 +// def scrollSize = '1m' // scroll上下文的生存时间 +// +//// 初始化HttpClient和ObjectMapper +// CloseableHttpClient httpClient = HttpClients.createDefault() +// ObjectMapper objectMapper = new ObjectMapper() +// +//// 初始化scroll请求 +// def scrollRequest = """ +//{ +// "size": $pageSize, +// "query": { +// "match_all": {} +// }, +// "scroll": "$scrollSize" +//} +//""" +// +//// 发送scroll初始化请求 +// HttpGet scrollInitRequest = new HttpGet(esUrl) +// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8")) +// scrollInitRequest.setHeader("Content-Type", "application/json") +// +// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest) +// HttpEntity entity = scrollInitResponse.getEntity() +// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString() +// +//// 存储所有结果 +// def allResults = [] +// +//// 发送scroll请求获取数据,直到没有更多数据为止 +// while (true) { +// def scrollRequestWithId = """ +// { +// "scroll_id": "$scrollId", +// "scroll": "$scrollSize" +// } +// """ +// +// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll") +// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8")) +// scrollGetRequest.setHeader("Content-Type", "application/json") +// +// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest) +// entity = scrollGetResponse.getEntity() +// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class) +// +// def hits = responseMap.get("hits").get("hits") +// if (hits.isEmpty()) { +// break // 没有更多数据,跳出循环 +// } +// +// // 合并当前页的数据 +// allResults.addAll(hits.map({ it.get("_source") })) +// +// // 更新scrollId以进行下一次请求 +// scrollId = responseMap.get("_scroll_id").toString() +// } +// +// // 关闭HttpClient +// httpClient.close() +// +// // 输出或处理全量数据 +// println(allResults) +// +// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。 +// } }