diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index 2fa4dbabb..c4ffa723f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -12,7 +12,6 @@ import org.elasticsearch.client.RequestOptions import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders - /** * 区域标准化 * 1.小区库区域标准化,包含小区,商铺,写字楼 @@ -164,7 +163,7 @@ class RegionStandardBatch extends BaseBatch{ } def findEsPoiByCity(city){ - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("district", "蕉岭县")) def rets = [] def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder) @@ -267,6 +266,75 @@ class RegionStandardBatch extends BaseBatch{ } +// def queryPoiDatasByCity(){ +// def esUrl = "http://localhost:9200/your_index/_search" +// def pageSize = 1000 // 每页返回的记录数 +// def scrollSize = '1m' // scroll上下文的生存时间 +// +//// 初始化HttpClient和ObjectMapper +// CloseableHttpClient httpClient = HttpClients.createDefault() +// ObjectMapper objectMapper = new ObjectMapper() +// +//// 初始化scroll请求 +// def scrollRequest = """ +//{ +// "size": $pageSize, +// "query": { +// "match_all": {} +// }, +// "scroll": "$scrollSize" +//} +//""" +// +//// 发送scroll初始化请求 +// HttpGet scrollInitRequest = new HttpGet(esUrl) +// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8")) +// scrollInitRequest.setHeader("Content-Type", "application/json") +// +// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest) +// HttpEntity entity = scrollInitResponse.getEntity() +// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString() +// +//// 存储所有结果 +// def allResults = [] +// +//// 发送scroll请求获取数据,直到没有更多数据为止 +// while (true) { +// def scrollRequestWithId = """ +// { +// "scroll_id": "$scrollId", +// "scroll": "$scrollSize" +// } +// """ +// +// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll") +// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8")) +// scrollGetRequest.setHeader("Content-Type", "application/json") +// +// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest) +// entity = scrollGetResponse.getEntity() +// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class) +// +// def hits = responseMap.get("hits").get("hits") +// if (hits.isEmpty()) { +// break // 没有更多数据,跳出循环 +// } +// +// // 合并当前页的数据 +// allResults.addAll(hits.map({ it.get("_source") })) +// +// // 更新scrollId以进行下一次请求 +// scrollId = responseMap.get("_scroll_id").toString() +// } +// +// // 关闭HttpClient +// httpClient.close() +// +// // 输出或处理全量数据 +// println(allResults) +// +// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。 +// } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy index e0324502c..4efd79e7f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy @@ -12,15 +12,13 @@ import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import com.rabbitmq.client.AMQP - /** * 厦门涉台园区数据同步 */ @BatchOwner(Watcher.GaoJuan) class RelatedTaiWanBatch extends BaseBatch{ - def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMM")); + def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")); DBCollection yuanqu_qiye_coll = MongoUtils.getCollectionByName("crawl", "qzd_shetai_company_detail", "192.168.0.206", 27020) def relateTaiwn_yuanqu_datas = ["福州台商投资区","杏林台商投资区","集美台商投资区","海沧台商投资区","漳州台商投资区","泉州台商投资区","漳浦台湾农民创业园","漳平台湾农民创业园","仙游台湾农民创业园","清流台湾农民创业园","福清台湾农民创业园","惠安台湾农民创业园","海峡两岸集成电路产业合作试验区(厦门、泉州产业园)","海峡两岸生技和医疗健康产业合作区","海峡两岸农业合作试验区","海峡两岸(三明)现代林业合作实验区","漳州古雷石化基地","永泰闽台农业融合发展产业园","上杭闽台农业融合发展产业园","同安闽台农业融合发展产业园","南靖闽台农业融合发展产业园","漳浦闽台农业融合发展产业园","永春闽台农业融合发展产业园","永安闽台农业融合发展产业园","邵武闽台农业融合发展产业园","平潭闽台农业融合发展产业园","南靖闽台精密机械产业园"] @@ -36,9 +34,17 @@ class RelatedTaiWanBatch extends BaseBatch{ @Override public void process(Object args) { -// createDatas() //企业库获取涉台园区企业 - updateRangeByPoint() //更改企业所在自贸区 - export() //导出数据 + + def day = new Date().getDate() + if(day == 10){ + //createDatas() //企业库获取涉台园区企业 + updateRangeByPoint() //更改企业所在自贸区 + export() //导出数据 + }else{ + exportEmptyFile() + } + + } @@ -158,4 +164,21 @@ class RelatedTaiWanBatch extends BaseBatch{ FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|"); XmBankFtpUtils.ftpUpload(fileName2, curDate) } + + def exportEmptyFile(){ + def fileName = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat" + } + FileUtils.batchCreateFileByCustomSuffix([], fileName, null, "utf-8", false, "|@|","|+|"); + XmBankFtpUtils.ftpUpload(fileName, curDate) + + def fileName2 = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat.rdy" + if(CommonDatautils.isWindow()){ + fileName2 = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat.rdy" + } + def rdy_datas = ["0"] + FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|"); + XmBankFtpUtils.ftpUpload(fileName2, curDate) + } }