Merge branch 'script_gj' into 'master'
Script gj See merge request !989
This commit is contained in:
+70
-2
@@ -12,7 +12,6 @@ import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
/**
|
||||
* 区域标准化
|
||||
* 1.小区库区域标准化,包含小区,商铺,写字楼
|
||||
@@ -164,7 +163,7 @@ class RegionStandardBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def findEsPoiByCity(city){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("district", "蕉岭县"))
|
||||
def rets = []
|
||||
|
||||
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder)
|
||||
@@ -267,6 +266,75 @@ class RegionStandardBatch extends BaseBatch{
|
||||
|
||||
}
|
||||
|
||||
// def queryPoiDatasByCity(){
|
||||
// def esUrl = "http://localhost:9200/your_index/_search"
|
||||
// def pageSize = 1000 // 每页返回的记录数
|
||||
// def scrollSize = '1m' // scroll上下文的生存时间
|
||||
//
|
||||
//// 初始化HttpClient和ObjectMapper
|
||||
// CloseableHttpClient httpClient = HttpClients.createDefault()
|
||||
// ObjectMapper objectMapper = new ObjectMapper()
|
||||
//
|
||||
//// 初始化scroll请求
|
||||
// def scrollRequest = """
|
||||
//{
|
||||
// "size": $pageSize,
|
||||
// "query": {
|
||||
// "match_all": {}
|
||||
// },
|
||||
// "scroll": "$scrollSize"
|
||||
//}
|
||||
//"""
|
||||
//
|
||||
//// 发送scroll初始化请求
|
||||
// HttpGet scrollInitRequest = new HttpGet(esUrl)
|
||||
// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8"))
|
||||
// scrollInitRequest.setHeader("Content-Type", "application/json")
|
||||
//
|
||||
// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest)
|
||||
// HttpEntity entity = scrollInitResponse.getEntity()
|
||||
// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString()
|
||||
//
|
||||
//// 存储所有结果
|
||||
// def allResults = []
|
||||
//
|
||||
//// 发送scroll请求获取数据,直到没有更多数据为止
|
||||
// while (true) {
|
||||
// def scrollRequestWithId = """
|
||||
// {
|
||||
// "scroll_id": "$scrollId",
|
||||
// "scroll": "$scrollSize"
|
||||
// }
|
||||
// """
|
||||
//
|
||||
// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll")
|
||||
// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8"))
|
||||
// scrollGetRequest.setHeader("Content-Type", "application/json")
|
||||
//
|
||||
// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest)
|
||||
// entity = scrollGetResponse.getEntity()
|
||||
// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class)
|
||||
//
|
||||
// def hits = responseMap.get("hits").get("hits")
|
||||
// if (hits.isEmpty()) {
|
||||
// break // 没有更多数据,跳出循环
|
||||
// }
|
||||
//
|
||||
// // 合并当前页的数据
|
||||
// allResults.addAll(hits.map({ it.get("_source") }))
|
||||
//
|
||||
// // 更新scrollId以进行下一次请求
|
||||
// scrollId = responseMap.get("_scroll_id").toString()
|
||||
// }
|
||||
//
|
||||
// // 关闭HttpClient
|
||||
// httpClient.close()
|
||||
//
|
||||
// // 输出或处理全量数据
|
||||
// println(allResults)
|
||||
//
|
||||
// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。
|
||||
// }
|
||||
|
||||
|
||||
}
|
||||
|
||||
+29
-6
@@ -12,15 +12,13 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
|
||||
/**
|
||||
* 厦门涉台园区数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class RelatedTaiWanBatch extends BaseBatch{
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMM"));
|
||||
def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"));
|
||||
|
||||
DBCollection yuanqu_qiye_coll = MongoUtils.getCollectionByName("crawl", "qzd_shetai_company_detail", "192.168.0.206", 27020)
|
||||
def relateTaiwn_yuanqu_datas = ["福州台商投资区","杏林台商投资区","集美台商投资区","海沧台商投资区","漳州台商投资区","泉州台商投资区","漳浦台湾农民创业园","漳平台湾农民创业园","仙游台湾农民创业园","清流台湾农民创业园","福清台湾农民创业园","惠安台湾农民创业园","海峡两岸集成电路产业合作试验区(厦门、泉州产业园)","海峡两岸生技和医疗健康产业合作区","海峡两岸农业合作试验区","海峡两岸(三明)现代林业合作实验区","漳州古雷石化基地","永泰闽台农业融合发展产业园","上杭闽台农业融合发展产业园","同安闽台农业融合发展产业园","南靖闽台农业融合发展产业园","漳浦闽台农业融合发展产业园","永春闽台农业融合发展产业园","永安闽台农业融合发展产业园","邵武闽台农业融合发展产业园","平潭闽台农业融合发展产业园","南靖闽台精密机械产业园"]
|
||||
@@ -36,9 +34,17 @@ class RelatedTaiWanBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
// createDatas() //企业库获取涉台园区企业
|
||||
updateRangeByPoint() //更改企业所在自贸区
|
||||
export() //导出数据
|
||||
|
||||
def day = new Date().getDate()
|
||||
if(day == 10){
|
||||
//createDatas() //企业库获取涉台园区企业
|
||||
updateRangeByPoint() //更改企业所在自贸区
|
||||
export() //导出数据
|
||||
}else{
|
||||
exportEmptyFile()
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -158,4 +164,21 @@ class RelatedTaiWanBatch extends BaseBatch{
|
||||
FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|");
|
||||
XmBankFtpUtils.ftpUpload(fileName2, curDate)
|
||||
}
|
||||
|
||||
def exportEmptyFile(){
|
||||
def fileName = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat"
|
||||
}
|
||||
FileUtils.batchCreateFileByCustomSuffix([], fileName, null, "utf-8", false, "|@|","|+|");
|
||||
XmBankFtpUtils.ftpUpload(fileName, curDate)
|
||||
|
||||
def fileName2 = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat.rdy"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName2 = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat.rdy"
|
||||
}
|
||||
def rdy_datas = ["0"]
|
||||
FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|");
|
||||
XmBankFtpUtils.ftpUpload(fileName2, curDate)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user