From ac7f29e912507fe348d3cecc16b110b8e49912d1 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Fri, 1 Jul 2022 14:25:24 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=BB=B7=E6=A0=BC?= =?UTF-8?q?=E5=81=8F=E5=B7=AE=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/oracle/ExportExceptionPrice.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/oracle/ExportExceptionPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/oracle/ExportExceptionPrice.groovy index ce6a0b94b..9bb7c8c4b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/oracle/ExportExceptionPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/oracle/ExportExceptionPrice.groovy @@ -34,7 +34,7 @@ sqlInstance.eachRow(sql, { r -> if (year_month == '202205') { fiveMap.put(key, r['avg_price'] + "|" + r['calc_type']) } else if ((year_month == '202206')) { - fiveMap.put(key, r['avg_price'] + "|" + r['calc_type']) + sixMap.put(key, r['avg_price'] + "|" + r['calc_type']) } }) for (String key : sixMap.keySet()) { @@ -47,4 +47,4 @@ for (String key : sixMap.keySet()) { } } } -ExcelUtils.write("F:\\数据\\价格\\东莞上海价格偏差大.xlsx", result, "统计") \ No newline at end of file +ExcelUtils.write("D:\\数据\\价格\\东莞上海价格偏差大.xlsx", result, "统计") \ No newline at end of file From 09734a698dc477daa10564eb13870d4a7e153c83 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Sat, 2 Jul 2022 14:47:36 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=AC=AC=E4=B8=89?= =?UTF-8?q?=E6=96=B9=E5=B0=8F=E5=8C=BA=E6=8A=93=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/poi/GetPoiFromCommunity.groovy | 84 +++++++++++++ .../temp/shenjian/poi/entity/Community.groovy | 49 ++++++++ .../temp/shenjian/poi/entity/MyId.groovy | 17 +++ .../script/temp/shenjian/util/EsUtils.groovy | 116 ++++++++++++++++++ 4 files changed, 266 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/MyId.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy new file mode 100644 index 000000000..c466f3250 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy @@ -0,0 +1,84 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.statistics.poi.AmapPoiDetail +import com.fangjia.script.temp.shenjian.poi.entity.Community +import com.fangjia.script.temp.shenjian.util.CommonDtoUtils +import com.fangjia.script.temp.shenjian.util.EsUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import org.apache.commons.collections.CollectionUtils +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.stream.Collectors + +/** + * 从ES获取POI + */ +class GetPoiFromCommunity { + + static final ExecutorService executorService = Executors.newFixedThreadPool(10) + static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail" + // 住宅类 + static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304"; + + static void main(String[] args) { + AtomicLong atomicLong = new AtomicLong(0) + Map map = new ConcurrentHashMap<>() + def rets = this.findCommunities() + for (def retTemp : rets) { + final def ret = retTemp + executorService.submit(new Runnable() { + @Override + void run() { + final String id = ret._id // 索引ID,并非小区ID + String city = ret.city + String region = ret.region + String seaweedName = ret.name + String address = ret.address + List pois = new ArrayList<>(); + List namePois = GaoDeUtils.searchPoi(city, seaweedName, region, typecodes); + List addressPois = GaoDeUtils.searchPoi(city, address, region, typecodes); + if (CollectionUtils.isNotEmpty(namePois)) { + pois.addAll(namePois) + } + if (CollectionUtils.isNotEmpty(addressPois)) { + pois.addAll(addressPois) + } + if (CollectionUtils.isNotEmpty(pois)) { + List detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois); + String jsonStr = JSONObject.toJSONString(detailList) + HttpUtils.postForJson(URL, jsonStr) + } + synchronized (map) { + map.put(id, id) + if (map.size() % 100 == 0) { + atomicLong.addAndGet(map.size()) + List ids = map.values().stream().collect(Collectors.toList()) + EsUtils.bulkUpdateFieldByIds("community", "crawl_flag", 0, ids) + map.clear() + println(atomicLong.intValue()) + } + } + } + }) + } + executorService.shutdown() + } + + static def findCommunities(){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery() + .filter(QueryBuilders.termQuery("city", "东莞")) + .mustNot(QueryBuilders.termQuery("crawl_flag", 0)) + List communities = EsUtils.scrollQuery("community", queryBuilder, Community.class) + return communities + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy new file mode 100644 index 000000000..ba2bfaaea --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy @@ -0,0 +1,49 @@ +package com.fangjia.script.temp.shenjian.poi.entity; + +public class Community extends MyId { + private String id; + private String city; + private String region; + private String name; + private String address; + + String getId() { + return id + } + + void setId(String id) { + this.id = id + } + + String getCity() { + return city + } + + void setCity(String city) { + this.city = city + } + + String getRegion() { + return region + } + + void setRegion(String region) { + this.region = region + } + + String getName() { + return name + } + + void setName(String name) { + this.name = name + } + + String getAddress() { + return address + } + + void setAddress(String address) { + this.address = address + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/MyId.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/MyId.groovy new file mode 100644 index 000000000..53bd1b98b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/MyId.groovy @@ -0,0 +1,17 @@ +package com.fangjia.script.temp.shenjian.poi.entity + +class MyId { + + /** + * 索引id + */ + String _id; + + String get_id() { + return _id + } + + void set_id(String _id) { + this._id = _id + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy new file mode 100644 index 000000000..ee66cfce7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy @@ -0,0 +1,116 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils +import com.fangjia.script.temp.shenjian.poi.entity.MyId +import org.elasticsearch.action.DocWriteRequest +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.search.* +import org.elasticsearch.action.update.UpdateRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.client.RestHighLevelClient +import org.elasticsearch.common.unit.TimeValue +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.search.Scroll +import org.elasticsearch.search.SearchHit +import org.elasticsearch.search.builder.SearchSourceBuilder +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * @author shenjian + * @since 2022/07/02 + */ +class EsUtils { + + private static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient() + private static final Logger logger = LoggerFactory.getLogger(EsUtils.class) + + /** + * 滚动方式查询大批量es数据 + * + * @param index 索引 + * @param boolQuery 查询语句 + * @param clazz 要返回的实体类型 + * @param + * @return + */ + public static List scrollQuery(String index, QueryBuilder boolQuery, Class clazz) { + List result = new ArrayList<>(); + // 存活时间,当索引数据量特别大时,出现超时可能性大,此值适当调大 + Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30L)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(boolQuery); + // TEST 用!!!!!!!!!!!!!!!!!!!!!! + //searchSourceBuilder.size(1); + searchSourceBuilder.size(10000); + SearchRequest searchRequest = new SearchRequest().indices(index).scroll(scroll).source(searchSourceBuilder); + SearchResponse searchResponse = null; + try { + searchResponse = ES_CLIENT.search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("查询es数据异常:{}", e); + } + String scrollId = searchResponse.getScrollId(); + SearchHit[] searchHits = searchResponse.getHits().getHits(); + if (searchHits != null) { + for (SearchHit searchHit : searchHits) { + T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz); + t.set_id(searchHit.getId()) + result.add(t); + } + } + // 遍历搜索命中的数据,直到没有数据 + while (searchHits != null && searchHits.length > 0) { + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + try { + searchResponse = ES_CLIENT.scroll(scrollRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("查询es数据异常:{}", e); + } + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + if (searchHits != null && searchHits.length > 0) { + for (SearchHit searchHit : searchHits) { + T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz); + t.set_id(searchHit.getId()) + result.add(t); + } + } + } + // clean scroll + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + ClearScrollResponse clearScrollResponse = null; + try { + clearScrollResponse = ES_CLIENT.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + } catch (IOException e) {} + boolean succeeded = clearScrollResponse.isSucceeded(); + if (succeeded) { + return result; + } + return null; + } + + + /** + * 批量更新es某个字段值 + * @param index 索引名 + * @param field 字段名 + * @param val 字段新值 + * @param ids 要更新的文档id + */ + public static void bulkUpdateFieldByIds(String index, String field, Object val, List ids) { + if (ids.size() > 1000) { + throw new RuntimeException("批量更新数量过多") + } + BulkRequest bulkRequest = new BulkRequest(); + for (String id : ids) { + Map doc = new HashMap<>(); + doc.put(field, val) + bulkRequest.add(new UpdateRequest(index, id).doc(doc)); + } + ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT); + } +} From acc809b3ae930177f2dfcf0e1dac686277e13bec Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Sat, 2 Jul 2022 15:35:04 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=A4=9A=E7=BA=BF?= =?UTF-8?q?=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/GetPoiFromCommunity.groovy | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy index c466f3250..1aab2c40b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy @@ -14,6 +14,7 @@ import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong @@ -33,6 +34,7 @@ class GetPoiFromCommunity { AtomicLong atomicLong = new AtomicLong(0) Map map = new ConcurrentHashMap<>() def rets = this.findCommunities() + CountDownLatch countDownLatch = new CountDownLatch(rets.size()) for (def retTemp : rets) { final def ret = retTemp executorService.submit(new Runnable() { @@ -59,6 +61,7 @@ class GetPoiFromCommunity { } synchronized (map) { map.put(id, id) + countDownLatch.countDown() if (map.size() % 100 == 0) { atomicLong.addAndGet(map.size()) List ids = map.values().stream().collect(Collectors.toList()) @@ -70,7 +73,15 @@ class GetPoiFromCommunity { } }) } - executorService.shutdown() + countDownLatch.await() + if (map.size() > 0) { + atomicLong.addAndGet(map.size()) + List ids = map.values().stream().collect(Collectors.toList()) + EsUtils.bulkUpdateFieldByIds("community", "crawl_flag", 0, ids) + map.clear() + println(atomicLong.intValue()) + } + System.exit(1) } static def findCommunities(){ From 2fec35aa97996e2f9a3b721cdb622f90ff8cad9d Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Sat, 2 Jul 2022 15:49:24 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E4=BC=98=E5=8C=96es=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/poi/GetPoiFromCommunity.groovy | 1 - .../com/fangjia/script/temp/shenjian/util/EsUtils.groovy | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy index 1aab2c40b..f93b844d7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy @@ -86,7 +86,6 @@ class GetPoiFromCommunity { static def findCommunities(){ QueryBuilder queryBuilder = QueryBuilders.boolQuery() - .filter(QueryBuilders.termQuery("city", "东莞")) .mustNot(QueryBuilders.termQuery("crawl_flag", 0)) List communities = EsUtils.scrollQuery("community", queryBuilder, Community.class) return communities diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy index ee66cfce7..b158c9351 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy @@ -35,7 +35,7 @@ class EsUtils { * @param * @return */ - public static List scrollQuery(String index, QueryBuilder boolQuery, Class clazz) { + public static List scrollQuery(String index, QueryBuilder boolQuery, Class clazz, def limitSize=300000) { List result = new ArrayList<>(); // 存活时间,当索引数据量特别大时,出现超时可能性大,此值适当调大 Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30L)); @@ -61,7 +61,7 @@ class EsUtils { } } // 遍历搜索命中的数据,直到没有数据 - while (searchHits != null && searchHits.length > 0) { + while (searchHits != null && searchHits.length > 0 && result.size() < limitSize) { SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); scrollRequest.scroll(scroll); try {