Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj
This commit is contained in:
+2
-2
@@ -34,7 +34,7 @@ sqlInstance.eachRow(sql, { r ->
|
||||
if (year_month == '202205') {
|
||||
fiveMap.put(key, r['avg_price'] + "|" + r['calc_type'])
|
||||
} else if ((year_month == '202206')) {
|
||||
fiveMap.put(key, r['avg_price'] + "|" + r['calc_type'])
|
||||
sixMap.put(key, r['avg_price'] + "|" + r['calc_type'])
|
||||
}
|
||||
})
|
||||
for (String key : sixMap.keySet()) {
|
||||
@@ -47,4 +47,4 @@ for (String key : sixMap.keySet()) {
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("F:\\数据\\价格\\东莞上海价格偏差大.xlsx", result, "统计")
|
||||
ExcelUtils.write("D:\\数据\\价格\\东莞上海价格偏差大.xlsx", result, "统计")
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
package com.fangjia.script.temp.shenjian.poi
|
||||
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.statistics.poi.AmapPoi
|
||||
import com.fangjia.script.statistics.poi.AmapPoiDetail
|
||||
import com.fangjia.script.temp.shenjian.poi.entity.Community
|
||||
import com.fangjia.script.temp.shenjian.util.CommonDtoUtils
|
||||
import com.fangjia.script.temp.shenjian.util.EsUtils
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import org.apache.commons.collections.CollectionUtils
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.stream.Collectors
|
||||
|
||||
/**
|
||||
* 从ES获取POI
|
||||
*/
|
||||
class GetPoiFromCommunity {
|
||||
|
||||
static final ExecutorService executorService = Executors.newFixedThreadPool(10)
|
||||
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail"
|
||||
// 住宅类
|
||||
static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304";
|
||||
|
||||
static void main(String[] args) {
|
||||
AtomicLong atomicLong = new AtomicLong(0)
|
||||
Map<String, String> map = new ConcurrentHashMap<>()
|
||||
def rets = this.findCommunities()
|
||||
CountDownLatch countDownLatch = new CountDownLatch(rets.size())
|
||||
for (def retTemp : rets) {
|
||||
final def ret = retTemp
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
final String id = ret._id // 索引ID,并非小区ID
|
||||
String city = ret.city
|
||||
String region = ret.region
|
||||
String seaweedName = ret.name
|
||||
String address = ret.address
|
||||
List<AmapPoi> pois = new ArrayList<>();
|
||||
List<AmapPoi> namePois = GaoDeUtils.searchPoi(city, seaweedName, region, typecodes);
|
||||
List<AmapPoi> addressPois = GaoDeUtils.searchPoi(city, address, region, typecodes);
|
||||
if (CollectionUtils.isNotEmpty(namePois)) {
|
||||
pois.addAll(namePois)
|
||||
}
|
||||
if (CollectionUtils.isNotEmpty(addressPois)) {
|
||||
pois.addAll(addressPois)
|
||||
}
|
||||
if (CollectionUtils.isNotEmpty(pois)) {
|
||||
List<AmapPoiDetail> detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois);
|
||||
String jsonStr = JSONObject.toJSONString(detailList)
|
||||
HttpUtils.postForJson(URL, jsonStr)
|
||||
}
|
||||
synchronized (map) {
|
||||
map.put(id, id)
|
||||
countDownLatch.countDown()
|
||||
if (map.size() % 100 == 0) {
|
||||
atomicLong.addAndGet(map.size())
|
||||
List<String> ids = map.values().stream().collect(Collectors.toList())
|
||||
EsUtils.bulkUpdateFieldByIds("community", "crawl_flag", 0, ids)
|
||||
map.clear()
|
||||
println(atomicLong.intValue())
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
countDownLatch.await()
|
||||
if (map.size() > 0) {
|
||||
atomicLong.addAndGet(map.size())
|
||||
List<String> ids = map.values().stream().collect(Collectors.toList())
|
||||
EsUtils.bulkUpdateFieldByIds("community", "crawl_flag", 0, ids)
|
||||
map.clear()
|
||||
println(atomicLong.intValue())
|
||||
}
|
||||
System.exit(1)
|
||||
}
|
||||
|
||||
static def findCommunities(){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
||||
.mustNot(QueryBuilders.termQuery("crawl_flag", 0))
|
||||
List<Community> communities = EsUtils.scrollQuery("community", queryBuilder, Community.class)
|
||||
return communities
|
||||
}
|
||||
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package com.fangjia.script.temp.shenjian.poi.entity;
|
||||
|
||||
public class Community extends MyId {
|
||||
private String id;
|
||||
private String city;
|
||||
private String region;
|
||||
private String name;
|
||||
private String address;
|
||||
|
||||
String getId() {
|
||||
return id
|
||||
}
|
||||
|
||||
void setId(String id) {
|
||||
this.id = id
|
||||
}
|
||||
|
||||
String getCity() {
|
||||
return city
|
||||
}
|
||||
|
||||
void setCity(String city) {
|
||||
this.city = city
|
||||
}
|
||||
|
||||
String getRegion() {
|
||||
return region
|
||||
}
|
||||
|
||||
void setRegion(String region) {
|
||||
this.region = region
|
||||
}
|
||||
|
||||
String getName() {
|
||||
return name
|
||||
}
|
||||
|
||||
void setName(String name) {
|
||||
this.name = name
|
||||
}
|
||||
|
||||
String getAddress() {
|
||||
return address
|
||||
}
|
||||
|
||||
void setAddress(String address) {
|
||||
this.address = address
|
||||
}
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package com.fangjia.script.temp.shenjian.poi.entity
|
||||
|
||||
class MyId {
|
||||
|
||||
/**
|
||||
* Ë÷Òýid
|
||||
*/
|
||||
String _id;
|
||||
|
||||
String get_id() {
|
||||
return _id
|
||||
}
|
||||
|
||||
void set_id(String _id) {
|
||||
this._id = _id
|
||||
}
|
||||
}
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
package com.fangjia.script.temp.shenjian.util
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils
|
||||
import com.fangjia.script.temp.shenjian.poi.entity.MyId
|
||||
import org.elasticsearch.action.DocWriteRequest
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.search.*
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.search.Scroll
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder
|
||||
import org.slf4j.Logger
|
||||
import org.slf4j.LoggerFactory
|
||||
|
||||
/**
|
||||
* @author shenjian
|
||||
* @since 2022/07/02
|
||||
*/
|
||||
class EsUtils {
|
||||
|
||||
private static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient()
|
||||
private static final Logger logger = LoggerFactory.getLogger(EsUtils.class)
|
||||
|
||||
/**
|
||||
* 滚动方式查询大批量es数据
|
||||
*
|
||||
* @param index 索引
|
||||
* @param boolQuery 查询语句
|
||||
* @param clazz 要返回的实体类型
|
||||
* @param <T>
|
||||
* @return
|
||||
*/
|
||||
public static <T extends MyId> List<T> scrollQuery(String index, QueryBuilder boolQuery, Class<T> clazz, def limitSize=300000) {
|
||||
List<T> result = new ArrayList<>();
|
||||
// 存活时间,当索引数据量特别大时,出现超时可能性大,此值适当调大
|
||||
Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30L));
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(boolQuery);
|
||||
// TEST 用!!!!!!!!!!!!!!!!!!!!!!
|
||||
//searchSourceBuilder.size(1);
|
||||
searchSourceBuilder.size(10000);
|
||||
SearchRequest searchRequest = new SearchRequest().indices(index).scroll(scroll).source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = null;
|
||||
try {
|
||||
searchResponse = ES_CLIENT.search(searchRequest, RequestOptions.DEFAULT);
|
||||
} catch (IOException e) {
|
||||
logger.error("查询es数据异常:{}", e);
|
||||
}
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
if (searchHits != null) {
|
||||
for (SearchHit searchHit : searchHits) {
|
||||
T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz);
|
||||
t.set_id(searchHit.getId())
|
||||
result.add(t);
|
||||
}
|
||||
}
|
||||
// 遍历搜索命中的数据,直到没有数据
|
||||
while (searchHits != null && searchHits.length > 0 && result.size() < limitSize) {
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
try {
|
||||
searchResponse = ES_CLIENT.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
} catch (IOException e) {
|
||||
logger.error("查询es数据异常:{}", e);
|
||||
}
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
if (searchHits != null && searchHits.length > 0) {
|
||||
for (SearchHit searchHit : searchHits) {
|
||||
T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz);
|
||||
t.set_id(searchHit.getId())
|
||||
result.add(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
// clean scroll
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
ClearScrollResponse clearScrollResponse = null;
|
||||
try {
|
||||
clearScrollResponse = ES_CLIENT.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
} catch (IOException e) {}
|
||||
boolean succeeded = clearScrollResponse.isSucceeded();
|
||||
if (succeeded) {
|
||||
return result;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 批量更新es某个字段值
|
||||
* @param index 索引名
|
||||
* @param field 字段名
|
||||
* @param val 字段新值
|
||||
* @param ids 要更新的文档id
|
||||
*/
|
||||
public static void bulkUpdateFieldByIds(String index, String field, Object val, List<String> ids) {
|
||||
if (ids.size() > 1000) {
|
||||
throw new RuntimeException("批量更新数量过多")
|
||||
}
|
||||
BulkRequest bulkRequest = new BulkRequest();
|
||||
for (String id : ids) {
|
||||
Map<String, Object> doc = new HashMap<>();
|
||||
doc.put(field, val)
|
||||
bulkRequest.add(new UpdateRequest(index, id).doc(doc));
|
||||
}
|
||||
ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user