Merge branch 'script_gj' into 'master'

Script gj

See merge request !372
This commit is contained in:
2022-08-19 15:17:46 +08:00
5 changed files with 171 additions and 9 deletions
@@ -0,0 +1,102 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.RegionData
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.statistics.poi.AmapPoiDetail
import com.fangjia.script.temp.shenjian.poi.entity.Community
import com.fangjia.script.temp.shenjian.util.CommonDtoUtils
import com.fangjia.script.temp.shenjian.util.EsUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import org.apache.commons.collections.CollectionUtils
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.stream.Collectors
/**
* 逆解析第三方小区经纬度
*/
class ReGeoFromCommunity {
static final ExecutorService executorService = Executors.newFixedThreadPool(5)
static void main(String[] args) {
AtomicLong atomicLong = new AtomicLong(0)
Map<String, String> map = new ConcurrentHashMap<>()
def rets = this.findCommunities()
CountDownLatch countDownLatch = new CountDownLatch(rets.size())
final BulkRequest bulkRequest = new BulkRequest();
for (def retTemp : rets) {
final def ret = retTemp
executorService.submit(new Runnable() {
@Override
void run() {
try {
final String id = ret._id // 索引ID,并非小区ID
if (ret.location) {
String lon = ret.location.lon
String lat = ret.location.lat
def addressComponent = GaoDeUtils.reGeoLocation(lon + "," + lat)
if (addressComponent) {
def city_format = RegionData.formatCity(addressComponent.city)
def region_format = RegionData.formatRegion(city_format, addressComponent.district)
def township = addressComponent.township
Map<String, String> obj = new HashMap<>()
obj.put("city", city_format)
obj.put("region", region_format)
obj.put("township", township)
Map<String, Object> doc = new HashMap<>();
doc.put("address_component", obj)
bulkRequest.add(new UpdateRequest("community", id).doc(doc));
}
}
synchronized (map) {
map.put(id, id)
if (map.size() % 1000 == 0) {
atomicLong.addAndGet(map.size())
EsUtils.ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT)
bulkRequest = new BulkRequest();
map.clear()
println(atomicLong.intValue())
}
}
} catch(Exception ex) {
} finally {
countDownLatch.countDown()
}
}
})
};
countDownLatch.await();
if (map.size() > 0) {
atomicLong.addAndGet(map.size())
EsUtils.ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT)
bulkRequest = new BulkRequest();
map.clear()
println(atomicLong.intValue())
}
System.exit(1)
}
static def findCommunities(){
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
.mustNot(QueryBuilders.existsQuery("address_component"))
.must(QueryBuilders.existsQuery("location"))
List<Community> communities = EsUtils.scrollQuery("community", queryBuilder, Community.class, 200000)
return communities
}
}
@@ -1,6 +1,6 @@
package com.fangjia.script.temp.shenjian.poi.entity;
package com.fangjia.script.temp.shenjian.poi.entity
public class Community extends MyId {
class Community extends MyLocation {
private String id;
private String city;
private String region;
@@ -0,0 +1,16 @@
package com.fangjia.script.temp.shenjian.poi.entity
import org.elasticsearch.common.geo.GeoPoint
class MyLocation extends MyId {
GeoPoint location;
GeoPoint getLocation() {
return location
}
void setLocation(GeoPoint location) {
this.location = location
}
}
@@ -3,12 +3,14 @@ package com.fangjia.script.temp.shenjian.util
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils
import com.fangjia.script.temp.shenjian.poi.entity.MyId
import com.fangjia.script.temp.shenjian.poi.entity.MyLocation
import org.elasticsearch.action.DocWriteRequest
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.search.*
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.common.geo.GeoPoint
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.search.Scroll
@@ -23,7 +25,7 @@ import org.slf4j.LoggerFactory
*/
class EsUtils {
private static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient()
public static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient()
private static final Logger logger = LoggerFactory.getLogger(EsUtils.class)
/**
@@ -35,7 +37,7 @@ class EsUtils {
* @param <T>
* @return
*/
public static <T extends MyId> List<T> scrollQuery(String index, QueryBuilder boolQuery, Class<T> clazz, def limitSize=300000) {
public static <T extends MyLocation> List<T> scrollQuery(String index, QueryBuilder boolQuery, Class<T> clazz, def limitSize=300000) {
List<T> result = new ArrayList<>();
// 存活时间,当索引数据量特别大时,出现超时可能性大,此值适当调大
Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30L));
@@ -43,7 +45,11 @@ class EsUtils {
searchSourceBuilder.query(boolQuery);
// TEST 用!!!!!!!!!!!!!!!!!!!!!!
//searchSourceBuilder.size(1);
searchSourceBuilder.size(10000);
if (limitSize < 10000) {
searchSourceBuilder.size(limitSize)
} else {
searchSourceBuilder.size(10000)
}
SearchRequest searchRequest = new SearchRequest().indices(index).scroll(scroll).source(searchSourceBuilder);
SearchResponse searchResponse = null;
try {
@@ -57,6 +63,13 @@ class EsUtils {
for (SearchHit searchHit : searchHits) {
T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz);
t.set_id(searchHit.getId())
Map<String, Double> location = searchHit.getSourceAsMap().get("location")
if (location != null) {
Double lon = location["lon"]
Double lat = location["lat"]
GeoPoint geoPoint = new GeoPoint(lat, lon)
t.setLocation(geoPoint)
}
result.add(t);
}
}
@@ -334,10 +334,6 @@ class GaoDeUtils {
/**
* 经纬度转地址<br>
* 规则: 最多支持20个坐标点。多个点之间用"|"分割。经度在前,纬度在后,经纬度间以“,”分割,经纬度小数点后不得超过6位
* @author yinjihuan
* @date 2016年9月9日
* @param location
* @return
*/
static def getGaoDeAddress(location) {
Map<String, String> params = new HashMap<String, String>()
@@ -369,6 +365,41 @@ class GaoDeUtils {
return null
}
/**
* 地址逆解析
* @param location
* @return 经度在前,纬度在后,经纬度间以“,”分割,经纬度小数点后不得超过6位
*/
static def reGeoLocation(location) {
Map<String, String> params = new HashMap<String, String>()
params.put("location", location)
String content
try {
params.put("key", this.getGaodeKey("v3/geocode/regeo"))
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
} catch (Exception e) {
params.put("key", this.getGaodeKey("v3/geocode/regeo"))
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
}
net.sf.json.JSONObject json = net.sf.json.JSONObject.fromObject(content)
Map<String, Object> infoMap = DataHandleUtils.JsonToMap(json);
if (infoMap && infoMap.info == "OK") {
def regeocode = infoMap.regeocode
if (regeocode) {
return regeocode.addressComponent
}
} else if (json && json.infocode && json.infocode == "10044") {
System.exit(1)
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
}
return null
}
/**
* 获取地址经纬度
*/