Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj
This commit is contained in:
+102
@@ -0,0 +1,102 @@
|
|||||||
|
package com.fangjia.script.temp.shenjian.poi
|
||||||
|
|
||||||
|
|
||||||
|
import com.alibaba.fastjson.JSONObject
|
||||||
|
import com.fangjia.core.RegionData
|
||||||
|
import com.fangjia.script.statistics.poi.AmapPoi
|
||||||
|
import com.fangjia.script.statistics.poi.AmapPoiDetail
|
||||||
|
import com.fangjia.script.temp.shenjian.poi.entity.Community
|
||||||
|
import com.fangjia.script.temp.shenjian.util.CommonDtoUtils
|
||||||
|
import com.fangjia.script.temp.shenjian.util.EsUtils
|
||||||
|
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||||
|
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||||
|
import org.apache.commons.collections.CollectionUtils
|
||||||
|
import org.elasticsearch.action.bulk.BulkRequest
|
||||||
|
import org.elasticsearch.action.update.UpdateRequest
|
||||||
|
import org.elasticsearch.client.RequestOptions
|
||||||
|
import org.elasticsearch.index.query.QueryBuilder
|
||||||
|
import org.elasticsearch.index.query.QueryBuilders
|
||||||
|
|
||||||
|
import java.util.concurrent.ConcurrentHashMap
|
||||||
|
import java.util.concurrent.CountDownLatch
|
||||||
|
import java.util.concurrent.ExecutorService
|
||||||
|
import java.util.concurrent.Executors
|
||||||
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
import java.util.stream.Collectors
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 逆解析第三方小区经纬度
|
||||||
|
*/
|
||||||
|
class ReGeoFromCommunity {
|
||||||
|
|
||||||
|
static final ExecutorService executorService = Executors.newFixedThreadPool(5)
|
||||||
|
|
||||||
|
static void main(String[] args) {
|
||||||
|
AtomicLong atomicLong = new AtomicLong(0)
|
||||||
|
Map<String, String> map = new ConcurrentHashMap<>()
|
||||||
|
def rets = this.findCommunities()
|
||||||
|
CountDownLatch countDownLatch = new CountDownLatch(rets.size())
|
||||||
|
final BulkRequest bulkRequest = new BulkRequest();
|
||||||
|
for (def retTemp : rets) {
|
||||||
|
final def ret = retTemp
|
||||||
|
executorService.submit(new Runnable() {
|
||||||
|
@Override
|
||||||
|
void run() {
|
||||||
|
try {
|
||||||
|
final String id = ret._id // 索引ID,并非小区ID
|
||||||
|
if (ret.location) {
|
||||||
|
String lon = ret.location.lon
|
||||||
|
String lat = ret.location.lat
|
||||||
|
def addressComponent = GaoDeUtils.reGeoLocation(lon + "," + lat)
|
||||||
|
if (addressComponent) {
|
||||||
|
def city_format = RegionData.formatCity(addressComponent.city)
|
||||||
|
def region_format = RegionData.formatRegion(city_format, addressComponent.district)
|
||||||
|
def township = addressComponent.township
|
||||||
|
Map<String, String> obj = new HashMap<>()
|
||||||
|
obj.put("city", city_format)
|
||||||
|
obj.put("region", region_format)
|
||||||
|
obj.put("township", township)
|
||||||
|
|
||||||
|
Map<String, Object> doc = new HashMap<>();
|
||||||
|
doc.put("address_component", obj)
|
||||||
|
bulkRequest.add(new UpdateRequest("community", id).doc(doc));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
synchronized (map) {
|
||||||
|
map.put(id, id)
|
||||||
|
if (map.size() % 1000 == 0) {
|
||||||
|
atomicLong.addAndGet(map.size())
|
||||||
|
EsUtils.ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT)
|
||||||
|
bulkRequest = new BulkRequest();
|
||||||
|
map.clear()
|
||||||
|
println(atomicLong.intValue())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch(Exception ex) {
|
||||||
|
|
||||||
|
} finally {
|
||||||
|
countDownLatch.countDown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
};
|
||||||
|
countDownLatch.await();
|
||||||
|
if (map.size() > 0) {
|
||||||
|
atomicLong.addAndGet(map.size())
|
||||||
|
EsUtils.ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT)
|
||||||
|
bulkRequest = new BulkRequest();
|
||||||
|
map.clear()
|
||||||
|
println(atomicLong.intValue())
|
||||||
|
}
|
||||||
|
System.exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
static def findCommunities(){
|
||||||
|
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
||||||
|
.mustNot(QueryBuilders.existsQuery("address_component"))
|
||||||
|
.must(QueryBuilders.existsQuery("location"))
|
||||||
|
List<Community> communities = EsUtils.scrollQuery("community", queryBuilder, Community.class, 200000)
|
||||||
|
return communities
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
+2
-2
@@ -1,6 +1,6 @@
|
|||||||
package com.fangjia.script.temp.shenjian.poi.entity;
|
package com.fangjia.script.temp.shenjian.poi.entity
|
||||||
|
|
||||||
public class Community extends MyId {
|
class Community extends MyLocation {
|
||||||
private String id;
|
private String id;
|
||||||
private String city;
|
private String city;
|
||||||
private String region;
|
private String region;
|
||||||
|
|||||||
+16
@@ -0,0 +1,16 @@
|
|||||||
|
package com.fangjia.script.temp.shenjian.poi.entity
|
||||||
|
|
||||||
|
import org.elasticsearch.common.geo.GeoPoint
|
||||||
|
|
||||||
|
class MyLocation extends MyId {
|
||||||
|
|
||||||
|
GeoPoint location;
|
||||||
|
|
||||||
|
GeoPoint getLocation() {
|
||||||
|
return location
|
||||||
|
}
|
||||||
|
|
||||||
|
void setLocation(GeoPoint location) {
|
||||||
|
this.location = location
|
||||||
|
}
|
||||||
|
}
|
||||||
+16
-3
@@ -3,12 +3,14 @@ package com.fangjia.script.temp.shenjian.util
|
|||||||
import com.alibaba.fastjson.JSONObject
|
import com.alibaba.fastjson.JSONObject
|
||||||
import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils
|
import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils
|
||||||
import com.fangjia.script.temp.shenjian.poi.entity.MyId
|
import com.fangjia.script.temp.shenjian.poi.entity.MyId
|
||||||
|
import com.fangjia.script.temp.shenjian.poi.entity.MyLocation
|
||||||
import org.elasticsearch.action.DocWriteRequest
|
import org.elasticsearch.action.DocWriteRequest
|
||||||
import org.elasticsearch.action.bulk.BulkRequest
|
import org.elasticsearch.action.bulk.BulkRequest
|
||||||
import org.elasticsearch.action.search.*
|
import org.elasticsearch.action.search.*
|
||||||
import org.elasticsearch.action.update.UpdateRequest
|
import org.elasticsearch.action.update.UpdateRequest
|
||||||
import org.elasticsearch.client.RequestOptions
|
import org.elasticsearch.client.RequestOptions
|
||||||
import org.elasticsearch.client.RestHighLevelClient
|
import org.elasticsearch.client.RestHighLevelClient
|
||||||
|
import org.elasticsearch.common.geo.GeoPoint
|
||||||
import org.elasticsearch.common.unit.TimeValue
|
import org.elasticsearch.common.unit.TimeValue
|
||||||
import org.elasticsearch.index.query.QueryBuilder
|
import org.elasticsearch.index.query.QueryBuilder
|
||||||
import org.elasticsearch.search.Scroll
|
import org.elasticsearch.search.Scroll
|
||||||
@@ -23,7 +25,7 @@ import org.slf4j.LoggerFactory
|
|||||||
*/
|
*/
|
||||||
class EsUtils {
|
class EsUtils {
|
||||||
|
|
||||||
private static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient()
|
public static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient()
|
||||||
private static final Logger logger = LoggerFactory.getLogger(EsUtils.class)
|
private static final Logger logger = LoggerFactory.getLogger(EsUtils.class)
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -35,7 +37,7 @@ class EsUtils {
|
|||||||
* @param <T>
|
* @param <T>
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
public static <T extends MyId> List<T> scrollQuery(String index, QueryBuilder boolQuery, Class<T> clazz, def limitSize=300000) {
|
public static <T extends MyLocation> List<T> scrollQuery(String index, QueryBuilder boolQuery, Class<T> clazz, def limitSize=300000) {
|
||||||
List<T> result = new ArrayList<>();
|
List<T> result = new ArrayList<>();
|
||||||
// 存活时间,当索引数据量特别大时,出现超时可能性大,此值适当调大
|
// 存活时间,当索引数据量特别大时,出现超时可能性大,此值适当调大
|
||||||
Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30L));
|
Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30L));
|
||||||
@@ -43,7 +45,11 @@ class EsUtils {
|
|||||||
searchSourceBuilder.query(boolQuery);
|
searchSourceBuilder.query(boolQuery);
|
||||||
// TEST 用!!!!!!!!!!!!!!!!!!!!!!
|
// TEST 用!!!!!!!!!!!!!!!!!!!!!!
|
||||||
//searchSourceBuilder.size(1);
|
//searchSourceBuilder.size(1);
|
||||||
searchSourceBuilder.size(10000);
|
if (limitSize < 10000) {
|
||||||
|
searchSourceBuilder.size(limitSize)
|
||||||
|
} else {
|
||||||
|
searchSourceBuilder.size(10000)
|
||||||
|
}
|
||||||
SearchRequest searchRequest = new SearchRequest().indices(index).scroll(scroll).source(searchSourceBuilder);
|
SearchRequest searchRequest = new SearchRequest().indices(index).scroll(scroll).source(searchSourceBuilder);
|
||||||
SearchResponse searchResponse = null;
|
SearchResponse searchResponse = null;
|
||||||
try {
|
try {
|
||||||
@@ -57,6 +63,13 @@ class EsUtils {
|
|||||||
for (SearchHit searchHit : searchHits) {
|
for (SearchHit searchHit : searchHits) {
|
||||||
T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz);
|
T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz);
|
||||||
t.set_id(searchHit.getId())
|
t.set_id(searchHit.getId())
|
||||||
|
Map<String, Double> location = searchHit.getSourceAsMap().get("location")
|
||||||
|
if (location != null) {
|
||||||
|
Double lon = location["lon"]
|
||||||
|
Double lat = location["lat"]
|
||||||
|
GeoPoint geoPoint = new GeoPoint(lat, lon)
|
||||||
|
t.setLocation(geoPoint)
|
||||||
|
}
|
||||||
result.add(t);
|
result.add(t);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+35
-4
@@ -334,10 +334,6 @@ class GaoDeUtils {
|
|||||||
/**
|
/**
|
||||||
* 经纬度转地址<br>
|
* 经纬度转地址<br>
|
||||||
* 规则: 最多支持20个坐标点。多个点之间用"|"分割。经度在前,纬度在后,经纬度间以“,”分割,经纬度小数点后不得超过6位
|
* 规则: 最多支持20个坐标点。多个点之间用"|"分割。经度在前,纬度在后,经纬度间以“,”分割,经纬度小数点后不得超过6位
|
||||||
* @author yinjihuan
|
|
||||||
* @date 2016年9月9日
|
|
||||||
* @param location
|
|
||||||
* @return
|
|
||||||
*/
|
*/
|
||||||
static def getGaoDeAddress(location) {
|
static def getGaoDeAddress(location) {
|
||||||
Map<String, String> params = new HashMap<String, String>()
|
Map<String, String> params = new HashMap<String, String>()
|
||||||
@@ -369,6 +365,41 @@ class GaoDeUtils {
|
|||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 地址逆解析
|
||||||
|
* @param location
|
||||||
|
* @return 经度在前,纬度在后,经纬度间以“,”分割,经纬度小数点后不得超过6位
|
||||||
|
*/
|
||||||
|
static def reGeoLocation(location) {
|
||||||
|
Map<String, String> params = new HashMap<String, String>()
|
||||||
|
params.put("location", location)
|
||||||
|
String content
|
||||||
|
try {
|
||||||
|
params.put("key", this.getGaodeKey("v3/geocode/regeo"))
|
||||||
|
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
|
||||||
|
} catch (Exception e) {
|
||||||
|
params.put("key", this.getGaodeKey("v3/geocode/regeo"))
|
||||||
|
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
|
||||||
|
}
|
||||||
|
net.sf.json.JSONObject json = net.sf.json.JSONObject.fromObject(content)
|
||||||
|
Map<String, Object> infoMap = DataHandleUtils.JsonToMap(json);
|
||||||
|
if (infoMap && infoMap.info == "OK") {
|
||||||
|
def regeocode = infoMap.regeocode
|
||||||
|
if (regeocode) {
|
||||||
|
return regeocode.addressComponent
|
||||||
|
}
|
||||||
|
} else if (json && json.infocode && json.infocode == "10044") {
|
||||||
|
System.exit(1)
|
||||||
|
} else if (json && json.infocode && json.infocode == "10004") {
|
||||||
|
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
|
||||||
|
Thread.sleep(60000)
|
||||||
|
} else if (json && json.infocode && json.infocode == "10017") {
|
||||||
|
// 所请求的资源不可用
|
||||||
|
System.exit(1)
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 获取地址经纬度
|
* 获取地址经纬度
|
||||||
*/
|
*/
|
||||||
|
|||||||
Reference in New Issue
Block a user