Merge branch 'script_gj' into 'master'
Script gj See merge request !372
This commit is contained in:
+102
@@ -0,0 +1,102 @@
|
||||
package com.fangjia.script.temp.shenjian.poi
|
||||
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.statistics.poi.AmapPoi
|
||||
import com.fangjia.script.statistics.poi.AmapPoiDetail
|
||||
import com.fangjia.script.temp.shenjian.poi.entity.Community
|
||||
import com.fangjia.script.temp.shenjian.util.CommonDtoUtils
|
||||
import com.fangjia.script.temp.shenjian.util.EsUtils
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import org.apache.commons.collections.CollectionUtils
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.stream.Collectors
|
||||
|
||||
/**
|
||||
* 逆解析第三方小区经纬度
|
||||
*/
|
||||
class ReGeoFromCommunity {
|
||||
|
||||
static final ExecutorService executorService = Executors.newFixedThreadPool(5)
|
||||
|
||||
static void main(String[] args) {
|
||||
AtomicLong atomicLong = new AtomicLong(0)
|
||||
Map<String, String> map = new ConcurrentHashMap<>()
|
||||
def rets = this.findCommunities()
|
||||
CountDownLatch countDownLatch = new CountDownLatch(rets.size())
|
||||
final BulkRequest bulkRequest = new BulkRequest();
|
||||
for (def retTemp : rets) {
|
||||
final def ret = retTemp
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try {
|
||||
final String id = ret._id // 索引ID,并非小区ID
|
||||
if (ret.location) {
|
||||
String lon = ret.location.lon
|
||||
String lat = ret.location.lat
|
||||
def addressComponent = GaoDeUtils.reGeoLocation(lon + "," + lat)
|
||||
if (addressComponent) {
|
||||
def city_format = RegionData.formatCity(addressComponent.city)
|
||||
def region_format = RegionData.formatRegion(city_format, addressComponent.district)
|
||||
def township = addressComponent.township
|
||||
Map<String, String> obj = new HashMap<>()
|
||||
obj.put("city", city_format)
|
||||
obj.put("region", region_format)
|
||||
obj.put("township", township)
|
||||
|
||||
Map<String, Object> doc = new HashMap<>();
|
||||
doc.put("address_component", obj)
|
||||
bulkRequest.add(new UpdateRequest("community", id).doc(doc));
|
||||
}
|
||||
}
|
||||
synchronized (map) {
|
||||
map.put(id, id)
|
||||
if (map.size() % 1000 == 0) {
|
||||
atomicLong.addAndGet(map.size())
|
||||
EsUtils.ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT)
|
||||
bulkRequest = new BulkRequest();
|
||||
map.clear()
|
||||
println(atomicLong.intValue())
|
||||
}
|
||||
}
|
||||
} catch(Exception ex) {
|
||||
|
||||
} finally {
|
||||
countDownLatch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
countDownLatch.await();
|
||||
if (map.size() > 0) {
|
||||
atomicLong.addAndGet(map.size())
|
||||
EsUtils.ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT)
|
||||
bulkRequest = new BulkRequest();
|
||||
map.clear()
|
||||
println(atomicLong.intValue())
|
||||
}
|
||||
System.exit(1)
|
||||
}
|
||||
|
||||
static def findCommunities(){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
||||
.mustNot(QueryBuilders.existsQuery("address_component"))
|
||||
.must(QueryBuilders.existsQuery("location"))
|
||||
List<Community> communities = EsUtils.scrollQuery("community", queryBuilder, Community.class, 200000)
|
||||
return communities
|
||||
}
|
||||
|
||||
}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
package com.fangjia.script.temp.shenjian.poi.entity;
|
||||
package com.fangjia.script.temp.shenjian.poi.entity
|
||||
|
||||
public class Community extends MyId {
|
||||
class Community extends MyLocation {
|
||||
private String id;
|
||||
private String city;
|
||||
private String region;
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
package com.fangjia.script.temp.shenjian.poi.entity
|
||||
|
||||
import org.elasticsearch.common.geo.GeoPoint
|
||||
|
||||
class MyLocation extends MyId {
|
||||
|
||||
GeoPoint location;
|
||||
|
||||
GeoPoint getLocation() {
|
||||
return location
|
||||
}
|
||||
|
||||
void setLocation(GeoPoint location) {
|
||||
this.location = location
|
||||
}
|
||||
}
|
||||
+16
-3
@@ -3,12 +3,14 @@ package com.fangjia.script.temp.shenjian.util
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils
|
||||
import com.fangjia.script.temp.shenjian.poi.entity.MyId
|
||||
import com.fangjia.script.temp.shenjian.poi.entity.MyLocation
|
||||
import org.elasticsearch.action.DocWriteRequest
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.search.*
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.common.geo.GeoPoint
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.search.Scroll
|
||||
@@ -23,7 +25,7 @@ import org.slf4j.LoggerFactory
|
||||
*/
|
||||
class EsUtils {
|
||||
|
||||
private static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient()
|
||||
public static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient()
|
||||
private static final Logger logger = LoggerFactory.getLogger(EsUtils.class)
|
||||
|
||||
/**
|
||||
@@ -35,7 +37,7 @@ class EsUtils {
|
||||
* @param <T>
|
||||
* @return
|
||||
*/
|
||||
public static <T extends MyId> List<T> scrollQuery(String index, QueryBuilder boolQuery, Class<T> clazz, def limitSize=300000) {
|
||||
public static <T extends MyLocation> List<T> scrollQuery(String index, QueryBuilder boolQuery, Class<T> clazz, def limitSize=300000) {
|
||||
List<T> result = new ArrayList<>();
|
||||
// 存活时间,当索引数据量特别大时,出现超时可能性大,此值适当调大
|
||||
Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30L));
|
||||
@@ -43,7 +45,11 @@ class EsUtils {
|
||||
searchSourceBuilder.query(boolQuery);
|
||||
// TEST 用!!!!!!!!!!!!!!!!!!!!!!
|
||||
//searchSourceBuilder.size(1);
|
||||
searchSourceBuilder.size(10000);
|
||||
if (limitSize < 10000) {
|
||||
searchSourceBuilder.size(limitSize)
|
||||
} else {
|
||||
searchSourceBuilder.size(10000)
|
||||
}
|
||||
SearchRequest searchRequest = new SearchRequest().indices(index).scroll(scroll).source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = null;
|
||||
try {
|
||||
@@ -57,6 +63,13 @@ class EsUtils {
|
||||
for (SearchHit searchHit : searchHits) {
|
||||
T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz);
|
||||
t.set_id(searchHit.getId())
|
||||
Map<String, Double> location = searchHit.getSourceAsMap().get("location")
|
||||
if (location != null) {
|
||||
Double lon = location["lon"]
|
||||
Double lat = location["lat"]
|
||||
GeoPoint geoPoint = new GeoPoint(lat, lon)
|
||||
t.setLocation(geoPoint)
|
||||
}
|
||||
result.add(t);
|
||||
}
|
||||
}
|
||||
|
||||
+35
-4
@@ -334,10 +334,6 @@ class GaoDeUtils {
|
||||
/**
|
||||
* 经纬度转地址<br>
|
||||
* 规则: 最多支持20个坐标点。多个点之间用"|"分割。经度在前,纬度在后,经纬度间以“,”分割,经纬度小数点后不得超过6位
|
||||
* @author yinjihuan
|
||||
* @date 2016年9月9日
|
||||
* @param location
|
||||
* @return
|
||||
*/
|
||||
static def getGaoDeAddress(location) {
|
||||
Map<String, String> params = new HashMap<String, String>()
|
||||
@@ -369,6 +365,41 @@ class GaoDeUtils {
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* 地址逆解析
|
||||
* @param location
|
||||
* @return 经度在前,纬度在后,经纬度间以“,”分割,经纬度小数点后不得超过6位
|
||||
*/
|
||||
static def reGeoLocation(location) {
|
||||
Map<String, String> params = new HashMap<String, String>()
|
||||
params.put("location", location)
|
||||
String content
|
||||
try {
|
||||
params.put("key", this.getGaodeKey("v3/geocode/regeo"))
|
||||
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
|
||||
} catch (Exception e) {
|
||||
params.put("key", this.getGaodeKey("v3/geocode/regeo"))
|
||||
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
|
||||
}
|
||||
net.sf.json.JSONObject json = net.sf.json.JSONObject.fromObject(content)
|
||||
Map<String, Object> infoMap = DataHandleUtils.JsonToMap(json);
|
||||
if (infoMap && infoMap.info == "OK") {
|
||||
def regeocode = infoMap.regeocode
|
||||
if (regeocode) {
|
||||
return regeocode.addressComponent
|
||||
}
|
||||
} else if (json && json.infocode && json.infocode == "10044") {
|
||||
System.exit(1)
|
||||
} else if (json && json.infocode && json.infocode == "10004") {
|
||||
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
|
||||
Thread.sleep(60000)
|
||||
} else if (json && json.infocode && json.infocode == "10017") {
|
||||
// 所请求的资源不可用
|
||||
System.exit(1)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取地址经纬度
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user