diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/ReGeoFromCommunity.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/ReGeoFromCommunity.groovy new file mode 100644 index 000000000..018bc06c0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/ReGeoFromCommunity.groovy @@ -0,0 +1,102 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.RegionData +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.statistics.poi.AmapPoiDetail +import com.fangjia.script.temp.shenjian.poi.entity.Community +import com.fangjia.script.temp.shenjian.util.CommonDtoUtils +import com.fangjia.script.temp.shenjian.util.EsUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import org.apache.commons.collections.CollectionUtils +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.update.UpdateRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.stream.Collectors + +/** + * 逆解析第三方小区经纬度 + */ +class ReGeoFromCommunity { + + static final ExecutorService executorService = Executors.newFixedThreadPool(5) + + static void main(String[] args) { + AtomicLong atomicLong = new AtomicLong(0) + Map map = new ConcurrentHashMap<>() + def rets = this.findCommunities() + CountDownLatch countDownLatch = new CountDownLatch(rets.size()) + final BulkRequest bulkRequest = new BulkRequest(); + for (def retTemp : rets) { + final def ret = retTemp + executorService.submit(new Runnable() { + @Override + void run() { + try { + final String id = ret._id // 索引ID,并非小区ID + if (ret.location) { + String lon = ret.location.lon + String lat = ret.location.lat + def addressComponent = GaoDeUtils.reGeoLocation(lon + "," + lat) + if (addressComponent) { + def city_format = RegionData.formatCity(addressComponent.city) + def region_format = RegionData.formatRegion(city_format, addressComponent.district) + def township = addressComponent.township + Map obj = new HashMap<>() + obj.put("city", city_format) + obj.put("region", region_format) + obj.put("township", township) + + Map doc = new HashMap<>(); + doc.put("address_component", obj) + bulkRequest.add(new UpdateRequest("community", id).doc(doc)); + } + } + synchronized (map) { + map.put(id, id) + if (map.size() % 1000 == 0) { + atomicLong.addAndGet(map.size()) + EsUtils.ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT) + bulkRequest = new BulkRequest(); + map.clear() + println(atomicLong.intValue()) + } + } + } catch(Exception ex) { + + } finally { + countDownLatch.countDown() + } + } + }) + }; + countDownLatch.await(); + if (map.size() > 0) { + atomicLong.addAndGet(map.size()) + EsUtils.ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT) + bulkRequest = new BulkRequest(); + map.clear() + println(atomicLong.intValue()) + } + System.exit(1) + } + + static def findCommunities(){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery() + .mustNot(QueryBuilders.existsQuery("address_component")) + .must(QueryBuilders.existsQuery("location")) + List communities = EsUtils.scrollQuery("community", queryBuilder, Community.class, 200000) + return communities + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy index ba2bfaaea..dd1a07103 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy @@ -1,6 +1,6 @@ -package com.fangjia.script.temp.shenjian.poi.entity; +package com.fangjia.script.temp.shenjian.poi.entity -public class Community extends MyId { +class Community extends MyLocation { private String id; private String city; private String region; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/MyLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/MyLocation.groovy new file mode 100644 index 000000000..7fc56e0fe --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/MyLocation.groovy @@ -0,0 +1,16 @@ +package com.fangjia.script.temp.shenjian.poi.entity + +import org.elasticsearch.common.geo.GeoPoint + +class MyLocation extends MyId { + + GeoPoint location; + + GeoPoint getLocation() { + return location + } + + void setLocation(GeoPoint location) { + this.location = location + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy index b158c9351..1ac5a78fc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy @@ -3,12 +3,14 @@ package com.fangjia.script.temp.shenjian.util import com.alibaba.fastjson.JSONObject import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils import com.fangjia.script.temp.shenjian.poi.entity.MyId +import com.fangjia.script.temp.shenjian.poi.entity.MyLocation import org.elasticsearch.action.DocWriteRequest import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.search.* import org.elasticsearch.action.update.UpdateRequest import org.elasticsearch.client.RequestOptions import org.elasticsearch.client.RestHighLevelClient +import org.elasticsearch.common.geo.GeoPoint import org.elasticsearch.common.unit.TimeValue import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.search.Scroll @@ -23,7 +25,7 @@ import org.slf4j.LoggerFactory */ class EsUtils { - private static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient() + public static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient() private static final Logger logger = LoggerFactory.getLogger(EsUtils.class) /** @@ -35,7 +37,7 @@ class EsUtils { * @param * @return */ - public static List scrollQuery(String index, QueryBuilder boolQuery, Class clazz, def limitSize=300000) { + public static List scrollQuery(String index, QueryBuilder boolQuery, Class clazz, def limitSize=300000) { List result = new ArrayList<>(); // 存活时间,当索引数据量特别大时,出现超时可能性大,此值适当调大 Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30L)); @@ -43,7 +45,11 @@ class EsUtils { searchSourceBuilder.query(boolQuery); // TEST 用!!!!!!!!!!!!!!!!!!!!!! //searchSourceBuilder.size(1); - searchSourceBuilder.size(10000); + if (limitSize < 10000) { + searchSourceBuilder.size(limitSize) + } else { + searchSourceBuilder.size(10000) + } SearchRequest searchRequest = new SearchRequest().indices(index).scroll(scroll).source(searchSourceBuilder); SearchResponse searchResponse = null; try { @@ -57,6 +63,13 @@ class EsUtils { for (SearchHit searchHit : searchHits) { T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz); t.set_id(searchHit.getId()) + Map location = searchHit.getSourceAsMap().get("location") + if (location != null) { + Double lon = location["lon"] + Double lat = location["lat"] + GeoPoint geoPoint = new GeoPoint(lat, lon) + t.setLocation(geoPoint) + } result.add(t); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 2a2972c21..f08b96c89 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -334,10 +334,6 @@ class GaoDeUtils { /** * 经纬度转地址
* 规则: 最多支持20个坐标点。多个点之间用"|"分割。经度在前,纬度在后,经纬度间以“,”分割,经纬度小数点后不得超过6位 - * @author yinjihuan - * @date 2016年9月9日 - * @param location - * @return */ static def getGaoDeAddress(location) { Map params = new HashMap() @@ -369,6 +365,41 @@ class GaoDeUtils { return null } + /** + * 地址逆解析 + * @param location + * @return 经度在前,纬度在后,经纬度间以“,”分割,经纬度小数点后不得超过6位 + */ + static def reGeoLocation(location) { + Map params = new HashMap() + params.put("location", location) + String content + try { + params.put("key", this.getGaodeKey("v3/geocode/regeo")) + content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8") + } catch (Exception e) { + params.put("key", this.getGaodeKey("v3/geocode/regeo")) + content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8") + } + net.sf.json.JSONObject json = net.sf.json.JSONObject.fromObject(content) + Map infoMap = DataHandleUtils.JsonToMap(json); + if (infoMap && infoMap.info == "OK") { + def regeocode = infoMap.regeocode + if (regeocode) { + return regeocode.addressComponent + } + } else if (json && json.infocode && json.infocode == "10044") { + System.exit(1) + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + System.exit(1) + } + return null + } + /** * 获取地址经纬度 */