Merge remote-tracking branch 'origin/master'
This commit is contained in:
+29
@@ -12,6 +12,7 @@ import com.fangjia.dal.mongo.model.*;
|
||||
import com.fangjia.open.dto.addressGovernance.MatchedBuildingDto;
|
||||
import com.fangjia.open.model.AvgPriceQueryModel;
|
||||
import com.fangjia.open.model.BasicModel;
|
||||
import com.fangjia.open.model.weixin.DistrictModel;
|
||||
import com.fangjia.open.util.CommonDtoUtils;
|
||||
import com.fangjia.service.datastore.RegionStatService;
|
||||
import com.fangjia.service.district.DistrictFacilitysService;
|
||||
@@ -20,6 +21,7 @@ import com.fangjia.service.district.RegionStatRentService;
|
||||
import com.fangjia.service.mongo.fangjia.AverageAnalyService;
|
||||
import com.fangjia.service.mongo.fangjia.RegionBlockService;
|
||||
|
||||
import com.fangjia.service.mongo.fangjia.SeaweedImagesService;
|
||||
import net.sf.json.JSONArray;
|
||||
import net.sf.json.JSONException;
|
||||
import net.sf.json.JSONObject;
|
||||
@@ -95,6 +97,7 @@ public class DistrictController {
|
||||
private RegionStatService regionStatService;
|
||||
private RegionStatRentService regionStatRentService;
|
||||
private DistrictFacilitysService districtFacilitysService;
|
||||
private SeaweedImagesService seaweedImagesService;
|
||||
|
||||
|
||||
public static final String[] COLUMNS = { "id", "name", "estateType", "buildType", "licence", "buildArea", "allArea",
|
||||
@@ -1168,6 +1171,28 @@ public class DistrictController {
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取小区图片
|
||||
*
|
||||
* @param districtModel
|
||||
* @return
|
||||
*/
|
||||
public Object listPicAction(DistrictModel districtModel) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(districtModel.getId())) {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "ID必须输入"));
|
||||
return mav;
|
||||
}
|
||||
SeaweedImages seaweedImages = seaweedImagesService.findById(districtModel.getId());
|
||||
if (seaweedImages != null) {
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(seaweedImages.getOutPics()));
|
||||
} else {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "图片数据不存在"));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
|
||||
public Object addrFacilitysAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "address") String address) throws Exception{
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
@@ -1288,4 +1313,8 @@ public class DistrictController {
|
||||
public void setDistrictFacilitysService(DistrictFacilitysService districtFacilitysService) {
|
||||
this.districtFacilitysService = districtFacilitysService;
|
||||
}
|
||||
|
||||
public void setSeaweedImagesService(SeaweedImagesService seaweedImagesService) {
|
||||
this.seaweedImagesService = seaweedImagesService;
|
||||
}
|
||||
}
|
||||
|
||||
-16
@@ -102,22 +102,6 @@ public class DistrictController {
|
||||
return model;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取小区图片
|
||||
*
|
||||
* @param districtModel
|
||||
* @return
|
||||
*/
|
||||
public Object listPicAction(DistrictModel districtModel) {
|
||||
if (StringUtils.isBlank(districtModel.getId())) {
|
||||
return null;
|
||||
}
|
||||
SeaweedImages seaweedImages = seaweedImagesService.findById(districtModel.getId());
|
||||
if (seaweedImages != null) {
|
||||
return seaweedImages.getOutPics();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private List<AutoCompleteBo> getDistrictByKeyword(String cat, String city, String keyword, int limit, final Double lng, final Double lat) {
|
||||
|
||||
|
||||
+166
-2
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
@@ -8,13 +9,19 @@ import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
@@ -39,8 +46,8 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
run()
|
||||
// oldPoiMagre()
|
||||
// run()
|
||||
oldPoiMagre()
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -349,7 +356,164 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def oldPoiMagre(){
|
||||
def cityList = CommonDatautils.findFullNameCitys().reverse()
|
||||
for(def cityObj : cityList){
|
||||
// if(!cityObj.city.contains("固原")){
|
||||
// continue
|
||||
// }
|
||||
def fileName = "/home/goojia/gj/poi/poi/${cityObj.city}.csv"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\merge_poi\\${cityObj.city}.csv"
|
||||
}
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
def results = readCSV(fileName)
|
||||
println cityObj.province+" " +cityObj.city+" "+results.size()
|
||||
if(results){
|
||||
def megerMap = poiRepeat(cityObj.city, results)
|
||||
save(megerMap.values())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def poiRepeat(city, oldPois){
|
||||
def rets = new ConcurrentHashMap()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(oldPois.size())
|
||||
def townMap = CommonDatautils.queryTownByCity(city)
|
||||
def esMap = findEsPoiByCity(city)
|
||||
|
||||
println "Es 加载完成:"+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
for(def i=0;i<oldPois.size();i++){
|
||||
final def p = oldPois[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def nameMap = esMap.get(p.district)
|
||||
def nameKeys = nameMap.keySet()
|
||||
def key = p.name.replace("-","").replace("·","")
|
||||
if(!nameKeys.contains(key)){
|
||||
def simlarObj = null
|
||||
if(esMap){
|
||||
simlarObj = findSimlarPoi(p, esMap)
|
||||
}
|
||||
if(!simlarObj){
|
||||
def newKey = p.city+"##"+p.district+"##"+p.name.replace("-","").replace("·","")
|
||||
if(!rets.containsKey(newKey)){
|
||||
p.data_source_year = "2023-08"
|
||||
rets.put(newKey, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def findSimlarPoi(ret, esMap){
|
||||
def simlarObj = null
|
||||
def region = ""
|
||||
if(ret.district){
|
||||
region = ret.district
|
||||
}
|
||||
def regionMap = esMap.get(region)
|
||||
if(regionMap){
|
||||
def keys = regionMap.keySet()
|
||||
|
||||
for(def key : keys){
|
||||
def esValue = regionMap.get(key)
|
||||
if(esValue.township && ret.township){
|
||||
if(esValue.name.contains(ret.name) || ret.name.contains(esValue.name)){//||
|
||||
def distance = GeoUtils.calculateDistance(ret.location.lon, ret.location.lat, esValue.location.lon, esValue.location.lat);
|
||||
if(distance <100){
|
||||
simlarObj = esValue
|
||||
println esValue.name +" "+ret.name+" "+distance
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println "无数据"
|
||||
}
|
||||
|
||||
return simlarObj
|
||||
}
|
||||
|
||||
def findEsPoiByCity(city){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
def map = new HashMap()
|
||||
ElasticUtils89.executeQuery("192.168.0.89", "didata_map_poi", queryBuilder, { String id, Map d->
|
||||
def district = ""
|
||||
if(d.district){
|
||||
district = d.district
|
||||
}
|
||||
def regionMap = map.get(district)
|
||||
if(!regionMap){
|
||||
regionMap = new HashMap()
|
||||
}
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", d.city)
|
||||
o.put("district", d.district)
|
||||
o.put("name", d.name)
|
||||
o.put("township", d.township)
|
||||
o.put("location", d.location)
|
||||
regionMap.put(d.name.replace("-","").replace("·",""), o)
|
||||
map.put(district, regionMap)
|
||||
})
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def readCSV(filename){
|
||||
def lines = FileUtils.readLines(new File(filename))
|
||||
def rets = [];
|
||||
for(line in lines) {
|
||||
if(line){
|
||||
String[] arrs = line.split("\\|@\\|")
|
||||
def o = new BasicDBObject()
|
||||
o.put("province",arrs[0])
|
||||
o.put("city" , arrs[1])
|
||||
o.put("district",arrs[2])
|
||||
o.put("name",arrs[3])
|
||||
o.put("township",arrs[4])
|
||||
o.put("address",arrs[5])
|
||||
o.put("cat",arrs[6])
|
||||
o.put("type",arrs[7])
|
||||
o.put("secondary_type",arrs[8])
|
||||
o.put("small_type" ,arrs[9])
|
||||
if(arrs[10]){
|
||||
o.put("data_id",arrs[10])
|
||||
}else{
|
||||
o.put("data_id",arrs[11])
|
||||
}
|
||||
|
||||
o.put("spell_name",arrs[12])
|
||||
def location = [:]
|
||||
if(arrs[13]){
|
||||
location.lon = arrs[13].toString().toDouble()
|
||||
location.lat = arrs[14].toString().toDouble()
|
||||
}
|
||||
o.put("location",location)
|
||||
o.put("data_source",arrs[15])
|
||||
o.put("status",arrs[16])
|
||||
o.put("version",arrs[17])
|
||||
rets << o
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user