Merge remote-tracking branch 'origin/master'

This commit is contained in:
majintao
2024-12-19 11:30:39 +08:00
3 changed files with 195 additions and 18 deletions
@@ -12,6 +12,7 @@ import com.fangjia.dal.mongo.model.*;
import com.fangjia.open.dto.addressGovernance.MatchedBuildingDto;
import com.fangjia.open.model.AvgPriceQueryModel;
import com.fangjia.open.model.BasicModel;
import com.fangjia.open.model.weixin.DistrictModel;
import com.fangjia.open.util.CommonDtoUtils;
import com.fangjia.service.datastore.RegionStatService;
import com.fangjia.service.district.DistrictFacilitysService;
@@ -20,6 +21,7 @@ import com.fangjia.service.district.RegionStatRentService;
import com.fangjia.service.mongo.fangjia.AverageAnalyService;
import com.fangjia.service.mongo.fangjia.RegionBlockService;
import com.fangjia.service.mongo.fangjia.SeaweedImagesService;
import net.sf.json.JSONArray;
import net.sf.json.JSONException;
import net.sf.json.JSONObject;
@@ -95,6 +97,7 @@ public class DistrictController {
private RegionStatService regionStatService;
private RegionStatRentService regionStatRentService;
private DistrictFacilitysService districtFacilitysService;
private SeaweedImagesService seaweedImagesService;
public static final String[] COLUMNS = { "id", "name", "estateType", "buildType", "licence", "buildArea", "allArea",
@@ -1168,6 +1171,28 @@ public class DistrictController {
return mav;
}
/**
* 获取小区图片
*
* @param districtModel
* @return
*/
public Object listPicAction(DistrictModel districtModel) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(districtModel.getId())) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "ID必须输入"));
return mav;
}
SeaweedImages seaweedImages = seaweedImagesService.findById(districtModel.getId());
if (seaweedImages != null) {
mav.setModel(BasicModel.buildOKModel().setResult(seaweedImages.getOutPics()));
} else {
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "图片数据不存在"));
}
return mav;
}
public Object addrFacilitysAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "address") String address) throws Exception{
RestfulModelAndView mav = new RestfulModelAndView();
@@ -1288,4 +1313,8 @@ public class DistrictController {
public void setDistrictFacilitysService(DistrictFacilitysService districtFacilitysService) {
this.districtFacilitysService = districtFacilitysService;
}
public void setSeaweedImagesService(SeaweedImagesService seaweedImagesService) {
this.seaweedImagesService = seaweedImagesService;
}
}
@@ -102,22 +102,6 @@ public class DistrictController {
return model;
}
/**
* 获取小区图片
*
* @param districtModel
* @return
*/
public Object listPicAction(DistrictModel districtModel) {
if (StringUtils.isBlank(districtModel.getId())) {
return null;
}
SeaweedImages seaweedImages = seaweedImagesService.findById(districtModel.getId());
if (seaweedImages != null) {
return seaweedImages.getOutPics();
}
return null;
}
private List<AutoCompleteBo> getDistrictByKeyword(String cat, String city, String keyword, int limit, final Double lng, final Double lat) {
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -8,13 +9,19 @@ import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
import org.apache.commons.io.FileUtils
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -39,8 +46,8 @@ class NewGaoDePoiBatch extends BaseBatch{
@Override
void process(Object args) {
run()
// oldPoiMagre()
// run()
oldPoiMagre()
}
def run(){
@@ -349,7 +356,164 @@ class NewGaoDePoiBatch extends BaseBatch{
}
def oldPoiMagre(){
def cityList = CommonDatautils.findFullNameCitys().reverse()
for(def cityObj : cityList){
// if(!cityObj.city.contains("固原")){
// continue
// }
def fileName = "/home/goojia/gj/poi/poi/${cityObj.city}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\merge_poi\\${cityObj.city}.csv"
}
File file=new File(fileName);
if(file.exists()){
def results = readCSV(fileName)
println cityObj.province+" " +cityObj.city+" "+results.size()
if(results){
def megerMap = poiRepeat(cityObj.city, results)
save(megerMap.values())
}
}
}
}
def poiRepeat(city, oldPois){
def rets = new ConcurrentHashMap()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(oldPois.size())
def townMap = CommonDatautils.queryTownByCity(city)
def esMap = findEsPoiByCity(city)
println "Es 加载完成:"+new Date().format("yyyy-MM-dd HH:mm:ss")
for(def i=0;i<oldPois.size();i++){
final def p = oldPois[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def nameMap = esMap.get(p.district)
def nameKeys = nameMap.keySet()
def key = p.name.replace("-","").replace("·","")
if(!nameKeys.contains(key)){
def simlarObj = null
if(esMap){
simlarObj = findSimlarPoi(p, esMap)
}
if(!simlarObj){
def newKey = p.city+"##"+p.district+"##"+p.name.replace("-","").replace("·","")
if(!rets.containsKey(newKey)){
p.data_source_year = "2023-08"
rets.put(newKey, p)
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def findSimlarPoi(ret, esMap){
def simlarObj = null
def region = ""
if(ret.district){
region = ret.district
}
def regionMap = esMap.get(region)
if(regionMap){
def keys = regionMap.keySet()
for(def key : keys){
def esValue = regionMap.get(key)
if(esValue.township && ret.township){
if(esValue.name.contains(ret.name) || ret.name.contains(esValue.name)){//||
def distance = GeoUtils.calculateDistance(ret.location.lon, ret.location.lat, esValue.location.lon, esValue.location.lat);
if(distance <100){
simlarObj = esValue
println esValue.name +" "+ret.name+" "+distance
break;
}
}
}
}
}else{
println "无数据"
}
return simlarObj
}
def findEsPoiByCity(city){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
def map = new HashMap()
ElasticUtils89.executeQuery("192.168.0.89", "didata_map_poi", queryBuilder, { String id, Map d->
def district = ""
if(d.district){
district = d.district
}
def regionMap = map.get(district)
if(!regionMap){
regionMap = new HashMap()
}
def o = new BasicDBObject()
o.put("city", d.city)
o.put("district", d.district)
o.put("name", d.name)
o.put("township", d.township)
o.put("location", d.location)
regionMap.put(d.name.replace("-","").replace("·",""), o)
map.put(district, regionMap)
})
return map
}
def readCSV(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
for(line in lines) {
if(line){
String[] arrs = line.split("\\|@\\|")
def o = new BasicDBObject()
o.put("province",arrs[0])
o.put("city" , arrs[1])
o.put("district",arrs[2])
o.put("name",arrs[3])
o.put("township",arrs[4])
o.put("address",arrs[5])
o.put("cat",arrs[6])
o.put("type",arrs[7])
o.put("secondary_type",arrs[8])
o.put("small_type" ,arrs[9])
if(arrs[10]){
o.put("data_id",arrs[10])
}else{
o.put("data_id",arrs[11])
}
o.put("spell_name",arrs[12])
def location = [:]
if(arrs[13]){
location.lon = arrs[13].toString().toDouble()
location.lat = arrs[14].toString().toDouble()
}
o.put("location",location)
o.put("data_source",arrs[15])
o.put("status",arrs[16])
o.put("version",arrs[17])
rets << o
}
}
return rets;
}
}