From 51138cb6286f93d94923f1fd255e2fa4f1c46310 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 28 Aug 2024 17:34:36 +0800 Subject: [PATCH 1/3] =?UTF-8?q?poi=E5=85=A5=E5=BA=93=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/xiamen/GaoDePoiDataProcess.groovy | 53 +++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index 56bd37f9a..2fa07a6d6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -128,7 +128,7 @@ class GaoDePoiDataProcess extends BaseBatch{ } def sysnchVillage(){ - def buducitys = [ "湖州市","舟山市","宁波市","丽水市","杭州市","温州市","嘉兴市","台州市","绍兴市","金华市","衢州市",'茂名市', + def buducitys = [ "宁波市","丽水市","杭州市","温州市","嘉兴市","台州市","绍兴市","金华市","衢州市",'茂名市', '荆州市', '荆门市', '莆田市','苏州市','菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市', '衢州市', '襄阳市', '西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市', '辽源市', '辽阳市', '达州市', '运城市', '连云港市', '通化市', '通辽市', '遂宁市', '遵义市', '中山市', '临夏回族自治州', @@ -168,7 +168,7 @@ class GaoDePoiDataProcess extends BaseBatch{ '鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市', '黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市'] - def runbuducitys = [] + def runbuducitys = ["舟山市","湖州市"] for(def city : buducitys){ if(runbuducitys.contains(city)){ continue @@ -205,7 +205,10 @@ class GaoDePoiDataProcess extends BaseBatch{ def prov = ProvinceData.getProvince(p.city.replace("市","")) def ret = createVillageData(prov, p, townMap) if(ret){ - def simlarObj = findSimlarPoi(ret, esMap) + def simlarObj = null + if(esMap){ + simlarObj = findSimlarPoi(ret, esMap) + } if(!simlarObj){ def newKey = ret.city+"##"+ret.district+"##"+ret.name.replace("-","").replace("·","") if(!rets.containsKey(newKey)){ @@ -227,19 +230,26 @@ class GaoDePoiDataProcess extends BaseBatch{ } def findSimlarPoi(ret, esMap){ - def keys = esMap.keySet() def simlarObj = null - for(def key : keys){ - def esValue = esMap.get(key) - if(esValue.township && ret.township){ - if(esValue.name.contains(ret.name) || ret.name.contains(esValue.name)){//|| - def distance = GeoUtils.calculateDistance(ret.location.lon, ret.location.lat, esValue.location.lon, esValue.location.lat); - if(distance <100){ - simlarObj = esValue - println esValue.name +" "+ret.name+" "+distance - break; + def region = "" + if(ret.district){ + region = ret.district + } + def regionMap = esMap.get(region) + if(regionMap){ + def keys = regionMap.keySet() + + for(def key : keys){ + def esValue = regionMap.get(key) + if(esValue.township && ret.township){ + if(esValue.name.contains(ret.name) || ret.name.contains(esValue.name)){//|| + def distance = GeoUtils.calculateDistance(ret.location.lon, ret.location.lat, esValue.location.lon, esValue.location.lat); + if(distance <100){ + simlarObj = esValue + println esValue.name +" "+ret.name+" "+distance + break; + } } - } // else{ // def esKey = esValue.name.replace("-","").replace("·","").replace("(", "").replace(")", "") // def bdKey = ret.name.replace("-","").replace("·","").replace("(", "").replace(")", "") @@ -253,8 +263,12 @@ class GaoDePoiDataProcess extends BaseBatch{ // } // } // } + } } + }else{ + println "无数据" } + return simlarObj } @@ -264,13 +278,22 @@ class GaoDePoiDataProcess extends BaseBatch{ // BloomFilter filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001); def map = new HashMap() ElasticUtils89.executeQuery("192.168.0.89", "didata_map_poi_single", queryBuilder, { String id, Map d-> + def district = "" + if(d.district){ + district = d.district + } + def regionMap = map.get(district) + if(!regionMap){ + regionMap = new HashMap() + } def o = new BasicDBObject() o.put("city", d.city) o.put("district", d.district) o.put("name", d.name) o.put("township", d.township) o.put("location", d.location) - map.put(d.city +"##"+d.district+"##"+d.name.replace("-","").replace("·",""), o) + regionMap.put(d.name.replace("-","").replace("·",""), o) + map.put(district, regionMap) }) return map } From 9d97bd6679fdbf81469d94ff99e88506867f2429 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Fri, 30 Aug 2024 10:11:34 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E4=B8=8A=E6=B5=B7=E6=88=90=E4=BA=A4?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=97=B6=E9=97=B4=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy index 936382db2..d3269f5b3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy @@ -580,8 +580,8 @@ class FjwBigdataDealBatch extends BaseBatch{ } def getYouDaData(statMap){ -// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) - def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd")) + def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) +// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd")) def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) def q = new BasicDBObject() q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate)); From 0671c826f7a8063f21b2985f7b035f228045aecb Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Fri, 30 Aug 2024 10:54:07 +0800 Subject: [PATCH 3/3] update --- .../script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy index d3269f5b3..9b84e83c3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy @@ -90,6 +90,7 @@ class FjwBigdataDealBatch extends BaseBatch{ } + println "txt 生成 " + rows.size() saveToPgSql(rows) // def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat") @@ -356,7 +357,7 @@ class FjwBigdataDealBatch extends BaseBatch{ def converData(def trade, def cityCodeDatas, def distCodeMap){ def ret = [] - def data = distCodeMap.get(trade.region) + def data = distCodeMap.get(trade.match_region) if(!data){ println trade.city +" "+trade.region+" 没有找到对应的区域" return ret