From f2a370ffabf95bccbd8d6d14e3bbf70e121b7881 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 7 Nov 2024 10:41:33 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E8=84=9A=E6=9C=AC=EF=BC=8C=E5=8F=AA=E8=BF=90=E8=A1=8C=E4=B8=8A?= =?UTF-8?q?=E6=B5=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy index f6a8c1208..8e1a842fe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy @@ -37,7 +37,7 @@ class AutoAddDistrictBatch extends BaseBatch{ @Override void process(Object args) { //查找第三方匹配没有匹配上的小区 - def citys = CommonDatautils.findAllCity() + def citys = ["上海"]//CommonDatautils.findAllCity() def list = [] list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"] for(def c : citys){ From 052c96ead66222abfd4b7579057b19bdb6415fab Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 7 Nov 2024 10:42:40 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E4=B9=A1=E9=95=87=E8=A1=97=E9=81=93?= =?UTF-8?q?=E6=A0=87=E5=87=86=E5=8C=96=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/commData/CommonDatautils.groovy | 17 ++-- .../gj/data_handle/RegionStandardBatch.groovy | 80 ++++++++++------- .../gj/data_handle/TownStandardBatch.groovy | 86 ++++++++++++++----- 3 files changed, 124 insertions(+), 59 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 15c76f043..a649c273c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4858,6 +4858,7 @@ class CommonDatautils { keys.put("city",1) keys.put("district",1) keys.put("alias",1) + keys.put("mzb_district_name",1) def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def retMap = [:] for(def d : dbcursor){ @@ -4866,7 +4867,7 @@ class CommonDatautils { regionMap = [:] retMap.put(d.city, regionMap) } - regionMap.put(d.district, d.alias) + regionMap.put(d.district, d) } return retMap @@ -4975,18 +4976,22 @@ class CommonDatautils { def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) def townMap = [:] for(def d : dbcursor){ - def list = townMap.get(d.district) - if(!list){ - list = [] + if(d.mzb_district_name){ + def list = townMap.get(d.mzb_district_name) + if(!list){ + list = [] + } + list << d + townMap.put(d.mzb_district_name, list) } - list << d - townMap.put(d.district, list) } println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss") return townMap } + + static def queryTownByCity(city){ def q = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index 7536d908f..c78e0a84b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -23,6 +23,7 @@ class RegionStandardBatch extends BaseBatch{ def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity() def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity() + static main(args) { runBatch(new RegionStandardBatch(), args) } @@ -37,14 +38,15 @@ class RegionStandardBatch extends BaseBatch{ seaweedRegionStandardByCat("district") } def poiRegionStandard(){ + def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"] def index = 0 - def citys = CommonDatautils.findAllCity() + def citys = ["攀枝花"]//CommonDatautils.findAllCity() for(def city : citys){ index ++ - if(index<32){ - continue - } +// if(index<32){ +// continue +// } if(runCitys.contains(city)){ continue } @@ -74,6 +76,7 @@ class RegionStandardBatch extends BaseBatch{ } + } @@ -195,39 +198,56 @@ class RegionStandardBatch extends BaseBatch{ def poiHaveRegionProcess(city, set, rMap, datas){ def results = [] for(def d : datas){ - if(set.contains(d.district)){ + def tjjObj = rMap.get(d.district) + if(tjjObj && tjjObj.mzb_district_name && tjjObj.mzb_district_name==d.district){ continue } def updateRegion = null - for(def key : set){ - def alias = rMap.get(key) - if(alias.contains(d.district)){ - updateRegion = key - break; + def updateFlag = true + if(tjjObj){ + if(tjjObj.mzb_district_name && tjjObj.mzb_district_name!=d.district){ + updateRegion = tjjObj.mzb_district_name } - } - if(updateRegion){ - def m = [:] - m.put(d.id, ["district":updateRegion]) - results << m - if(results.size()%100==0){ - updateRegionByEs(results) - results.clear() - } - println d.city +" "+d.district+" "+d.name +" "+d.status+" 区域修改为:"+updateRegion - }else{ - def m = [:] - m.put(d.id, ["status":1]) - results << m - if(results.size()%100==0){ - updateRegionByEs(results) - results.clear() + for(def key : set){ + def tempObj = rMap.get(key) + def alias = tempObj.alias + if(alias.contains(d.district) || d.district==tempObj.mzb_district_name){ + if(d.district!=tempObj.mzb_district_name){ + updateRegion = key + }else{ + updateFlag = false + } + break; + } } - - println d.city +" "+d.district+" "+d.name +" "+d.status+" 状态修改为:删除" - } + + if(updateFlag){ + if(updateRegion){ + def m = [:] + m.put(d.id, ["district":updateRegion]) + results << m + if(results.size()%100==0){ + updateRegionByEs(results) + results.clear() + } + + println d.city +" "+d.district+" "+d.name +" "+d.status+" 区域修改为:"+updateRegion + + }else{ + def m = [:] + m.put(d.id, ["status":1]) + results << m + if(results.size()%100==0){ + updateRegionByEs(results) + results.clear() + } + + println d.city +" "+d.district+" "+d.name +" "+d.status+" 状态修改为:删除" + } + } + } if(results.size()>0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy index 1cbbde65c..276887d34 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -1,17 +1,17 @@ package com.fangjia.script.gj.data_handle - import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject +import org.apache.commons.io.FileUtils import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.update.UpdateRequest import org.elasticsearch.client.RequestOptions import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders - /** * 区域标准化 * 1.小区库乡镇街道标准化,包含小区,商铺,写字楼 @@ -23,12 +23,7 @@ class TownStandardBatch extends BaseBatch{ def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity() def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity() - def poiEsUrl = "http://192.168.0.89:9200/didata_map_poi/_search" - - def elasticsearchUrl = "http://192.168.0.89:9200/_search/scroll" - def initialSearchUrl = "http://192.168.0.89:9200/didata_map_poi/_search?scroll=1m" - def scrollTime = "1m" // Scroll context time-to-live - def pageSize = 1000 // Number of hits to return per scroll + def updateList = [] static main(args) { runBatch(new TownStandardBatch(), args) @@ -70,7 +65,8 @@ class TownStandardBatch extends BaseBatch{ } def townStandardByPoi(){ - def citys = ["三明"]//CommonDatautils.findAllCity() + updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"] + def citys = CommonDatautils.findAllCity() for(def city : citys){ def data = fullCityMap.get(city) @@ -79,22 +75,33 @@ class TownStandardBatch extends BaseBatch{ fullCityName = data.city } if(fullCityName){ - def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"]) - def rMap = regionFullNameMap.get(fullCityName) - if(rMap){ - def set = rMap.keySet() - def rets = findEsPoiByCity(fullCityName) - println city +" "+rets.size() - if(set.size()>1){ - poiHaveRegionProcess(fullCityName, townMap, rets) - }else{ + + def fileName = "/home/goojia/gj/poi/${fullCityName}.csv" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\${fullCityName}.csv" + } + File file=new File(fileName); + if(file.exists()){ + def rMap = regionFullNameMap.get(fullCityName) + if(rMap){ + def mzbRegionMap = CommonDatautils.queryMzbRegion(fullCityName) + def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district","mzb_district_name", "street", "alias"]) + def set = rMap.keySet() +// def rets = findEsPoiByCity(fullCityName) + def rets = readPoiCsv(fullCityName) + println city +" "+rets.size() + if(set.size()>1){ + poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap) + }else{ // haveStreetProcess(city, set, cat) + } } } }else{ println "没找到相应的全称:"+city } } + ExcelUtils.write("D:\\数据\\乡镇街道待查数据.xlsx", updateList, "数据") } def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){ def results = [] @@ -149,12 +156,15 @@ class TownStandardBatch extends BaseBatch{ } } } + + } - def poiHaveRegionProcess(fullCityName, townMap, rets){ + def poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap){ def results = [] if(fullCityName){ for(def d : rets){ + if(d.township){ def towns = null def updateTown = null @@ -163,6 +173,8 @@ class TownStandardBatch extends BaseBatch{ if(townDatas){ towns = townDatas break + }else{ + println(d.city +" "+d.district+" 没找下辖街道") } } @@ -193,6 +205,7 @@ class TownStandardBatch extends BaseBatch{ // updateByEs(results) // results.clear() // } + updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown] println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown }else{ def m = [:] @@ -202,6 +215,7 @@ class TownStandardBatch extends BaseBatch{ // updateByEs(results) // results.clear() // } + updateList << [d.id, d.city, d.district, d.name, d.township, d.status, ""] println d.city +" "+d.district+" "+d.name +" "+d.township+" 状态修改为:空值" } } @@ -210,10 +224,10 @@ class TownStandardBatch extends BaseBatch{ } } - if(results.size()>0){ - updateByEs(results) - results.clear() - } +// if(results.size()>0){ +// updateByEs(results) +// results.clear() +// } println fullCityName +" 处理完成" } @@ -276,4 +290,30 @@ class TownStandardBatch extends BaseBatch{ } + def readPoiCsv(filename){ + def datas = readCSV(filename) + return datas + } + + def readCSV(filename){ + def lines = FileUtils.readLines(new File(filename)) + def rets = []; + for(line in lines) { + if(line){ + String[] arrs = line.split("\\|@\\|") + def o = new BasicDBObject() + o.put("city", arrs[0]) + o.put("district", arrs[1]) + o.put("name", arrs[2]) + o.put("township", arrs[3]) + o.put("status", arrs[4]) + o.put("id", arrs[5]) + rets << o + } + + } + return rets; + } + + } From b268f7c9be29d5f8b11f36df3e9a4a1175110b80 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 7 Nov 2024 16:47:31 +0800 Subject: [PATCH 3/3] update --- .../data_handle/AddDistrictShapeBatch.groovy | 67 ++++++++++++++++--- 1 file changed, 56 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy index d250b8944..12879d5ed 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy @@ -10,7 +10,6 @@ import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import com.rabbitmq.client.impl.AMQImpl import groovy.json.JsonSlurper import net.sf.json.JSONObject @@ -19,7 +18,6 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong - /** * 添加高德地图,百度地图围栏 */ @@ -32,6 +30,9 @@ class AddDistrictShapeBatch extends BaseBatch{ DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com") + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) static main(args) { @@ -41,17 +42,20 @@ class AddDistrictShapeBatch extends BaseBatch{ @Override public void process(Object args) { -// findGaoDeDatas2("苏州市","吴中区", "水岸清华瀚宫" ) - def citys = CommonDatautils.findAllCity() - def index = 1 - for(def c : citys){ - println c +" "+index - index ++ - def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"]) +// findGaoDeDatas2("苏州市","吴中区", "水岸清华瀚宫" )//测试 +// def citys = CommonDatautils.findAllCity() +// def index = 1 +// for(def c : citys){ +// println c +" "+index +// index ++ +// def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"]) // baiduMatch(c, districts) - gaodeMatch(c, districts) +// gaodeMatch(c, districts) +// +// } - } +// checkRangeCount() + synch100() } def baiduMatch(city,districts){ @@ -373,4 +377,45 @@ class AddDistrictShapeBatch extends BaseBatch{ } + def checkRangeCount(){ + def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100) + + def count = 0 + def index =0 + for(def d : dbcursor){ + index ++ + if(d.location){ + def coordinates = d.location.coordinates + if(coordinates.size()>1){ + count++ + println count +" "+ d.city +" "+d.name + } + }else{ + println d.city + " "+d.name + } + + if(index%1000==0){ + println index + } + } + + } + + def synch100(){ + def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100) + def rets = [] + for(def d : dbcursor) { + rets << d + if(rets.size()%1000==0){ + seaweed_fence100_CONN.insert(rets) + rets.clear() + } + } + if(rets.size()>0){ + seaweed_fence100_CONN.insert(rets) + rets.clear() + } + } + + }