From 2da4238b3fe87136d90fddd80ef3bb826d9a8cfe Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Mon, 16 Dec 2024 15:39:50 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=A0=E9=99=A4=E5=AE=A4=E7=A7=9F=E9=87=91?= =?UTF-8?q?=E4=B8=AD=E4=B8=8D=E5=AD=98=E5=9C=A8=E7=9A=84=E5=B0=8F=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/commData/CommonDatautils.groovy | 4 +- .../gj/data_handle/AddDistrictAddress.groovy | 323 ++++++++++++++++++ .../DistrictStatRentCleanBatch.groovy | 53 +++ 3 files changed, 378 insertions(+), 2 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictAddress.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictStatRentCleanBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 9db36e3d8..42b0e3dc2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -597,9 +597,9 @@ class CommonDatautils { keys.put("region", 1); keys.put("name", 1); keys.put("_id", 1); - DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000); + DSCursor cursor = DbUtils.seaweed.find(q, keys).batchSize(2000); Iterator iters = cursor.iterator() - def result = []; + def result = new HashSet(); try{ while(iters.hasNext()) { DBObject obj = iters.next() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictAddress.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictAddress.groovy new file mode 100644 index 000000000..d85eefdca --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictAddress.groovy @@ -0,0 +1,323 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.json.JsonSlurper +import net.sf.json.JSONObject + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +//补充库中没有详细地址的小区 +class AddDistrictAddress extends BaseBatch{ + + def fullCityMap = CommonDatautils.queryFullNameCity() + def mcaDistMap = CommonDatautils.queryDistNameCityByMapping() + + DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) + public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com") + + + public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") + + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + static main(args) { + runBatch(new AddDistrictAddress(), args) + } + + @Override + public void process(Object args) { +// def citys = CommonDatautils.findAllCity() +// def index = 0 +// def allrets = [] +// for(def c : citys){ +// index ++ +// def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2", "address"]) +// def noRoadInfoDists = noDetailRoadInfoDistricts(districts) +// println c +" "+index +" 原始小区量:"+districts.size() +" 需新增地址量:"+noRoadInfoDists.size() +// noRoadInfoDists = baiduMatch(c, noRoadInfoDists) +// noRoadInfoDists = gaodeMatch(c, noRoadInfoDists) +// +// def rets = [] +// rets << ["id", "城市", "区域", "名称", "地址", "百度原地址", "百度标准地址", "高德原地址", "高德标准地址"] +// for(def d : noRoadInfoDists){ +// rets << [d._id.toString(), d.city, d.region, d.name, d.address, d.baidu_old_addr?d.baidu_old_addr:'',d.baidu_address?d.baidu_address:'', d.gaode_old_addr?d.gaode_old_addr:'', d.gaode_addr?d.gaode_addr:""] +// } +// allrets.addAll(rets) +// if(allrets.size()>50000){ +// ExcelUtils.write("D:\\数据\\小区\\小区模糊地址补充"+index+".xlsx", allrets, "新增地址") +// allrets.clear() +// } +// } + + updateDistAddr() + + } + + def updateDistAddr(){ + ExcelUtils.multiThreadingModify("D:\\数据\\小区\\待新增小区地址.xlsx",{d-> + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + def udata = new BasicDBObject() + udata.put("address", d['J']) + def addr_alias = dist.addr_alias + if(!dist.addr_alias){ + addr_alias = [] + } + if(d['K'] && !addr_alias.contains(d['K'])){ + addr_alias << d['K'] + } + + if(addr_alias){ + udata.put("addr_alias", addr_alias) + } + DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata)) + d['L'] = "已更新" + } + }) + } + + def noDetailRoadInfoDistricts(districts){ + def rets = [] + for(def d : districts){ + if(d.address){ + def flag = DataHandleUtils.isStandardRoad(d.address) + if(!flag){ + rets << d + } + }else{ + rets << d + } + } + return rets; + } + + def baiduMatch(city,districts){ + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + final CountDownLatch latch = new CountDownLatch(districts.size()) + + if(fullCityName){ + def totalCount = new AtomicLong() + for(def i=0;i=100){ + def delquery = new BasicDBObject() + delquery.put("_id", new BasicDBObject('$in', delRets)) + RENT_DIST_STAT_COLL.remove(delquery) + println new Date() + } + } + } + + +}