diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 9db36e3d8..42b0e3dc2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -597,9 +597,9 @@ class CommonDatautils { keys.put("region", 1); keys.put("name", 1); keys.put("_id", 1); - DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000); + DSCursor cursor = DbUtils.seaweed.find(q, keys).batchSize(2000); Iterator iters = cursor.iterator() - def result = []; + def result = new HashSet(); try{ while(iters.hasNext()) { DBObject obj = iters.next() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictAddress.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictAddress.groovy new file mode 100644 index 000000000..d85eefdca --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictAddress.groovy @@ -0,0 +1,323 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.json.JsonSlurper +import net.sf.json.JSONObject + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +//补充库中没有详细地址的小区 +class AddDistrictAddress extends BaseBatch{ + + def fullCityMap = CommonDatautils.queryFullNameCity() + def mcaDistMap = CommonDatautils.queryDistNameCityByMapping() + + DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) + public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com") + + + public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") + + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + static main(args) { + runBatch(new AddDistrictAddress(), args) + } + + @Override + public void process(Object args) { +// def citys = CommonDatautils.findAllCity() +// def index = 0 +// def allrets = [] +// for(def c : citys){ +// index ++ +// def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2", "address"]) +// def noRoadInfoDists = noDetailRoadInfoDistricts(districts) +// println c +" "+index +" 原始小区量:"+districts.size() +" 需新增地址量:"+noRoadInfoDists.size() +// noRoadInfoDists = baiduMatch(c, noRoadInfoDists) +// noRoadInfoDists = gaodeMatch(c, noRoadInfoDists) +// +// def rets = [] +// rets << ["id", "城市", "区域", "名称", "地址", "百度原地址", "百度标准地址", "高德原地址", "高德标准地址"] +// for(def d : noRoadInfoDists){ +// rets << [d._id.toString(), d.city, d.region, d.name, d.address, d.baidu_old_addr?d.baidu_old_addr:'',d.baidu_address?d.baidu_address:'', d.gaode_old_addr?d.gaode_old_addr:'', d.gaode_addr?d.gaode_addr:""] +// } +// allrets.addAll(rets) +// if(allrets.size()>50000){ +// ExcelUtils.write("D:\\数据\\小区\\小区模糊地址补充"+index+".xlsx", allrets, "新增地址") +// allrets.clear() +// } +// } + + updateDistAddr() + + } + + def updateDistAddr(){ + ExcelUtils.multiThreadingModify("D:\\数据\\小区\\待新增小区地址.xlsx",{d-> + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + def udata = new BasicDBObject() + udata.put("address", d['J']) + def addr_alias = dist.addr_alias + if(!dist.addr_alias){ + addr_alias = [] + } + if(d['K'] && !addr_alias.contains(d['K'])){ + addr_alias << d['K'] + } + + if(addr_alias){ + udata.put("addr_alias", addr_alias) + } + DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata)) + d['L'] = "已更新" + } + }) + } + + def noDetailRoadInfoDistricts(districts){ + def rets = [] + for(def d : districts){ + if(d.address){ + def flag = DataHandleUtils.isStandardRoad(d.address) + if(!flag){ + rets << d + } + }else{ + rets << d + } + } + return rets; + } + + def baiduMatch(city,districts){ + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + final CountDownLatch latch = new CountDownLatch(districts.size()) + + if(fullCityName){ + def totalCount = new AtomicLong() + for(def i=0;i=100){ + def delquery = new BasicDBObject() + delquery.put("_id", new BasicDBObject('$in', delRets)) + RENT_DIST_STAT_COLL.remove(delquery) + println new Date() + } + } + } + + +}