diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy index d250b8944..12879d5ed 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy @@ -10,7 +10,6 @@ import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import com.rabbitmq.client.impl.AMQImpl import groovy.json.JsonSlurper import net.sf.json.JSONObject @@ -19,7 +18,6 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong - /** * 添加高德地图,百度地图围栏 */ @@ -32,6 +30,9 @@ class AddDistrictShapeBatch extends BaseBatch{ DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com") + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) static main(args) { @@ -41,17 +42,20 @@ class AddDistrictShapeBatch extends BaseBatch{ @Override public void process(Object args) { -// findGaoDeDatas2("苏州市","吴中区", "水岸清华瀚宫" ) - def citys = CommonDatautils.findAllCity() - def index = 1 - for(def c : citys){ - println c +" "+index - index ++ - def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"]) +// findGaoDeDatas2("苏州市","吴中区", "水岸清华瀚宫" )//测试 +// def citys = CommonDatautils.findAllCity() +// def index = 1 +// for(def c : citys){ +// println c +" "+index +// index ++ +// def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"]) // baiduMatch(c, districts) - gaodeMatch(c, districts) +// gaodeMatch(c, districts) +// +// } - } +// checkRangeCount() + synch100() } def baiduMatch(city,districts){ @@ -373,4 +377,45 @@ class AddDistrictShapeBatch extends BaseBatch{ } + def checkRangeCount(){ + def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100) + + def count = 0 + def index =0 + for(def d : dbcursor){ + index ++ + if(d.location){ + def coordinates = d.location.coordinates + if(coordinates.size()>1){ + count++ + println count +" "+ d.city +" "+d.name + } + }else{ + println d.city + " "+d.name + } + + if(index%1000==0){ + println index + } + } + + } + + def synch100(){ + def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100) + def rets = [] + for(def d : dbcursor) { + rets << d + if(rets.size()%1000==0){ + seaweed_fence100_CONN.insert(rets) + rets.clear() + } + } + if(rets.size()>0){ + seaweed_fence100_CONN.insert(rets) + rets.clear() + } + } + + }