update
This commit is contained in:
+56
-11
@@ -10,7 +10,6 @@ import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import groovy.json.JsonSlurper
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
@@ -19,7 +18,6 @@ import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 添加高德地图,百度地图围栏
|
||||
*/
|
||||
@@ -32,6 +30,9 @@ class AddDistrictShapeBatch extends BaseBatch{
|
||||
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
@@ -41,17 +42,20 @@ class AddDistrictShapeBatch extends BaseBatch{
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
// findGaoDeDatas2("苏州市","吴中区", "水岸清华瀚宫" )
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 1
|
||||
for(def c : citys){
|
||||
println c +" "+index
|
||||
index ++
|
||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"])
|
||||
// findGaoDeDatas2("苏州市","吴中区", "水岸清华瀚宫" )//测试
|
||||
// def citys = CommonDatautils.findAllCity()
|
||||
// def index = 1
|
||||
// for(def c : citys){
|
||||
// println c +" "+index
|
||||
// index ++
|
||||
// def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"])
|
||||
// baiduMatch(c, districts)
|
||||
gaodeMatch(c, districts)
|
||||
// gaodeMatch(c, districts)
|
||||
//
|
||||
// }
|
||||
|
||||
}
|
||||
// checkRangeCount()
|
||||
synch100()
|
||||
}
|
||||
|
||||
def baiduMatch(city,districts){
|
||||
@@ -373,4 +377,45 @@ class AddDistrictShapeBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def checkRangeCount(){
|
||||
def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
|
||||
|
||||
def count = 0
|
||||
def index =0
|
||||
for(def d : dbcursor){
|
||||
index ++
|
||||
if(d.location){
|
||||
def coordinates = d.location.coordinates
|
||||
if(coordinates.size()>1){
|
||||
count++
|
||||
println count +" "+ d.city +" "+d.name
|
||||
}
|
||||
}else{
|
||||
println d.city + " "+d.name
|
||||
}
|
||||
|
||||
if(index%1000==0){
|
||||
println index
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def synch100(){
|
||||
def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
|
||||
def rets = []
|
||||
for(def d : dbcursor) {
|
||||
rets << d
|
||||
if(rets.size()%1000==0){
|
||||
seaweed_fence100_CONN.insert(rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
if(rets.size()>0){
|
||||
seaweed_fence100_CONN.insert(rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user