+56
-11
@@ -10,7 +10,6 @@ import com.fangjia.script.util.ElasticUtils
|
|||||||
import com.fangjia.script.util.MongoUtils
|
import com.fangjia.script.util.MongoUtils
|
||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
import com.mongodb.DBCollection
|
import com.mongodb.DBCollection
|
||||||
import com.rabbitmq.client.impl.AMQImpl
|
|
||||||
import groovy.json.JsonSlurper
|
import groovy.json.JsonSlurper
|
||||||
import net.sf.json.JSONObject
|
import net.sf.json.JSONObject
|
||||||
|
|
||||||
@@ -19,7 +18,6 @@ import java.util.concurrent.CountDownLatch
|
|||||||
import java.util.concurrent.ExecutorService
|
import java.util.concurrent.ExecutorService
|
||||||
import java.util.concurrent.Executors
|
import java.util.concurrent.Executors
|
||||||
import java.util.concurrent.atomic.AtomicLong
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 添加高德地图,百度地图围栏
|
* 添加高德地图,百度地图围栏
|
||||||
*/
|
*/
|
||||||
@@ -32,6 +30,9 @@ class AddDistrictShapeBatch extends BaseBatch{
|
|||||||
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||||
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
|
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
|
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||||
|
DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
|
||||||
|
|
||||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
@@ -41,17 +42,20 @@ class AddDistrictShapeBatch extends BaseBatch{
|
|||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
|
|
||||||
// findGaoDeDatas2("苏州市","吴中区", "水岸清华瀚宫" )
|
// findGaoDeDatas2("苏州市","吴中区", "水岸清华瀚宫" )//测试
|
||||||
def citys = CommonDatautils.findAllCity()
|
// def citys = CommonDatautils.findAllCity()
|
||||||
def index = 1
|
// def index = 1
|
||||||
for(def c : citys){
|
// for(def c : citys){
|
||||||
println c +" "+index
|
// println c +" "+index
|
||||||
index ++
|
// index ++
|
||||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"])
|
// def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"])
|
||||||
// baiduMatch(c, districts)
|
// baiduMatch(c, districts)
|
||||||
gaodeMatch(c, districts)
|
// gaodeMatch(c, districts)
|
||||||
|
//
|
||||||
|
// }
|
||||||
|
|
||||||
}
|
// checkRangeCount()
|
||||||
|
synch100()
|
||||||
}
|
}
|
||||||
|
|
||||||
def baiduMatch(city,districts){
|
def baiduMatch(city,districts){
|
||||||
@@ -373,4 +377,45 @@ class AddDistrictShapeBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def checkRangeCount(){
|
||||||
|
def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
|
||||||
|
|
||||||
|
def count = 0
|
||||||
|
def index =0
|
||||||
|
for(def d : dbcursor){
|
||||||
|
index ++
|
||||||
|
if(d.location){
|
||||||
|
def coordinates = d.location.coordinates
|
||||||
|
if(coordinates.size()>1){
|
||||||
|
count++
|
||||||
|
println count +" "+ d.city +" "+d.name
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
println d.city + " "+d.name
|
||||||
|
}
|
||||||
|
|
||||||
|
if(index%1000==0){
|
||||||
|
println index
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
def synch100(){
|
||||||
|
def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
|
||||||
|
def rets = []
|
||||||
|
for(def d : dbcursor) {
|
||||||
|
rets << d
|
||||||
|
if(rets.size()%1000==0){
|
||||||
|
seaweed_fence100_CONN.insert(rets)
|
||||||
|
rets.clear()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(rets.size()>0){
|
||||||
|
seaweed_fence100_CONN.insert(rets)
|
||||||
|
rets.clear()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user