+96
-14
@@ -7,6 +7,7 @@ import com.fangjia.script.xingyebank.Data
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -17,9 +18,9 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
class PoiWash {
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_all", MAP_POI_URL)
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL)
|
||||
|
||||
// DBCollection MAPPOI_CONN2 = MongoUtils.getCollectionByName2("fangjia", "map_poi_all", MAP_POI_URL)
|
||||
DBCollection MAPPOI_CONN2 = MongoUtils.getCollectionByName2("fangjia", "map_poi_all", MAP_POI_URL)
|
||||
|
||||
DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", MAP_POI_URL)
|
||||
|
||||
@@ -30,22 +31,56 @@ class PoiWash {
|
||||
|
||||
static main(args) {
|
||||
def task = new PoiWash()
|
||||
task.updateId()
|
||||
// task.run()
|
||||
task.check()
|
||||
// task.checkSpecialSymbols()
|
||||
task.synch()
|
||||
// task.check()
|
||||
}
|
||||
|
||||
// def s = "002乡道与767县道交叉口东北约60米 "
|
||||
// def s1 = DataUtil.toDBCcase(s)
|
||||
// println s1
|
||||
// task.synch()
|
||||
def updateId(){
|
||||
def citys = ["上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
|
||||
for(def city : citys){
|
||||
def datas = findPois(city)
|
||||
update(datas)
|
||||
}
|
||||
}
|
||||
|
||||
def update(datas){
|
||||
def totalCount = new AtomicLong()
|
||||
def updateTotalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
|
||||
def count = datas.size()
|
||||
for(def i=0;i<datas.size();i++){
|
||||
def map = datas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
updateTotalCount.addAndGet(1)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("id", map._id.toString())
|
||||
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
|
||||
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city+" 更改id"
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def synch(){
|
||||
def rets = []
|
||||
def citys = ["上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
|
||||
for(def city : citys){
|
||||
def datas = findPois(city)
|
||||
println city +" "+datas.size()
|
||||
def datas = findAllFieldPois(city)
|
||||
println city +" "+datas.size()+" 同步"
|
||||
for(def d : datas){
|
||||
rets << d
|
||||
if(rets.size()%10000==0){
|
||||
@@ -62,6 +97,38 @@ class PoiWash {
|
||||
|
||||
}
|
||||
|
||||
def checkSpecialSymbols(){
|
||||
def totalCount = new AtomicLong()
|
||||
def updateTotalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
|
||||
def count = results.size()
|
||||
for(def i=0;i<results.size();i++){
|
||||
def map = results[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def newAddr = washing(map)
|
||||
if(map.address!=newAddr){
|
||||
updateTotalCount.addAndGet(1)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("address", newAddr)
|
||||
MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata))
|
||||
}
|
||||
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def run(){
|
||||
//"上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"
|
||||
//"重庆市","三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市"
|
||||
@@ -88,8 +155,8 @@ class PoiWash {
|
||||
def count = results.size()
|
||||
for(def i=0;i<results.size();i++){
|
||||
def map = results[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def newAddr = washing(map)
|
||||
@@ -105,8 +172,8 @@ class PoiWash {
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
// }
|
||||
// });
|
||||
}
|
||||
|
||||
latch.await()
|
||||
@@ -167,7 +234,7 @@ class PoiWash {
|
||||
def findPois(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
// q.put("name", '三明海大饲料有限公司')
|
||||
// q.put("name", '素匠泰茶')
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id", 1)
|
||||
@@ -188,6 +255,21 @@ class PoiWash {
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def findAllFieldPois(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
|
||||
def dbcursor = MAPPOI_CONN.find(q).batchSize(10000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def updateOldRegion(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
|
||||
Reference in New Issue
Block a user