修改id,同步

This commit is contained in:
2023-10-10 11:02:30 +08:00
parent c5a32dbaad
commit fcdf5a6ee0
@@ -7,6 +7,7 @@ import com.fangjia.script.xingyebank.Data
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
@@ -17,9 +18,9 @@ import java.util.concurrent.atomic.AtomicLong
class PoiWash {
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_all", MAP_POI_URL)
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL)
// DBCollection MAPPOI_CONN2 = MongoUtils.getCollectionByName2("fangjia", "map_poi_all", MAP_POI_URL)
DBCollection MAPPOI_CONN2 = MongoUtils.getCollectionByName2("fangjia", "map_poi_all", MAP_POI_URL)
DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", MAP_POI_URL)
@@ -30,22 +31,56 @@ class PoiWash {
static main(args) {
def task = new PoiWash()
task.updateId()
// task.run()
task.check()
// task.checkSpecialSymbols()
task.synch()
// task.check()
}
// def s = "002乡道与767县道交叉口东北约60米 "
// def s1 = DataUtil.toDBCcase(s)
// println s1
// task.synch()
def updateId(){
def citys = ["上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
for(def city : citys){
def datas = findPois(city)
update(datas)
}
}
def update(datas){
def totalCount = new AtomicLong()
def updateTotalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
def count = datas.size()
for(def i=0;i<datas.size();i++){
def map = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
updateTotalCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("id", map._id.toString())
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city+" 更改id"
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def synch(){
def rets = []
def citys = ["上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
for(def city : citys){
def datas = findPois(city)
println city +" "+datas.size()
def datas = findAllFieldPois(city)
println city +" "+datas.size()+" 同步"
for(def d : datas){
rets << d
if(rets.size()%10000==0){
@@ -62,6 +97,38 @@ class PoiWash {
}
def checkSpecialSymbols(){
def totalCount = new AtomicLong()
def updateTotalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
def count = results.size()
for(def i=0;i<results.size();i++){
def map = results[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
def newAddr = washing(map)
if(map.address!=newAddr){
updateTotalCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("address", newAddr)
MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata))
}
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// });
}
latch.await()
}
def run(){
//"上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"
//"重庆市","三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市"
@@ -88,8 +155,8 @@ class PoiWash {
def count = results.size()
for(def i=0;i<results.size();i++){
def map = results[i]
fixedThreadPool.execute(new Runnable() {
void run() {
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
def newAddr = washing(map)
@@ -105,8 +172,8 @@ class PoiWash {
} finally {
latch.countDown()
}
}
});
// }
// });
}
latch.await()
@@ -167,7 +234,7 @@ class PoiWash {
def findPois(def city){
def q = new BasicDBObject()
q.put("city", city)
// q.put("name", '三明海大饲料有限公司')
// q.put("name", '素匠泰茶')
def keys = new BasicDBObject()
keys.put("_id", 1)
@@ -188,6 +255,21 @@ class PoiWash {
}
def findAllFieldPois(def city){
def q = new BasicDBObject()
q.put("city", city)
def dbcursor = MAPPOI_CONN.find(q).batchSize(10000)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
def updateOldRegion(def city){
def q = new BasicDBObject()
q.put("city", city)