Merge branch 'script_gj' into 'master'

update

See merge request !1003
This commit is contained in:
2024-11-07 16:41:27 +08:00
@@ -10,7 +10,6 @@ import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import groovy.json.JsonSlurper
import net.sf.json.JSONObject
@@ -19,7 +18,6 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 添加高德地图,百度地图围栏
*/
@@ -32,6 +30,9 @@ class AddDistrictShapeBatch extends BaseBatch{
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
@@ -41,17 +42,20 @@ class AddDistrictShapeBatch extends BaseBatch{
@Override
public void process(Object args) {
// findGaoDeDatas2("苏州市","吴中区", "水岸清华瀚宫" )
def citys = CommonDatautils.findAllCity()
def index = 1
for(def c : citys){
println c +" "+index
index ++
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"])
// findGaoDeDatas2("苏州市","吴中区", "水岸清华瀚宫" )//测试
// def citys = CommonDatautils.findAllCity()
// def index = 1
// for(def c : citys){
// println c +" "+index
// index ++
// def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"])
// baiduMatch(c, districts)
gaodeMatch(c, districts)
// gaodeMatch(c, districts)
//
// }
}
// checkRangeCount()
synch100()
}
def baiduMatch(city,districts){
@@ -373,4 +377,45 @@ class AddDistrictShapeBatch extends BaseBatch{
}
def checkRangeCount(){
def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
def count = 0
def index =0
for(def d : dbcursor){
index ++
if(d.location){
def coordinates = d.location.coordinates
if(coordinates.size()>1){
count++
println count +" "+ d.city +" "+d.name
}
}else{
println d.city + " "+d.name
}
if(index%1000==0){
println index
}
}
}
def synch100(){
def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
def rets = []
for(def d : dbcursor) {
rets << d
if(rets.size()%1000==0){
seaweed_fence100_CONN.insert(rets)
rets.clear()
}
}
if(rets.size()>0){
seaweed_fence100_CONN.insert(rets)
rets.clear()
}
}
}