Merge branch 'script_gj' into 'master'

Script gj

See merge request !49
This commit is contained in:
2021-06-29 14:04:30 +08:00
3 changed files with 73 additions and 17 deletions
@@ -53,10 +53,11 @@ class GaoDeMapCrawl extends BaseBatch{
// def prefix = "district_"
// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
// def categorys = ["190301"]//道路相关
def prefix = "road_"
def categorys = ["190301"]//道路相关
def prefix = "village_"
def categorys = ["190108","190109"]//村庄
// def prefix = "village_"
// def categorys = ["190108","190109"]//村庄
def totalCount = new AtomicLong()
static main(args) {
@@ -114,7 +115,7 @@ class GaoDeMapCrawl extends BaseBatch{
def run(){//,"西安","重庆"
def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
// def citys = CommonUtils.findAllCity()
def totalCount = 0;
for(def city : citys){
@@ -502,18 +502,21 @@ class GaoDeUtils {
}
static String getRandomKey2() {
def KeyList = [
"2af0b8533415a612b98d59bb93d664b9",
"0246c57bbcf24aa246c207b3c99dfe41",
"042328b3f0fa5daf426802d379521d87",
"055d4f1b5b64a2bce18f4fe1f15c2639",
"d5484f5170502405d88bbaffe1b496f0",
"e74ccc4258079d2200199695110cceb1",
"f58629fc799fbc8537ca85046c6d44a5",
"291073f17ee0b963ccb1927ed92bf265",
"2a1fb75990f2b39f6221e50892dd3988"
]
// def KeyList = [
// "2af0b8533415a612b98d59bb93d664b9",
// "0246c57bbcf24aa246c207b3c99dfe41",
// "042328b3f0fa5daf426802d379521d87",
// "055d4f1b5b64a2bce18f4fe1f15c2639",
// "d5484f5170502405d88bbaffe1b496f0",
// "e74ccc4258079d2200199695110cceb1",
// "f58629fc799fbc8537ca85046c6d44a5",
// "291073f17ee0b963ccb1927ed92bf265",
// "2a1fb75990f2b39f6221e50892dd3988"
// ]
def KeyList = [
"10bcb421d2ee4e51ed9268d6dca58ad1"
]
def count = RandomUtils.nextInt(KeyList.size())
return KeyList.get(count);
}
@@ -9,6 +9,8 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
/**
* 浙商本地化部署同步脚本
*/
@@ -186,8 +188,57 @@ class BasicInfoDataSynch extends BaseBatch{
println "[${fjTowns.size()}]个乡镇同步完成"
}
def synchDict(){
def dbcursor = ZS_DICT_CONN.find(new BasicDBObject()).sort(new BasicDBObject("version",-1)).limit(1)
def maxObj = null
for(def dbo : dbcursor){
maxObj = dbo
}
def version = 1
if(maxObj && maxObj.version){
version = maxObj.version + 1
}
def fjDicts = CommonDatautils.getAllDictionary()
def zsDictCursor = ZS_DICT_CONN.find(new BasicDBObject())
def deleteQuery = new BasicDBObject()
if(maxObj && maxObj.version){
deleteQuery.put("version", maxObj.version)
}else{
deleteQuery.put("version", new BasicDBObject(QueryOperators.EXISTS, false))
}
ZS_DICT_CONN.remove(deleteQuery)
if(fjDicts) {
def addDatas = []
for (def fjDict : fjDicts) {
def udata = new BasicDBObject()
udata.put("cat",fjDict.cat)
udata.put("name",fjDict.name)
if(fjDict.address_components){
udata.put("address_components", fjDict.address_components)
}
udata.put("status",0)
udata.put("update_date",new Date().getTime())
udata.put("version", version)
udata.put("_id",fjDict._id)
addDatas << udata
if(addDatas.size()%10000==0){
println addDatas.size()/10000 +" 10000"
ZS_DICT_CONN.insert(addDatas);
addDatas = [];
}
}
if(addDatas.size()>0){
ZS_DICT_CONN.insert(addDatas);
addDatas = [];
}
}
/* def zsDictCursor = ZS_DICT_CONN.find(new BasicDBObject())
def zsDicts = []
def zsDictMap = [:]
def fjDictMap = [:]
@@ -205,6 +256,7 @@ class BasicInfoDataSynch extends BaseBatch{
udata.put("name",fjDict.name)
udata.put("status",0)
udata.put("update_date",new Date().getTime())
udata.put("version", version)
if(zsDictObj){
if(fjDict.cat!=zsDictObj.cat||fjDict.name!=zsDictObj.name||zsDictObj.status!=0){
ZS_DICT_CONN.update(new BasicDBObject("_id",zsDictObj._id), new BasicDBObject('$set',udata))
@@ -235,7 +287,7 @@ class BasicInfoDataSynch extends BaseBatch{
ZS_DICT_CONN.update(new BasicDBObject("_id",zsDict._id), new BasicDBObject('$set',udata))
}
}
}
}*/
println "[${fjDicts.size()}]个字典数据同步完成"
}
def synchDistrict(cat){