diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index e07d6eff0..844282d7e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -53,10 +53,11 @@ class GaoDeMapCrawl extends BaseBatch{ // def prefix = "district_" // def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关 -// def categorys = ["190301"]//道路相关 + def prefix = "road_" + def categorys = ["190301"]//道路相关 - def prefix = "village_" - def categorys = ["190108","190109"]//村庄 +// def prefix = "village_" +// def categorys = ["190108","190109"]//村庄 def totalCount = new AtomicLong() static main(args) { @@ -114,7 +115,7 @@ class GaoDeMapCrawl extends BaseBatch{ def run(){//,"西安","重庆" - def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] // def citys = CommonUtils.findAllCity() def totalCount = 0; for(def city : citys){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy index 81837c698..4313382e5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy @@ -502,18 +502,21 @@ class GaoDeUtils { } static String getRandomKey2() { - def KeyList = [ - "2af0b8533415a612b98d59bb93d664b9", - "0246c57bbcf24aa246c207b3c99dfe41", - "042328b3f0fa5daf426802d379521d87", - "055d4f1b5b64a2bce18f4fe1f15c2639", - "d5484f5170502405d88bbaffe1b496f0", - "e74ccc4258079d2200199695110cceb1", - "f58629fc799fbc8537ca85046c6d44a5", - "291073f17ee0b963ccb1927ed92bf265", - "2a1fb75990f2b39f6221e50892dd3988" - ] +// def KeyList = [ +// "2af0b8533415a612b98d59bb93d664b9", +// "0246c57bbcf24aa246c207b3c99dfe41", +// "042328b3f0fa5daf426802d379521d87", +// "055d4f1b5b64a2bce18f4fe1f15c2639", +// "d5484f5170502405d88bbaffe1b496f0", +// "e74ccc4258079d2200199695110cceb1", +// "f58629fc799fbc8537ca85046c6d44a5", +// "291073f17ee0b963ccb1927ed92bf265", +// "2a1fb75990f2b39f6221e50892dd3988" +// ] + def KeyList = [ + "10bcb421d2ee4e51ed9268d6dca58ad1" + ] def count = RandomUtils.nextInt(KeyList.size()) return KeyList.get(count); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/BasicInfoDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/BasicInfoDataSynch.groovy index 69a998c48..4cdcf5656 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/BasicInfoDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/BasicInfoDataSynch.groovy @@ -9,6 +9,8 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import com.mongodb.QueryOperators + /** * 娴欏晢鏈湴鍖栭儴缃插悓姝ヨ剼鏈� */ @@ -186,8 +188,57 @@ class BasicInfoDataSynch extends BaseBatch{ println "[${fjTowns.size()}]涓埂闀囧悓姝ュ畬鎴�" } def synchDict(){ + def dbcursor = ZS_DICT_CONN.find(new BasicDBObject()).sort(new BasicDBObject("version",-1)).limit(1) + def maxObj = null + for(def dbo : dbcursor){ + maxObj = dbo + } + def version = 1 + if(maxObj && maxObj.version){ + version = maxObj.version + 1 + } + + def fjDicts = CommonDatautils.getAllDictionary() - def zsDictCursor = ZS_DICT_CONN.find(new BasicDBObject()) + + def deleteQuery = new BasicDBObject() + if(maxObj && maxObj.version){ + deleteQuery.put("version", maxObj.version) + }else{ + deleteQuery.put("version", new BasicDBObject(QueryOperators.EXISTS, false)) + } + ZS_DICT_CONN.remove(deleteQuery) + + if(fjDicts) { + def addDatas = [] + for (def fjDict : fjDicts) { + def udata = new BasicDBObject() + udata.put("cat",fjDict.cat) + udata.put("name",fjDict.name) + if(fjDict.address_components){ + udata.put("address_components", fjDict.address_components) + } + udata.put("status",0) + udata.put("update_date",new Date().getTime()) + udata.put("version", version) + udata.put("_id",fjDict._id) + addDatas << udata + if(addDatas.size()%10000==0){ + println addDatas.size()/10000 +" 10000" + ZS_DICT_CONN.insert(addDatas); + addDatas = []; + } + } + if(addDatas.size()>0){ + ZS_DICT_CONN.insert(addDatas); + addDatas = []; + } + } + + + + + /* def zsDictCursor = ZS_DICT_CONN.find(new BasicDBObject()) def zsDicts = [] def zsDictMap = [:] def fjDictMap = [:] @@ -205,6 +256,7 @@ class BasicInfoDataSynch extends BaseBatch{ udata.put("name",fjDict.name) udata.put("status",0) udata.put("update_date",new Date().getTime()) + udata.put("version", version) if(zsDictObj){ if(fjDict.cat!=zsDictObj.cat||fjDict.name!=zsDictObj.name||zsDictObj.status!=0){ ZS_DICT_CONN.update(new BasicDBObject("_id",zsDictObj._id), new BasicDBObject('$set',udata)) @@ -235,7 +287,7 @@ class BasicInfoDataSynch extends BaseBatch{ ZS_DICT_CONN.update(new BasicDBObject("_id",zsDict._id), new BasicDBObject('$set',udata)) } } - } + }*/ println "[${fjDicts.size()}]涓瓧鍏告暟鎹悓姝ュ畬鎴�" } def synchDistrict(cat){