58种子优化
This commit is contained in:
+17
-9
@@ -252,23 +252,31 @@ class City58 {
|
||||
" }";
|
||||
|
||||
static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018)
|
||||
static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
|
||||
|
||||
static void main(String[] args) {
|
||||
JSONObject jsonObject = JSONObject.parseObject(cityList)
|
||||
Set<String> provinceSet = jsonObject.keySet()
|
||||
List<DBObject> ret = new ArrayList<>()
|
||||
|
||||
List<String> list = cityMapConn.distinct("fangjia_city")
|
||||
Set<String> citySet = new HashSet<>(list);
|
||||
|
||||
for (String province: provinceSet) {
|
||||
JSONObject provinceInfo = jsonObject[province]
|
||||
for (String city : provinceInfo.keySet()) {
|
||||
def city_short = provinceInfo[city].split("\\|")[0]
|
||||
DBObject dbObject = new BasicDBObject()
|
||||
dbObject.put("province", province)
|
||||
dbObject.put("city", city)
|
||||
dbObject.put("city_short", city_short)
|
||||
dbObject.put("c_date", new Date())
|
||||
dbObject.put("m_date", new Date())
|
||||
dbObject.put("root", "58.com")
|
||||
ret.add(dbObject)
|
||||
// 屏蔽县
|
||||
if (citySet.contains(city)) {
|
||||
def city_short = provinceInfo[city].split("\\|")[0]
|
||||
DBObject dbObject = new BasicDBObject()
|
||||
dbObject.put("province", province)
|
||||
dbObject.put("city", city)
|
||||
dbObject.put("city_short", city_short)
|
||||
dbObject.put("c_date", new Date())
|
||||
dbObject.put("m_date", new Date())
|
||||
dbObject.put("root", "58.com")
|
||||
ret.add(dbObject)
|
||||
}
|
||||
}
|
||||
}
|
||||
cityConn.insert(ret)
|
||||
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package com.fangjia.script.temp.shenjian.crawl
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
/**
|
||||
* 58有效种子梳理
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/5/14
|
||||
*/
|
||||
class Seaweed58 {
|
||||
|
||||
static DBCollection districtSeedsConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018)
|
||||
static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
|
||||
|
||||
static void main(String[] args) {
|
||||
List<String> list = cityMapConn.distinct("fangjia_city")
|
||||
Set<String> citySet = new HashSet<>(list);
|
||||
|
||||
DBObject query = new BasicDBObject()
|
||||
query.put("root", "58.com")
|
||||
|
||||
DBObject filter = new BasicDBObject()
|
||||
filter.put("city", "1")
|
||||
filter.put("_id", "1")
|
||||
def delIds = []
|
||||
districtSeedsConn.find(query, filter).batchSize(20000).each {obj ->
|
||||
if (!citySet.contains(obj.city)) {
|
||||
delIds.add(obj._id)
|
||||
if (delIds.size() == 10000) {
|
||||
districtSeedsConn.remove(new BasicDBObject("_id", new BasicDBObject('$in', delIds)))
|
||||
delIds = []
|
||||
}
|
||||
}
|
||||
}
|
||||
districtSeedsConn.remove(new BasicDBObject("_id", new BasicDBObject('$in', delIds)))
|
||||
delIds = []
|
||||
print("58种子清理成功")
|
||||
}
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
package com.fangjia.script.temp.shenjian.crawl
|
||||
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
/**
|
||||
* 种子数据备份
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/5/14
|
||||
*/
|
||||
class SeedBak {
|
||||
|
||||
static DBCollection districtSeedsConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018)
|
||||
static DBCollection aimConn = MongoUtils.getCollectionByName("seeddb_dev", "district_seeds", "192.168.0.235", 27018)
|
||||
|
||||
static void main(String[] args) {
|
||||
List<DBObject> list = new ArrayList<>()
|
||||
|
||||
districtSeedsConn.find().batchSize(20000).each {obj ->
|
||||
list.add(obj)
|
||||
if (list.size() == 20000) {
|
||||
aimConn.insert(list)
|
||||
list.clear()
|
||||
}
|
||||
}
|
||||
aimConn.insert(list)
|
||||
list.clear()
|
||||
println("种子备份成功")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user