58种子优化

This commit is contained in:
沈健
2021-05-14 14:32:09 +08:00
parent f0af12a0d7
commit 07e725fe6e
3 changed files with 94 additions and 9 deletions
@@ -252,23 +252,31 @@ class City58 {
" }";
static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018)
static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
static void main(String[] args) {
JSONObject jsonObject = JSONObject.parseObject(cityList)
Set<String> provinceSet = jsonObject.keySet()
List<DBObject> ret = new ArrayList<>()
List<String> list = cityMapConn.distinct("fangjia_city")
Set<String> citySet = new HashSet<>(list);
for (String province: provinceSet) {
JSONObject provinceInfo = jsonObject[province]
for (String city : provinceInfo.keySet()) {
def city_short = provinceInfo[city].split("\\|")[0]
DBObject dbObject = new BasicDBObject()
dbObject.put("province", province)
dbObject.put("city", city)
dbObject.put("city_short", city_short)
dbObject.put("c_date", new Date())
dbObject.put("m_date", new Date())
dbObject.put("root", "58.com")
ret.add(dbObject)
// 屏蔽县
if (citySet.contains(city)) {
def city_short = provinceInfo[city].split("\\|")[0]
DBObject dbObject = new BasicDBObject()
dbObject.put("province", province)
dbObject.put("city", city)
dbObject.put("city_short", city_short)
dbObject.put("c_date", new Date())
dbObject.put("m_date", new Date())
dbObject.put("root", "58.com")
ret.add(dbObject)
}
}
}
cityConn.insert(ret)
@@ -0,0 +1,44 @@
package com.fangjia.script.temp.shenjian.crawl
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
/**
* 58有效种子梳理
*
* @author shenjian
* @since 2021/5/14
*/
class Seaweed58 {
static DBCollection districtSeedsConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018)
static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
static void main(String[] args) {
List<String> list = cityMapConn.distinct("fangjia_city")
Set<String> citySet = new HashSet<>(list);
DBObject query = new BasicDBObject()
query.put("root", "58.com")
DBObject filter = new BasicDBObject()
filter.put("city", "1")
filter.put("_id", "1")
def delIds = []
districtSeedsConn.find(query, filter).batchSize(20000).each {obj ->
if (!citySet.contains(obj.city)) {
delIds.add(obj._id)
if (delIds.size() == 10000) {
districtSeedsConn.remove(new BasicDBObject("_id", new BasicDBObject('$in', delIds)))
delIds = []
}
}
}
districtSeedsConn.remove(new BasicDBObject("_id", new BasicDBObject('$in', delIds)))
delIds = []
print("58种子清理成功")
}
}
@@ -0,0 +1,33 @@
package com.fangjia.script.temp.shenjian.crawl
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
/**
* 种子数据备份
*
* @author shenjian
* @since 2021/5/14
*/
class SeedBak {
static DBCollection districtSeedsConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018)
static DBCollection aimConn = MongoUtils.getCollectionByName("seeddb_dev", "district_seeds", "192.168.0.235", 27018)
static void main(String[] args) {
List<DBObject> list = new ArrayList<>()
districtSeedsConn.find().batchSize(20000).each {obj ->
list.add(obj)
if (list.size() == 20000) {
aimConn.insert(list)
list.clear()
}
}
aimConn.insert(list)
list.clear()
println("种子备份成功")
}
}