diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy index 78f033fd4..8b9abf395 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy @@ -252,23 +252,31 @@ class City58 { " }"; static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) static void main(String[] args) { JSONObject jsonObject = JSONObject.parseObject(cityList) Set provinceSet = jsonObject.keySet() List ret = new ArrayList<>() + + List list = cityMapConn.distinct("fangjia_city") + Set citySet = new HashSet<>(list); + for (String province: provinceSet) { JSONObject provinceInfo = jsonObject[province] for (String city : provinceInfo.keySet()) { - def city_short = provinceInfo[city].split("\\|")[0] - DBObject dbObject = new BasicDBObject() - dbObject.put("province", province) - dbObject.put("city", city) - dbObject.put("city_short", city_short) - dbObject.put("c_date", new Date()) - dbObject.put("m_date", new Date()) - dbObject.put("root", "58.com") - ret.add(dbObject) + // 屏蔽县 + if (citySet.contains(city)) { + def city_short = provinceInfo[city].split("\\|")[0] + DBObject dbObject = new BasicDBObject() + dbObject.put("province", province) + dbObject.put("city", city) + dbObject.put("city_short", city_short) + dbObject.put("c_date", new Date()) + dbObject.put("m_date", new Date()) + dbObject.put("root", "58.com") + ret.add(dbObject) + } } } cityConn.insert(ret) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy new file mode 100644 index 000000000..79bf00e9a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy @@ -0,0 +1,44 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 58有效种子梳理 + * + * @author shenjian + * @since 2021/5/14 + */ +class Seaweed58 { + + static DBCollection districtSeedsConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + + static void main(String[] args) { + List list = cityMapConn.distinct("fangjia_city") + Set citySet = new HashSet<>(list); + + DBObject query = new BasicDBObject() + query.put("root", "58.com") + + DBObject filter = new BasicDBObject() + filter.put("city", "1") + filter.put("_id", "1") + def delIds = [] + districtSeedsConn.find(query, filter).batchSize(20000).each {obj -> + if (!citySet.contains(obj.city)) { + delIds.add(obj._id) + if (delIds.size() == 10000) { + districtSeedsConn.remove(new BasicDBObject("_id", new BasicDBObject('$in', delIds))) + delIds = [] + } + } + } + districtSeedsConn.remove(new BasicDBObject("_id", new BasicDBObject('$in', delIds))) + delIds = [] + print("58种子清理成功") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/SeedBak.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/SeedBak.groovy new file mode 100644 index 000000000..56a389f28 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/SeedBak.groovy @@ -0,0 +1,33 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 种子数据备份 + * + * @author shenjian + * @since 2021/5/14 + */ +class SeedBak { + + static DBCollection districtSeedsConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + static DBCollection aimConn = MongoUtils.getCollectionByName("seeddb_dev", "district_seeds", "192.168.0.235", 27018) + + static void main(String[] args) { + List list = new ArrayList<>() + + districtSeedsConn.find().batchSize(20000).each {obj -> + list.add(obj) + if (list.size() == 20000) { + aimConn.insert(list) + list.clear() + } + } + aimConn.insert(list) + list.clear() + println("种子备份成功") + } +}