diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy index 5da9281bd..a36b31539 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy @@ -6,7 +6,7 @@ import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.time.DateUtils /** - * 贝壳找房小区种子生成 + * 房天下小区种子生成 */ class CommFang { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy new file mode 100644 index 000000000..0e2baf552 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy @@ -0,0 +1,57 @@ +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils +import org.apache.commons.lang3.time.DateUtils + +/** + * 贝壳找房小区种子生成 + */ +class CommKe { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018) + static final def source = "贝壳找房" + static final def root = "ke.com" + + static void main(String[] args) { + Date c_date = new Date() + DBObject query = new BasicDBObject() + query.put("source",source) + query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd"))); + DBObject filter = new BasicDBObject() + filter.put("url", 1) + filter.put("district_id", 1) + filter.put("city", 1) + filter.put("region", 1) + Set set = new HashSet<>() + List doc = new ArrayList() + collection.find(query, filter).batchSize(1000).each { obj -> + def url = obj.url + def cityShort = url.substring(url.indexOf("https://") + 8, url.indexOf(".")) + def id = obj.district_id + if (!set.contains(id) && StringUtils.isNumeric(id)) { + DBObject dbObject = new BasicDBObject() + dbObject.put("city", obj.city) + dbObject.put("region", obj.region) + + dbObject.put("city_short", cityShort) + dbObject.put("district_id", id) + dbObject.put("root", root) + dbObject.put("c_date", c_date) + dbObject.put("m_date", c_date) + + set.add(id) + doc.add(dbObject) + if (doc.size() == 5000) { + districtSeeds.insert(doc) + doc.clear() + } + } + } + districtSeeds.insert(doc) + doc.clear() + print("操作完成") + } +} \ No newline at end of file