diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy new file mode 100644 index 000000000..3aa674810 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy @@ -0,0 +1,68 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.time.DateUtils + +/** + * @author shenjian + * @since 2021/4/30 + */ +class Comm58 { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018) + static final def source = "58同城" + static final def root = "58.com" + + static void main(String[] args) { + Date c_date = new Date() + DBObject query = new BasicDBObject() + query.put("source",source) + query.put("city","福州") + query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd"))); + DBObject filter = new BasicDBObject() + filter.put("district_id", 1) + filter.put("city", 1) + filter.put("region", 1) + Set set = new HashSet<>() + List doc = new ArrayList() + collection.find(query, filter).batchSize(10000).each { obj -> + try { + def url = obj.district_id + def cityShort = "fz" + + String[] info = url.split("/") + def id = info[info.size() - 1] + if (!set.contains(id)) { + DBObject dbObject = new BasicDBObject() + dbObject.put("city", obj.city) + dbObject.put("region", obj.region) + dbObject.put("city_short", cityShort) + dbObject.put("district_id", id) + dbObject.put("root", root) + dbObject.put("type", "comm") + dbObject.put("c_date", c_date) + dbObject.put("m_date", c_date) + + if (obj.region != null) { + set.add(id) + doc.add(dbObject) + } + if (doc.size() == 5000) { + districtSeeds.insert(doc) + doc.clear() + } + } + } catch(Exception ex) { + println("解析异常:" + obj) + } + } + districtSeeds.insert(doc) + doc.clear() + print("操作完成") + } + +}