From 7d05160e818177be19bfcbda28db1ec5abe9ed94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 30 Apr 2021 10:38:34 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E58=E7=A6=8F=E5=B7=9E=E5=B0=8F?= =?UTF-8?q?=E5=8C=BA=E7=A7=8D=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/crawl/Comm58.groovy | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy new file mode 100644 index 000000000..3aa674810 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy @@ -0,0 +1,68 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.time.DateUtils + +/** + * @author shenjian + * @since 2021/4/30 + */ +class Comm58 { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018) + static final def source = "58同城" + static final def root = "58.com" + + static void main(String[] args) { + Date c_date = new Date() + DBObject query = new BasicDBObject() + query.put("source",source) + query.put("city","福州") + query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd"))); + DBObject filter = new BasicDBObject() + filter.put("district_id", 1) + filter.put("city", 1) + filter.put("region", 1) + Set set = new HashSet<>() + List doc = new ArrayList() + collection.find(query, filter).batchSize(10000).each { obj -> + try { + def url = obj.district_id + def cityShort = "fz" + + String[] info = url.split("/") + def id = info[info.size() - 1] + if (!set.contains(id)) { + DBObject dbObject = new BasicDBObject() + dbObject.put("city", obj.city) + dbObject.put("region", obj.region) + dbObject.put("city_short", cityShort) + dbObject.put("district_id", id) + dbObject.put("root", root) + dbObject.put("type", "comm") + dbObject.put("c_date", c_date) + dbObject.put("m_date", c_date) + + if (obj.region != null) { + set.add(id) + doc.add(dbObject) + } + if (doc.size() == 5000) { + districtSeeds.insert(doc) + doc.clear() + } + } + } catch(Exception ex) { + println("解析异常:" + obj) + } + } + districtSeeds.insert(doc) + doc.clear() + print("操作完成") + } + +}