From f1e7cc1d442c4a2278297cbc430542a3aaddd785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 21 Apr 2021 14:39:40 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=88=BF=E5=A4=A9=E4=B8=8B?= =?UTF-8?q?=E7=A7=8D=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/crawl/CommFang.groovy | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy new file mode 100644 index 000000000..5da9281bd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy @@ -0,0 +1,54 @@ +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils +import org.apache.commons.lang3.time.DateUtils + +/** + * 贝壳找房小区种子生成 + */ +class CommFang { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018) + + static void main(String[] args) { + Date c_date = new Date() + DBObject query = new BasicDBObject() + query.put("source","搜房") + query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd"))); + DBObject filter = new BasicDBObject() + filter.put("district_id", 1) + filter.put("city", 1) + filter.put("region", 1) + Set set = new HashSet<>() + List doc = new ArrayList() + collection.find(query, filter).batchSize(1000).each { obj -> + def url = obj.district_id + def cityShort = url.substring(url.indexOf("https://") + 8, url.indexOf(".")) + def id = url.substring(url.lastIndexOf("/") + 1, url.lastIndexOf(".")) + if (!set.contains(id) && StringUtils.isNumeric(id)) { + DBObject dbObject = new BasicDBObject() + dbObject.put("city", obj.city) + dbObject.put("region", obj.region) + + dbObject.put("city_short", cityShort) + dbObject.put("district_id", id) + dbObject.put("root", "fang.com") + dbObject.put("c_date", c_date) + dbObject.put("m_date", c_date) + + set.add(id) + doc.add(dbObject) + if (doc.size() == 5000) { + districtSeeds.insert(doc) + doc.clear() + } + } + } + districtSeeds.insert(doc) + doc.clear() + print("操作完成") + } +} \ No newline at end of file