新增58福州小区种子

This commit is contained in:
沈健
2021-04-30 10:38:34 +08:00
parent a2081cf9df
commit 7d05160e81
@@ -0,0 +1,68 @@
package com.fangjia.script.temp.shenjian.crawl
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang3.time.DateUtils
/**
* @author shenjian
* @since 2021/4/30
*/
class Comm58 {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018)
static final def source = "58同城"
static final def root = "58.com"
static void main(String[] args) {
Date c_date = new Date()
DBObject query = new BasicDBObject()
query.put("source",source)
query.put("city","福州")
query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd")));
DBObject filter = new BasicDBObject()
filter.put("district_id", 1)
filter.put("city", 1)
filter.put("region", 1)
Set<String> set = new HashSet<>()
List doc = new ArrayList()
collection.find(query, filter).batchSize(10000).each { obj ->
try {
def url = obj.district_id
def cityShort = "fz"
String[] info = url.split("/")
def id = info[info.size() - 1]
if (!set.contains(id)) {
DBObject dbObject = new BasicDBObject()
dbObject.put("city", obj.city)
dbObject.put("region", obj.region)
dbObject.put("city_short", cityShort)
dbObject.put("district_id", id)
dbObject.put("root", root)
dbObject.put("type", "comm")
dbObject.put("c_date", c_date)
dbObject.put("m_date", c_date)
if (obj.region != null) {
set.add(id)
doc.add(dbObject)
}
if (doc.size() == 5000) {
districtSeeds.insert(doc)
doc.clear()
}
}
} catch(Exception ex) {
println("解析异常:" + obj)
}
}
districtSeeds.insert(doc)
doc.clear()
print("操作完成")
}
}