新增贝壳找房小区种子

This commit is contained in:
sjshenjian
2021-04-21 15:21:34 +08:00
parent f1e7cc1d44
commit 040bbc5eca
2 changed files with 58 additions and 1 deletions
@@ -6,7 +6,7 @@ import org.apache.commons.lang3.StringUtils
import org.apache.commons.lang3.time.DateUtils
/**
* 贝壳找房小区种子生成
* 房天下小区种子生成
*/
class CommFang {
@@ -0,0 +1,57 @@
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang3.StringUtils
import org.apache.commons.lang3.time.DateUtils
/**
* 贝壳找房小区种子生成
*/
class CommKe {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018)
static final def source = "贝壳找房"
static final def root = "ke.com"
static void main(String[] args) {
Date c_date = new Date()
DBObject query = new BasicDBObject()
query.put("source",source)
query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd")));
DBObject filter = new BasicDBObject()
filter.put("url", 1)
filter.put("district_id", 1)
filter.put("city", 1)
filter.put("region", 1)
Set<String> set = new HashSet<>()
List doc = new ArrayList()
collection.find(query, filter).batchSize(1000).each { obj ->
def url = obj.url
def cityShort = url.substring(url.indexOf("https://") + 8, url.indexOf("."))
def id = obj.district_id
if (!set.contains(id) && StringUtils.isNumeric(id)) {
DBObject dbObject = new BasicDBObject()
dbObject.put("city", obj.city)
dbObject.put("region", obj.region)
dbObject.put("city_short", cityShort)
dbObject.put("district_id", id)
dbObject.put("root", root)
dbObject.put("c_date", c_date)
dbObject.put("m_date", c_date)
set.add(id)
doc.add(dbObject)
if (doc.size() == 5000) {
districtSeeds.insert(doc)
doc.clear()
}
}
}
districtSeeds.insert(doc)
doc.clear()
print("操作完成")
}
}