新增房天下种子

This commit is contained in:
沈健
2021-04-21 14:39:40 +08:00
parent 7ef0189f77
commit f1e7cc1d44
@@ -0,0 +1,54 @@
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang3.StringUtils
import org.apache.commons.lang3.time.DateUtils
/**
* 贝壳找房小区种子生成
*/
class CommFang {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018)
static void main(String[] args) {
Date c_date = new Date()
DBObject query = new BasicDBObject()
query.put("source","搜房")
query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd")));
DBObject filter = new BasicDBObject()
filter.put("district_id", 1)
filter.put("city", 1)
filter.put("region", 1)
Set<String> set = new HashSet<>()
List doc = new ArrayList()
collection.find(query, filter).batchSize(1000).each { obj ->
def url = obj.district_id
def cityShort = url.substring(url.indexOf("https://") + 8, url.indexOf("."))
def id = url.substring(url.lastIndexOf("/") + 1, url.lastIndexOf("."))
if (!set.contains(id) && StringUtils.isNumeric(id)) {
DBObject dbObject = new BasicDBObject()
dbObject.put("city", obj.city)
dbObject.put("region", obj.region)
dbObject.put("city_short", cityShort)
dbObject.put("district_id", id)
dbObject.put("root", "fang.com")
dbObject.put("c_date", c_date)
dbObject.put("m_date", c_date)
set.add(id)
doc.add(dbObject)
if (doc.size() == 5000) {
districtSeeds.insert(doc)
doc.clear()
}
}
}
districtSeeds.insert(doc)
doc.clear()
print("操作完成")
}
}