新增房天下种子
This commit is contained in:
+54
@@ -0,0 +1,54 @@
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.apache.commons.lang3.time.DateUtils
|
||||
|
||||
/**
|
||||
* 贝壳找房小区种子生成
|
||||
*/
|
||||
class CommFang {
|
||||
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
|
||||
static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018)
|
||||
|
||||
static void main(String[] args) {
|
||||
Date c_date = new Date()
|
||||
DBObject query = new BasicDBObject()
|
||||
query.put("source","搜房")
|
||||
query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd")));
|
||||
DBObject filter = new BasicDBObject()
|
||||
filter.put("district_id", 1)
|
||||
filter.put("city", 1)
|
||||
filter.put("region", 1)
|
||||
Set<String> set = new HashSet<>()
|
||||
List doc = new ArrayList()
|
||||
collection.find(query, filter).batchSize(1000).each { obj ->
|
||||
def url = obj.district_id
|
||||
def cityShort = url.substring(url.indexOf("https://") + 8, url.indexOf("."))
|
||||
def id = url.substring(url.lastIndexOf("/") + 1, url.lastIndexOf("."))
|
||||
if (!set.contains(id) && StringUtils.isNumeric(id)) {
|
||||
DBObject dbObject = new BasicDBObject()
|
||||
dbObject.put("city", obj.city)
|
||||
dbObject.put("region", obj.region)
|
||||
|
||||
dbObject.put("city_short", cityShort)
|
||||
dbObject.put("district_id", id)
|
||||
dbObject.put("root", "fang.com")
|
||||
dbObject.put("c_date", c_date)
|
||||
dbObject.put("m_date", c_date)
|
||||
|
||||
set.add(id)
|
||||
doc.add(dbObject)
|
||||
if (doc.size() == 5000) {
|
||||
districtSeeds.insert(doc)
|
||||
doc.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
districtSeeds.insert(doc)
|
||||
doc.clear()
|
||||
print("操作完成")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user