增加样本过滤阀值
This commit is contained in:
+34
-40
@@ -50,7 +50,7 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
|
||||
void process(Object args) {
|
||||
String cityParam = System.getProperty("city");
|
||||
String calcDate = System.getProperty("calcDate");
|
||||
def cities = new ArrayList()
|
||||
def cities;
|
||||
if(StringUtils.isNotBlank(cityParam)){
|
||||
cities = new ArrayList()
|
||||
for(String c in cityParam.split(",")){
|
||||
@@ -69,31 +69,22 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
|
||||
//开始时间
|
||||
final Date startDate = new Date()
|
||||
for (String city : cities) {
|
||||
List<String> idList = getAllDistrictIds(city)
|
||||
logInfo(city + ":" + idList.size())
|
||||
|
||||
List<BasicDBObject> districts = getAllDistricts(city)
|
||||
logInfo(city + ":" + districts.size())
|
||||
def buffer = []
|
||||
//生成计算小区队列
|
||||
for (String id : idList) {
|
||||
DBObject fields = new BasicDBObject()
|
||||
fields.put("city", 1)
|
||||
fields.put("region", 1)
|
||||
fields.put("name", 1)
|
||||
fields.put("estate_type2", 1)
|
||||
DBObject district = SEAWEED_COLL.findOne(new BasicDBObject("_id", new ObjectId(id)), fields)
|
||||
if (district != null) {
|
||||
String region = district.get('region')
|
||||
String name = district.get('name')
|
||||
String estateType = district.get('estate_type2')
|
||||
districtCount++
|
||||
String ip = WORKER_MACHINE_IPS[districtCount % WORKER_MACHINE_IPS.size()]
|
||||
BasicDBObject cmd = saveDistrictCalcCommand(city, region, name, estateType, weekend, ip)
|
||||
logInfo("district:" + city + "-" + region + "-" + name + "-" + estateType + ",weekend:" + DateUtil.formatToYYYYMMDD(weekend) + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date()))
|
||||
buffer << cmd
|
||||
if(buffer.size() == 1000){
|
||||
COMMAND_COLL.insert(buffer)
|
||||
buffer.clear()
|
||||
}
|
||||
for (BasicDBObject district : districts) {
|
||||
String region = district.get('region')
|
||||
String name = district.get('name')
|
||||
String estateType = district.get('estate_type2')
|
||||
districtCount++
|
||||
String ip = WORKER_MACHINE_IPS[districtCount % WORKER_MACHINE_IPS.size()]
|
||||
BasicDBObject cmd = saveDistrictCalcCommand(city, region, name, estateType, weekend, ip)
|
||||
logInfo("district:" + city + "-" + region + "-" + name + "-" + estateType + ",weekend:" + DateUtil.formatToYYYYMMDD(weekend) + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date()))
|
||||
buffer << cmd
|
||||
if(buffer.size() == 1000){
|
||||
COMMAND_COLL.insert(buffer)
|
||||
buffer.clear()
|
||||
}
|
||||
}
|
||||
if(!buffer.isEmpty()){
|
||||
@@ -220,31 +211,34 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
|
||||
|
||||
}
|
||||
|
||||
private List<String> getAllDistrictIds(String city) {
|
||||
DBObject q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("cat", "district")
|
||||
q.put("visible", 0)
|
||||
private List<BasicDBObject> getAllDistricts(String city) {
|
||||
DBObject query = new BasicDBObject()
|
||||
query.put("city", city)
|
||||
query.put("cat", "district")
|
||||
query.put("visible", 0)
|
||||
|
||||
int trytimes = 5
|
||||
Set<String> uniqueIdList = new LinkedHashSet<String>()
|
||||
DBObject fields = new BasicDBObject()
|
||||
fields.put("city", 1)
|
||||
fields.put("region", 1)
|
||||
fields.put("name", 1)
|
||||
fields.put("estate_type2", 1)
|
||||
fields.put("_id", 1)
|
||||
|
||||
for(int i=0; i<trytimes; i++) {
|
||||
DSCursor cur = SEAWEED_COLL.find(q, new BasicDBObject("_id", 1)).batchSize(2000)
|
||||
int tryTimes = 5
|
||||
List<BasicDBObject> districts = new ArrayList<BasicDBObject>()
|
||||
for(int i=0; i < tryTimes; i++) {
|
||||
DSCursor cur = SEAWEED_COLL.find(query, fields).batchSize(5000)
|
||||
int totalCount = cur.count()
|
||||
Iterator<DBObject> iter = cur.iterator()
|
||||
while(iter.hasNext()) {
|
||||
DBObject dbo = iter.next()
|
||||
String id = dbo.get('_id')
|
||||
uniqueIdList.add(id)
|
||||
BasicDBObject dbo = (BasicDBObject)iter.next()
|
||||
districts.add(dbo)
|
||||
}
|
||||
|
||||
if(uniqueIdList.size() == totalCount) {
|
||||
if(districts.size() == totalCount) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return new ArrayList<String>(uniqueIdList)
|
||||
return districts
|
||||
}
|
||||
|
||||
private List<DBObject> getRegions(String city) {
|
||||
|
||||
Reference in New Issue
Block a user