增加样本过滤阀值

This commit is contained in:
2022-08-09 16:19:46 +08:00
parent 391efb3f63
commit a086686b7d
@@ -50,7 +50,7 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
void process(Object args) {
String cityParam = System.getProperty("city");
String calcDate = System.getProperty("calcDate");
def cities = new ArrayList()
def cities;
if(StringUtils.isNotBlank(cityParam)){
cities = new ArrayList()
for(String c in cityParam.split(",")){
@@ -69,31 +69,22 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
//开始时间
final Date startDate = new Date()
for (String city : cities) {
List<String> idList = getAllDistrictIds(city)
logInfo(city + ":" + idList.size())
List<BasicDBObject> districts = getAllDistricts(city)
logInfo(city + ":" + districts.size())
def buffer = []
//生成计算小区队列
for (String id : idList) {
DBObject fields = new BasicDBObject()
fields.put("city", 1)
fields.put("region", 1)
fields.put("name", 1)
fields.put("estate_type2", 1)
DBObject district = SEAWEED_COLL.findOne(new BasicDBObject("_id", new ObjectId(id)), fields)
if (district != null) {
String region = district.get('region')
String name = district.get('name')
String estateType = district.get('estate_type2')
districtCount++
String ip = WORKER_MACHINE_IPS[districtCount % WORKER_MACHINE_IPS.size()]
BasicDBObject cmd = saveDistrictCalcCommand(city, region, name, estateType, weekend, ip)
logInfo("district:" + city + "-" + region + "-" + name + "-" + estateType + ",weekend:" + DateUtil.formatToYYYYMMDD(weekend) + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date()))
buffer << cmd
if(buffer.size() == 1000){
COMMAND_COLL.insert(buffer)
buffer.clear()
}
for (BasicDBObject district : districts) {
String region = district.get('region')
String name = district.get('name')
String estateType = district.get('estate_type2')
districtCount++
String ip = WORKER_MACHINE_IPS[districtCount % WORKER_MACHINE_IPS.size()]
BasicDBObject cmd = saveDistrictCalcCommand(city, region, name, estateType, weekend, ip)
logInfo("district:" + city + "-" + region + "-" + name + "-" + estateType + ",weekend:" + DateUtil.formatToYYYYMMDD(weekend) + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date()))
buffer << cmd
if(buffer.size() == 1000){
COMMAND_COLL.insert(buffer)
buffer.clear()
}
}
if(!buffer.isEmpty()){
@@ -220,31 +211,34 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
}
private List<String> getAllDistrictIds(String city) {
DBObject q = new BasicDBObject()
q.put("city", city)
q.put("cat", "district")
q.put("visible", 0)
private List<BasicDBObject> getAllDistricts(String city) {
DBObject query = new BasicDBObject()
query.put("city", city)
query.put("cat", "district")
query.put("visible", 0)
int trytimes = 5
Set<String> uniqueIdList = new LinkedHashSet<String>()
DBObject fields = new BasicDBObject()
fields.put("city", 1)
fields.put("region", 1)
fields.put("name", 1)
fields.put("estate_type2", 1)
fields.put("_id", 1)
for(int i=0; i<trytimes; i++) {
DSCursor cur = SEAWEED_COLL.find(q, new BasicDBObject("_id", 1)).batchSize(2000)
int tryTimes = 5
List<BasicDBObject> districts = new ArrayList<BasicDBObject>()
for(int i=0; i < tryTimes; i++) {
DSCursor cur = SEAWEED_COLL.find(query, fields).batchSize(5000)
int totalCount = cur.count()
Iterator<DBObject> iter = cur.iterator()
while(iter.hasNext()) {
DBObject dbo = iter.next()
String id = dbo.get('_id')
uniqueIdList.add(id)
BasicDBObject dbo = (BasicDBObject)iter.next()
districts.add(dbo)
}
if(uniqueIdList.size() == totalCount) {
if(districts.size() == totalCount) {
break
}
}
return new ArrayList<String>(uniqueIdList)
return districts
}
private List<DBObject> getRegions(String city) {