diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy index cec0e03a9..ec769d504 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy @@ -50,7 +50,7 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch { void process(Object args) { String cityParam = System.getProperty("city"); String calcDate = System.getProperty("calcDate"); - def cities = new ArrayList() + def cities; if(StringUtils.isNotBlank(cityParam)){ cities = new ArrayList() for(String c in cityParam.split(",")){ @@ -69,31 +69,22 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch { //开始时间 final Date startDate = new Date() for (String city : cities) { - List idList = getAllDistrictIds(city) - logInfo(city + ":" + idList.size()) - + List districts = getAllDistricts(city) + logInfo(city + ":" + districts.size()) def buffer = [] //生成计算小区队列 - for (String id : idList) { - DBObject fields = new BasicDBObject() - fields.put("city", 1) - fields.put("region", 1) - fields.put("name", 1) - fields.put("estate_type2", 1) - DBObject district = SEAWEED_COLL.findOne(new BasicDBObject("_id", new ObjectId(id)), fields) - if (district != null) { - String region = district.get('region') - String name = district.get('name') - String estateType = district.get('estate_type2') - districtCount++ - String ip = WORKER_MACHINE_IPS[districtCount % WORKER_MACHINE_IPS.size()] - BasicDBObject cmd = saveDistrictCalcCommand(city, region, name, estateType, weekend, ip) - logInfo("district:" + city + "-" + region + "-" + name + "-" + estateType + ",weekend:" + DateUtil.formatToYYYYMMDD(weekend) + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date())) - buffer << cmd - if(buffer.size() == 1000){ - COMMAND_COLL.insert(buffer) - buffer.clear() - } + for (BasicDBObject district : districts) { + String region = district.get('region') + String name = district.get('name') + String estateType = district.get('estate_type2') + districtCount++ + String ip = WORKER_MACHINE_IPS[districtCount % WORKER_MACHINE_IPS.size()] + BasicDBObject cmd = saveDistrictCalcCommand(city, region, name, estateType, weekend, ip) + logInfo("district:" + city + "-" + region + "-" + name + "-" + estateType + ",weekend:" + DateUtil.formatToYYYYMMDD(weekend) + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date())) + buffer << cmd + if(buffer.size() == 1000){ + COMMAND_COLL.insert(buffer) + buffer.clear() } } if(!buffer.isEmpty()){ @@ -220,31 +211,34 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch { } - private List getAllDistrictIds(String city) { - DBObject q = new BasicDBObject() - q.put("city", city) - q.put("cat", "district") - q.put("visible", 0) + private List getAllDistricts(String city) { + DBObject query = new BasicDBObject() + query.put("city", city) + query.put("cat", "district") + query.put("visible", 0) - int trytimes = 5 - Set uniqueIdList = new LinkedHashSet() + DBObject fields = new BasicDBObject() + fields.put("city", 1) + fields.put("region", 1) + fields.put("name", 1) + fields.put("estate_type2", 1) + fields.put("_id", 1) - for(int i=0; i districts = new ArrayList() + for(int i=0; i < tryTimes; i++) { + DSCursor cur = SEAWEED_COLL.find(query, fields).batchSize(5000) int totalCount = cur.count() Iterator iter = cur.iterator() while(iter.hasNext()) { - DBObject dbo = iter.next() - String id = dbo.get('_id') - uniqueIdList.add(id) + BasicDBObject dbo = (BasicDBObject)iter.next() + districts.add(dbo) } - - if(uniqueIdList.size() == totalCount) { + if(districts.size() == totalCount) { break } } - - return new ArrayList(uniqueIdList) + return districts } private List getRegions(String city) {