diff --git a/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/TrendController.java b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/TrendController.java index 34543d7be..e3ee9031a 100644 --- a/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/TrendController.java +++ b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/TrendController.java @@ -287,8 +287,8 @@ public class TrendController { cityStatInfoBo.setSeq(i+1); cityStatInfoBo.setCity(cityName); cityStatInfoBo.setAvgPrice(regionStat.getAvgPrice()); - cityStatInfoBo.setHuanBi((int) (regionStat.getMom()*100)); - cityStatInfoBo.setTongBi((int) (regionStat.getYoy()*100)); + cityStatInfoBo.setHuanBi((int) (regionStat.getMom()*10000)); + cityStatInfoBo.setTongBi((int) (regionStat.getYoy()*10000)); cityStatInfoBos.add(cityStatInfoBo); } return cityStatInfoBos; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/BaseBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/BaseBatch.groovy index baf3e6040..183a4ccf9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/BaseBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/BaseBatch.groovy @@ -114,16 +114,16 @@ abstract class BaseBatch { if(SystemUtils.IS_OS_LINUX){ sendDingDingMsg("ae14f50045e9375c354940a2afb86f14e453bd1f08f78da0ba6e788d35bccec4",msg, phoneNum) - try { - Map params = new HashMap(); - params.put("phone", "13764369683"); - params.put("type", "taskFail"); - params.put("msg", batchName.substring(batchName.length() - 20, batchName.length())); - HttpRequest httpRequest = new HttpRequest(); - httpRequest.get("http://didata-cloud-server.fangjia.com/sms/send", params, "utf-8"); - } catch (Exception ex) { - - } +// try { +// Map params = new HashMap(); +// params.put("phone", "13764369683"); +// params.put("type", "taskFail"); +// params.put("msg", batchName.substring(batchName.length() - 20, batchName.length())); +// HttpRequest httpRequest = new HttpRequest(); +// httpRequest.get("http://didata-cloud-server.fangjia.com/sms/send", params, "utf-8"); +// } catch (Exception ex) { +// +// } task.exception = msg.toString() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorCalcCommandBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorCalcCommandBatch.groovy index 1ae1b4767..a29e9d1b8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorCalcCommandBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorCalcCommandBatch.groovy @@ -1,11 +1,9 @@ package com.fangjia.script.averager -import com.fangjia.core.util.DataUtil +import com.fangjia.core.Constants import com.fangjia.core.util.DateUtil import com.fangjia.core.util.MongoUtils -import com.fangjia.datastore.driver.DSCollection import com.fangjia.datastore.driver.DSCursor -import com.fangjia.datastore.driver.DataStore import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -16,7 +14,6 @@ import com.google.common.collect.MapDifference import com.google.common.collect.Maps import com.mongodb.* import org.bson.types.ObjectId - /** * 通知计算二手房均价 * @author duxiaoyu @@ -26,16 +23,11 @@ import org.bson.types.ObjectId class InsertMonitorCalcCommandBatch extends BaseBatch { - public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") - public final static DBCollection COMMAND_COLL = new Mongo("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command') - public final static DBCollection REGION_STAT_COLL = MongoUtils.getDbCollection(com.fangjia.core.Constants.FANGJIA_MONGO_URI, 'fangjia', 'region_stat') - public final static DBCollection DISTRICT_STAT_COLL = MongoUtils.getDbCollection(com.fangjia.core.Constants.FANGJIA_MONGO_URI, 'fangjia_stat', 'district_stat') - public final static DBCollection REGION_COLL = MongoUtils.getDbCollection(com.fangjia.core.Constants.FANGJIA_MONGO_URI, 'fangjia', 'region_block') - - - - public final static DBCollection DISTRICT_STAT_206_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.206",27020) - + public static final DBCollection SEAWEED_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia', 'seaweed') + public final static DBCollection COMMAND_COLL = MongoUtils.getDbCollection(Constants.LOG_MONGO_URI, 'fangjia_log', 'calc_command') + public final static DBCollection REGION_STAT_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia', 'region_stat') + public final static DBCollection DISTRICT_STAT_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia_stat', 'district_stat') + public final static DBCollection REGION_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia', 'region_block') public final static def WORKER_MACHINE_IPS = ["fangjia-priceanalysis-0", "fangjia-priceanalysis-1","fangjia-priceanalysis-2"] public final static def TOKEN="35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430" @@ -64,22 +56,14 @@ class InsertMonitorCalcCommandBatch extends BaseBatch { final Date startDate = new Date(); for (String city : cities) { - List idList = getAllDistrictIds(city) - logInfo(city + ":" + idList.size()) + List allDistrict = getAllDistrict(city) + logInfo(city + ":" + allDistrict.size()) //生成计算小区队列 - - for (String id : idList) { - DBObject fields = new BasicDBObject() - fields.put("city", 1) - fields.put("region", 1) - fields.put("name", 1) - fields.put("estate_type2", 1) - DBObject district = SEAWEED_COLL.findOne(new BasicDBObject("_id", new ObjectId(id)), fields) - - if (district != null && !districtHasPrice(district.get('city'), district.get('region'), weekend, district.get('name'))) { - String region = district.get('region') - String name = district.get('name') - String estateType = district.get('estate_type2') + for (DBObject district : allDistrict) { + if (!districtHasPrice((String)district.get('city'), (String)district.get('region'), weekend, (String)district.get('name'))) { + String region = (String)district.get('region') + String name = (String)district.get('name') + String estateType = (String)district.get('estate_type2') districtCount++ String ip = WORKER_MACHINE_IPS[districtCount % WORKER_MACHINE_IPS.size()] saveDistrictCalcCommand(city, region, name, estateType, weekend, ip) @@ -93,8 +77,6 @@ class InsertMonitorCalcCommandBatch extends BaseBatch { Thread.sleep(1000 * 5) println("${new Date().format("yyyyMMdd HH:mm:ss")} 等待均价计算完成") } - - for (String city : cities) { //计算该城市的城市、区域、板块均价 List regions = getRegions(city) @@ -177,7 +159,6 @@ class InsertMonitorCalcCommandBatch extends BaseBatch { appliesCount++; } } - DBCollection DISTRICT_STAT_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_stat').getCollection('district_stat') def statMap = [:] def preStatMap = [:] DISTRICT_STAT_COLL.find(new BasicDBObject("weekend", weekend)).batchSize(1000).each { BasicDBObject doc-> @@ -287,35 +268,6 @@ class InsertMonitorCalcCommandBatch extends BaseBatch { boolean isCalcFinish(){ DBObject q = new BasicDBObject("status",0) return COMMAND_COLL.count(q) == 0 - } - void DisSync206To43(Date startDate){ - //数据迁移 - def query = new BasicDBObject() - query.put("m_date", new BasicDBObject('$gte', startDate)) - DBCursor cursor = DISTRICT_STAT_206_COLL.find(query).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(10000) - long count = 0 - long total = cursor.count() - List bs = new ArrayList() - for (BasicDBObject log : cursor) { - bs.add(log) - count++ - if (bs.size() > 1000) { - try { - DISTRICT_STAT_COLL.insert(bs) - } catch (Exception e) { - e.printStackTrace() - } - bs.clear(); - println("数据迁移:${DateUtil.formatByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")} ${total}/${count} 进度:${DataUtil.roundDouble((count * 100l) / total, 2)}") - } - } - try { - DISTRICT_STAT_COLL.insert(bs) - } catch (Exception ex) { - } - - - } /** @@ -340,9 +292,9 @@ class InsertMonitorCalcCommandBatch extends BaseBatch { * @param weekend * @return */ - boolean districtHasPrice(String city, String region, Date weekend,String district_name){ - String ischeckDistrictHasPrice = System.getProperty("ischeckDistrictHasPrice") - if(ischeckDistrictHasPrice!=null && ischeckDistrictHasPrice.trim().equals("true")){ + boolean districtHasPrice(String city, String region, Date weekend, String district_name){ + String isCheckDistrictHasPrice = System.getProperty("isCheckDistrictHasPrice") + if(isCheckDistrictHasPrice!=null && isCheckDistrictHasPrice.trim().equalsIgnoreCase("true")){ def query = new BasicDBObject() query.append("city", city) query.append("region", region) @@ -399,31 +351,20 @@ class InsertMonitorCalcCommandBatch extends BaseBatch { } - private List getAllDistrictIds(String city) { + private List getAllDistrict(String city) { + DBObject q = new BasicDBObject() q.put("city", city) q.put("cat", "district") - q.put("visible", 0) + q.put("status", 0) - int trytimes = 5 - Set uniqueIdList = new LinkedHashSet() + DBObject fields = new BasicDBObject() + fields.put("city", 1) + fields.put("region", 1) + fields.put("name", 1) + fields.put("estate_type2", 1) - for(int i=0; i iter = cur.iterator() - while(iter.hasNext()) { - DBObject dbo = iter.next() - String id = dbo.get('_id') - uniqueIdList.add(id) - } - - if(uniqueIdList.size() == totalCount) { - break - } - } - - return new ArrayList(uniqueIdList) + return SEAWEED_COLL.find(q, fields).batchSize(2000).toArray() } private List getRegions(String city) {