优化均价算法命令脚本
This commit is contained in:
+25
-84
@@ -1,11 +1,9 @@
|
||||
package com.fangjia.script.averager
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
@@ -16,7 +14,6 @@ import com.google.common.collect.MapDifference
|
||||
import com.google.common.collect.Maps
|
||||
import com.mongodb.*
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
/**
|
||||
* 通知计算二手房均价
|
||||
* @author duxiaoyu
|
||||
@@ -26,16 +23,11 @@ import org.bson.types.ObjectId
|
||||
class InsertMonitorCalcCommandBatch extends BaseBatch {
|
||||
|
||||
|
||||
public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
public final static DBCollection COMMAND_COLL = new Mongo("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command')
|
||||
public final static DBCollection REGION_STAT_COLL = MongoUtils.getDbCollection(com.fangjia.core.Constants.FANGJIA_MONGO_URI, 'fangjia', 'region_stat')
|
||||
public final static DBCollection DISTRICT_STAT_COLL = MongoUtils.getDbCollection(com.fangjia.core.Constants.FANGJIA_MONGO_URI, 'fangjia_stat', 'district_stat')
|
||||
public final static DBCollection REGION_COLL = MongoUtils.getDbCollection(com.fangjia.core.Constants.FANGJIA_MONGO_URI, 'fangjia', 'region_block')
|
||||
|
||||
|
||||
|
||||
public final static DBCollection DISTRICT_STAT_206_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.206",27020)
|
||||
|
||||
public static final DBCollection SEAWEED_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia', 'seaweed')
|
||||
public final static DBCollection COMMAND_COLL = MongoUtils.getDbCollection(Constants.LOG_MONGO_URI, 'fangjia_log', 'calc_command')
|
||||
public final static DBCollection REGION_STAT_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia', 'region_stat')
|
||||
public final static DBCollection DISTRICT_STAT_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia_stat', 'district_stat')
|
||||
public final static DBCollection REGION_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia', 'region_block')
|
||||
public final static def WORKER_MACHINE_IPS = ["fangjia-priceanalysis-0", "fangjia-priceanalysis-1","fangjia-priceanalysis-2"]
|
||||
public final static def TOKEN="35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430"
|
||||
|
||||
@@ -64,22 +56,14 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
|
||||
final Date startDate = new Date();
|
||||
|
||||
for (String city : cities) {
|
||||
List<String> idList = getAllDistrictIds(city)
|
||||
logInfo(city + ":" + idList.size())
|
||||
List<DBObject> allDistrict = getAllDistrict(city)
|
||||
logInfo(city + ":" + allDistrict.size())
|
||||
//生成计算小区队列
|
||||
|
||||
for (String id : idList) {
|
||||
DBObject fields = new BasicDBObject()
|
||||
fields.put("city", 1)
|
||||
fields.put("region", 1)
|
||||
fields.put("name", 1)
|
||||
fields.put("estate_type2", 1)
|
||||
DBObject district = SEAWEED_COLL.findOne(new BasicDBObject("_id", new ObjectId(id)), fields)
|
||||
|
||||
if (district != null && !districtHasPrice(district.get('city'), district.get('region'), weekend, district.get('name'))) {
|
||||
String region = district.get('region')
|
||||
String name = district.get('name')
|
||||
String estateType = district.get('estate_type2')
|
||||
for (DBObject district : allDistrict) {
|
||||
if (!districtHasPrice((String)district.get('city'), (String)district.get('region'), weekend, (String)district.get('name'))) {
|
||||
String region = (String)district.get('region')
|
||||
String name = (String)district.get('name')
|
||||
String estateType = (String)district.get('estate_type2')
|
||||
districtCount++
|
||||
String ip = WORKER_MACHINE_IPS[districtCount % WORKER_MACHINE_IPS.size()]
|
||||
saveDistrictCalcCommand(city, region, name, estateType, weekend, ip)
|
||||
@@ -93,8 +77,6 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
|
||||
Thread.sleep(1000 * 5)
|
||||
println("${new Date().format("yyyyMMdd HH:mm:ss")} 等待均价计算完成")
|
||||
}
|
||||
|
||||
|
||||
for (String city : cities) {
|
||||
//计算该城市的城市、区域、板块均价
|
||||
List<DBObject> regions = getRegions(city)
|
||||
@@ -177,7 +159,6 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
|
||||
appliesCount++;
|
||||
}
|
||||
}
|
||||
DBCollection DISTRICT_STAT_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_stat').getCollection('district_stat')
|
||||
def statMap = [:]
|
||||
def preStatMap = [:]
|
||||
DISTRICT_STAT_COLL.find(new BasicDBObject("weekend", weekend)).batchSize(1000).each { BasicDBObject doc->
|
||||
@@ -287,35 +268,6 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
|
||||
boolean isCalcFinish(){
|
||||
DBObject q = new BasicDBObject("status",0)
|
||||
return COMMAND_COLL.count(q) == 0
|
||||
}
|
||||
void DisSync206To43(Date startDate){
|
||||
//数据迁移
|
||||
def query = new BasicDBObject()
|
||||
query.put("m_date", new BasicDBObject('$gte', startDate))
|
||||
DBCursor cursor = DISTRICT_STAT_206_COLL.find(query).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(10000)
|
||||
long count = 0
|
||||
long total = cursor.count()
|
||||
List<BasicDBObject> bs = new ArrayList<BasicDBObject>()
|
||||
for (BasicDBObject log : cursor) {
|
||||
bs.add(log)
|
||||
count++
|
||||
if (bs.size() > 1000) {
|
||||
try {
|
||||
DISTRICT_STAT_COLL.insert(bs)
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
bs.clear();
|
||||
println("数据迁移:${DateUtil.formatByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")} ${total}/${count} 进度:${DataUtil.roundDouble((count * 100l) / total, 2)}")
|
||||
}
|
||||
}
|
||||
try {
|
||||
DISTRICT_STAT_COLL.insert(bs)
|
||||
} catch (Exception ex) {
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -340,9 +292,9 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
|
||||
* @param weekend
|
||||
* @return
|
||||
*/
|
||||
boolean districtHasPrice(String city, String region, Date weekend,String district_name){
|
||||
String ischeckDistrictHasPrice = System.getProperty("ischeckDistrictHasPrice")
|
||||
if(ischeckDistrictHasPrice!=null && ischeckDistrictHasPrice.trim().equals("true")){
|
||||
boolean districtHasPrice(String city, String region, Date weekend, String district_name){
|
||||
String isCheckDistrictHasPrice = System.getProperty("isCheckDistrictHasPrice")
|
||||
if(isCheckDistrictHasPrice!=null && isCheckDistrictHasPrice.trim().equalsIgnoreCase("true")){
|
||||
def query = new BasicDBObject()
|
||||
query.append("city", city)
|
||||
query.append("region", region)
|
||||
@@ -399,31 +351,20 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
|
||||
|
||||
}
|
||||
|
||||
private List<String> getAllDistrictIds(String city) {
|
||||
private List<DBObject> getAllDistrict(String city) {
|
||||
|
||||
DBObject q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("cat", "district")
|
||||
q.put("visible", 0)
|
||||
q.put("status", 0)
|
||||
|
||||
int trytimes = 5
|
||||
Set<String> uniqueIdList = new LinkedHashSet<String>()
|
||||
DBObject fields = new BasicDBObject()
|
||||
fields.put("city", 1)
|
||||
fields.put("region", 1)
|
||||
fields.put("name", 1)
|
||||
fields.put("estate_type2", 1)
|
||||
|
||||
for(int i=0; i<trytimes; i++) {
|
||||
DSCursor cur = SEAWEED_COLL.find(q, new BasicDBObject("_id", 1)).batchSize(2000)
|
||||
int totalCount = cur.count()
|
||||
Iterator<DBObject> iter = cur.iterator()
|
||||
while(iter.hasNext()) {
|
||||
DBObject dbo = iter.next()
|
||||
String id = dbo.get('_id')
|
||||
uniqueIdList.add(id)
|
||||
}
|
||||
|
||||
if(uniqueIdList.size() == totalCount) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return new ArrayList<String>(uniqueIdList)
|
||||
return SEAWEED_COLL.find(q, fields).batchSize(2000).toArray()
|
||||
}
|
||||
|
||||
private List<DBObject> getRegions(String city) {
|
||||
|
||||
Reference in New Issue
Block a user