优化均价算法命令脚本

This commit is contained in:
2025-06-22 17:59:52 +08:00
parent ee73825b89
commit 4bc6975030
@@ -1,11 +1,9 @@
package com.fangjia.script.averager
import com.fangjia.core.util.DataUtil
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.MongoUtils
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -16,7 +14,6 @@ import com.google.common.collect.MapDifference
import com.google.common.collect.Maps
import com.mongodb.*
import org.bson.types.ObjectId
/**
* 通知计算二手房均价
* @author duxiaoyu
@@ -26,16 +23,11 @@ import org.bson.types.ObjectId
class InsertMonitorCalcCommandBatch extends BaseBatch {
public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
public final static DBCollection COMMAND_COLL = new Mongo("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command')
public final static DBCollection REGION_STAT_COLL = MongoUtils.getDbCollection(com.fangjia.core.Constants.FANGJIA_MONGO_URI, 'fangjia', 'region_stat')
public final static DBCollection DISTRICT_STAT_COLL = MongoUtils.getDbCollection(com.fangjia.core.Constants.FANGJIA_MONGO_URI, 'fangjia_stat', 'district_stat')
public final static DBCollection REGION_COLL = MongoUtils.getDbCollection(com.fangjia.core.Constants.FANGJIA_MONGO_URI, 'fangjia', 'region_block')
public final static DBCollection DISTRICT_STAT_206_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.206",27020)
public static final DBCollection SEAWEED_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia', 'seaweed')
public final static DBCollection COMMAND_COLL = MongoUtils.getDbCollection(Constants.LOG_MONGO_URI, 'fangjia_log', 'calc_command')
public final static DBCollection REGION_STAT_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia', 'region_stat')
public final static DBCollection DISTRICT_STAT_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia_stat', 'district_stat')
public final static DBCollection REGION_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia', 'region_block')
public final static def WORKER_MACHINE_IPS = ["fangjia-priceanalysis-0", "fangjia-priceanalysis-1","fangjia-priceanalysis-2"]
public final static def TOKEN="35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430"
@@ -64,22 +56,14 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
final Date startDate = new Date();
for (String city : cities) {
List<String> idList = getAllDistrictIds(city)
logInfo(city + ":" + idList.size())
List<DBObject> allDistrict = getAllDistrict(city)
logInfo(city + ":" + allDistrict.size())
//生成计算小区队列
for (String id : idList) {
DBObject fields = new BasicDBObject()
fields.put("city", 1)
fields.put("region", 1)
fields.put("name", 1)
fields.put("estate_type2", 1)
DBObject district = SEAWEED_COLL.findOne(new BasicDBObject("_id", new ObjectId(id)), fields)
if (district != null && !districtHasPrice(district.get('city'), district.get('region'), weekend, district.get('name'))) {
String region = district.get('region')
String name = district.get('name')
String estateType = district.get('estate_type2')
for (DBObject district : allDistrict) {
if (!districtHasPrice((String)district.get('city'), (String)district.get('region'), weekend, (String)district.get('name'))) {
String region = (String)district.get('region')
String name = (String)district.get('name')
String estateType = (String)district.get('estate_type2')
districtCount++
String ip = WORKER_MACHINE_IPS[districtCount % WORKER_MACHINE_IPS.size()]
saveDistrictCalcCommand(city, region, name, estateType, weekend, ip)
@@ -93,8 +77,6 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
Thread.sleep(1000 * 5)
println("${new Date().format("yyyyMMdd HH:mm:ss")} 等待均价计算完成")
}
for (String city : cities) {
//计算该城市的城市、区域、板块均价
List<DBObject> regions = getRegions(city)
@@ -177,7 +159,6 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
appliesCount++;
}
}
DBCollection DISTRICT_STAT_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_stat').getCollection('district_stat')
def statMap = [:]
def preStatMap = [:]
DISTRICT_STAT_COLL.find(new BasicDBObject("weekend", weekend)).batchSize(1000).each { BasicDBObject doc->
@@ -287,35 +268,6 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
boolean isCalcFinish(){
DBObject q = new BasicDBObject("status",0)
return COMMAND_COLL.count(q) == 0
}
void DisSync206To43(Date startDate){
//数据迁移
def query = new BasicDBObject()
query.put("m_date", new BasicDBObject('$gte', startDate))
DBCursor cursor = DISTRICT_STAT_206_COLL.find(query).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(10000)
long count = 0
long total = cursor.count()
List<BasicDBObject> bs = new ArrayList<BasicDBObject>()
for (BasicDBObject log : cursor) {
bs.add(log)
count++
if (bs.size() > 1000) {
try {
DISTRICT_STAT_COLL.insert(bs)
} catch (Exception e) {
e.printStackTrace()
}
bs.clear();
println("数据迁移:${DateUtil.formatByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")} ${total}/${count} 进度:${DataUtil.roundDouble((count * 100l) / total, 2)}")
}
}
try {
DISTRICT_STAT_COLL.insert(bs)
} catch (Exception ex) {
}
}
/**
@@ -340,9 +292,9 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
* @param weekend
* @return
*/
boolean districtHasPrice(String city, String region, Date weekend,String district_name){
String ischeckDistrictHasPrice = System.getProperty("ischeckDistrictHasPrice")
if(ischeckDistrictHasPrice!=null && ischeckDistrictHasPrice.trim().equals("true")){
boolean districtHasPrice(String city, String region, Date weekend, String district_name){
String isCheckDistrictHasPrice = System.getProperty("isCheckDistrictHasPrice")
if(isCheckDistrictHasPrice!=null && isCheckDistrictHasPrice.trim().equalsIgnoreCase("true")){
def query = new BasicDBObject()
query.append("city", city)
query.append("region", region)
@@ -399,31 +351,20 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
}
private List<String> getAllDistrictIds(String city) {
private List<DBObject> getAllDistrict(String city) {
DBObject q = new BasicDBObject()
q.put("city", city)
q.put("cat", "district")
q.put("visible", 0)
q.put("status", 0)
int trytimes = 5
Set<String> uniqueIdList = new LinkedHashSet<String>()
DBObject fields = new BasicDBObject()
fields.put("city", 1)
fields.put("region", 1)
fields.put("name", 1)
fields.put("estate_type2", 1)
for(int i=0; i<trytimes; i++) {
DSCursor cur = SEAWEED_COLL.find(q, new BasicDBObject("_id", 1)).batchSize(2000)
int totalCount = cur.count()
Iterator<DBObject> iter = cur.iterator()
while(iter.hasNext()) {
DBObject dbo = iter.next()
String id = dbo.get('_id')
uniqueIdList.add(id)
}
if(uniqueIdList.size() == totalCount) {
break
}
}
return new ArrayList<String>(uniqueIdList)
return SEAWEED_COLL.find(q, fields).batchSize(2000).toArray()
}
private List<DBObject> getRegions(String city) {