城市统计新增写字楼出租价格统计及加载小区换库

This commit is contained in:
2025-03-19 17:32:18 +08:00
parent 1f185ef22a
commit e1ecabc222
@@ -1,8 +1,18 @@
package com.fangjia.script.dataAna
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.core.util.DateUtil
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.Jms
import com.fangjia.script.util.MongoUtils
import com.google.common.collect.Lists
import com.google.common.collect.Sets
import com.mongodb.*
import org.apache.commons.codec.digest.DigestUtils
import java.text.DecimalFormat
import java.util.concurrent.CountDownLatch
@@ -10,30 +20,6 @@ import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.Semaphore
import org.apache.commons.codec.digest.DigestUtils
import org.bson.types.ObjectId
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.Jms
import com.google.common.collect.Lists
import com.google.common.collect.Sets
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.mongodb.Mongo
@BatchOwner(Watcher.LiXue)
class CityInfoAnaBatch extends BaseBatch {
@@ -53,8 +39,10 @@ class CityInfoAnaBatch extends BaseBatch {
final static DBCollection LOUDONG_CONN = MongoUtils.getCollectionByName("fangjia_base", "loudong", MongoUtils.MongoDB_Domain_mongo);
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000);
final static DBCollection FACILITY_CONN = MongoUtils.getCollectionByName("fangjia", "district_facilitys", MongoUtils.MongoDB_Domain_mongo)
final static DBCollection OFFICE_RENT_STAT_CONN = MongoUtils.getCollectionByName("fangjia_stat", "office_rent_stat", MongoUtils.MongoDB_Domain_mongo);
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616')
final static DSCollection SEAWEED = DbUtils.seaweed
// final static DSCollection SEAWEED = DbUtils.seaweed
static final DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
final static def CITYS = [];
@@ -125,7 +113,7 @@ class CityInfoAnaBatch extends BaseBatch {
SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); }
allCitys.each {k,v ->
// if(["南京"].contains(k.toString())){
// if(["北京"].contains(k.toString())){
CITYS << k.toString();
// }
}
@@ -140,72 +128,47 @@ class CityInfoAnaBatch extends BaseBatch {
// 城市小区数
def allCityDisCount = [:];
LinkedHashMap<String, Object> facet = new LinkedHashMap<String, Object>(2);
facet.put("fields", ["city"]);
facet.put('limit', 500);
DBObject disQuery = new BasicDBObject();
disQuery.put('$facet', facet);
disQuery.put("visible", 0);
disQuery.put("cat", 'district');
DSCursor sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
List<FacetResult> disFacetResult = sellDSCursor.getFacetResults();
if (disFacetResult) {
def obj = disFacetResult[0].values;
obj.each {
allCityDisCount.put(it.name, it.count.intValue());
}
BasicDBObject match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "district"));
DBObject groupFields = new BasicDBObject();
groupFields.put("_id", new BasicDBObject("city", '$city'))
groupFields.put("count", new BasicDBObject('$sum', 1));
DBObject group = new BasicDBObject('$group', groupFields);
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
AggregationOutput output = SEAWEED.aggregate(Arrays.asList(match, group));
for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
allCityDisCount.put(obj.get("city"), result.get("count"))
}
// 城市新盘数
def allCityNewDisCount = [:];
disQuery = new BasicDBObject();
disQuery.put('$facet', facet);
disQuery.put("visible", 0);
disQuery.put("cat", 'new_district');
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
disFacetResult = sellDSCursor.getFacetResults();
if (disFacetResult) {
def obj = disFacetResult[0].values;
obj.each {
allCityNewDisCount.put(it.name, it.count.intValue());
}
match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "new_district"));
output = SEAWEED.aggregate(Arrays.asList(match, group));
for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
allCityNewDisCount.put(obj.get("city"), result.get("count"))
}
// 城市写字楼数
def allCityOfficeCount = [:];
disQuery = new BasicDBObject();
disQuery.put('$facet', facet);
disQuery.put("visible", 0);
disQuery.put("cat", 'office');
match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "office"));
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
disFacetResult = sellDSCursor.getFacetResults();
if (disFacetResult) {
def obj = disFacetResult[0].values;
obj.each {
allCityOfficeCount.put(it.name, it.count.intValue());
}
output = SEAWEED.aggregate(Arrays.asList(match, group));
for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
allCityOfficeCount.put(obj.get("city"), result.get("count"))
}
// 城市商铺数
def allCityShopCount = [:];
disQuery = new BasicDBObject();
disQuery.put('$facet', facet);
disQuery.put("visible", 0);
disQuery.put("cat", 'shop');
match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "shop"));
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
disFacetResult = sellDSCursor.getFacetResults();
if (disFacetResult) {
def obj = disFacetResult[0].values;
obj.each {
allCityShopCount.put(it.name, it.count.intValue());
}
output = SEAWEED.aggregate(Arrays.asList(match, group));
for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
allCityShopCount.put(obj.get("city"), result.get("count"))
}
// 城市均价
def allCityStats = [:];
query = new BasicDBObject();
@@ -220,10 +183,10 @@ class CityInfoAnaBatch extends BaseBatch {
doProcess(allCitys.get(it), weekend, laseWeekend, allHouseStats, allCityDisCount, allCityNewDisCount, allCityOfficeCount, allCityShopCount, allCityStats);
}
//统计有楼栋的小区数和有房号的小区数
// countDistrictsHasLoudong(allCitys, weekend)
//countDistrictsHasLoudongNew(allCitys, weekend)
// //统计有楼栋的小区数和有房号的小区数
//// countDistrictsHasLoudong(allCitys, weekend)
// //countDistrictsHasLoudongNew(allCitys, weekend)
//
final CountDownLatch latch = new CountDownLatch(CITYS.size())
final Semaphore semaphore = new Semaphore(10)
CITYS.each {
@@ -239,13 +202,13 @@ class CityInfoAnaBatch extends BaseBatch {
} finally {
semaphore.release()
latch.countDown()
}
}
};
})
}
latch.await()
println "${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())}\t楼栋覆盖率统计完成"
latch = new CountDownLatch(CITYS.size())
semaphore = new Semaphore(10)
CITYS.each {
@@ -267,12 +230,12 @@ class CityInfoAnaBatch extends BaseBatch {
}
latch.await()
println "${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())}\t房号覆盖率统计完成"
//71个城市的覆盖率统计
sendCountData(weekend)
//最后统计之前因为太慢取消统计的本周估价次数
latch = new CountDownLatch(CITYS.size())
semaphore = new Semaphore(10)
@@ -292,14 +255,14 @@ class CityInfoAnaBatch extends BaseBatch {
query.put('city', cityName);
query.put('c_date', new BasicDBObject('$gte', laseWeekend).append('$lte', weekend));
entity.put('evaluation_count', (int) EVALUATION_CONN.count(query));
if (entity.get('evaluation_count')) {
entity.put('price_to_eva', DataUtil.roundDouble(entity.get('price_total_count'), entity.get('evaluation_count')));
}
CITY_INFO_CONN.save(entity)
println cityName+"本周估价次数统计完毕"
}
} catch (Exception e) {
println e.getMessage()
} finally {
@@ -585,8 +548,8 @@ class CityInfoAnaBatch extends BaseBatch {
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 小区完成]"
// if (allCityNewDisCount.get(cityName)) {
// def newDisCount = allCityNewDisCount.get(cityName);
// if (allCityNewDisCount.get(cityName)) {
// def newDisCount = allCityNewDisCount.get(cityName);
// entity.put('new_district_detail', newDistrictDetail(newDisCount, cityName));
// } else {
// entity.put('new_district_detail', [:]);
@@ -604,23 +567,30 @@ class CityInfoAnaBatch extends BaseBatch {
if (officePrices) {
detail.put('price', roundDouble(officePrices.size(), officeCount));
}
query = new BasicDBObject();
query.put('city', cityName);
query.put('kind', '写字楼');
query.put('weekend', weekend);
def officeRents = RENT_STAT_CONN.distinct("name", query);
def officeRents = findOfficeRent(cityName)
if (officeRents) {
detail.put('rentPrice', roundDouble(officeRents.size(), officeCount));
}
def halfYearofficeRentCount = findHalfYearOfficeRentCount(cityName)
if (halfYearofficeRentCount) {
detail.put('halfYearRentPrice', roundDouble(halfYearofficeRentCount, officeCount));
}
entity.put('office_detail', detail);
entity.put('existing_office_count', officeCount);
} else {
entity.put('office_detail', [:]);
}
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 写字楼完成]"
// if(curInfo){
// CITY_INFO_CONN.update(new BasicDBObject("_id", curInfo._id), new BasicDBObject('$set', entity))
// print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 写字楼完成]"
// }
if (entity.get('official_population') && entity.get('district_avg_population')) {
entity.put('prediction_district_count', (int) (entity.get('official_population') / entity.get('district_avg_population')));
@@ -628,12 +598,12 @@ class CityInfoAnaBatch extends BaseBatch {
entity.put('district_coverage', roundDouble(entity.get('existing_district_count'), entity.get('prediction_district_count')));
}
}
if (allCityShopCount.get(cityName)) {
def shopCount = allCityShopCount.get(cityName);
def detail = shopDetail(shopCount, cityName);
DBObject query = new BasicDBObject();
query.put('city', cityName);
//query.put('estate_type', '商铺');
@@ -642,7 +612,7 @@ class CityInfoAnaBatch extends BaseBatch {
if (shopPrices) {
detail.put('price', roundDouble(shopPrices.size(), shopCount));
}
query = new BasicDBObject();
query.put('city', cityName);
query.put('kind', '商铺');
@@ -651,45 +621,41 @@ class CityInfoAnaBatch extends BaseBatch {
if (shopRents) {
detail.put('rentPrice', roundDouble(shopRents.size(), shopCount));
}
entity.put('shop_detail', detail);
entity.put('existing_shop_count', shopCount);
}else {
entity.put('shop_detail', [:]);
}
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 商铺完成]"
def cityStat = allCityStats.get(cityName);
if (cityStat) {
entity.put('avg_price', cityStat.avg_price);
}
DBObject query = new BasicDBObject();
query.put('city', cityName);
query.put('c_date', new BasicDBObject('$gte', lastWeekend).append('$lte', weekend));
//entity.put('evaluation_count', (int) EVALUATION_CONN.count(query));由于估价记录数据量大,查询慢,先去掉这个,最后面在用多线程统计
entity.put('price_total_count', (int) PRICE_JIUCUO_CONN.count(query));
query.put('price_lev', '偏高');
entity.put('price_high_count', (int) PRICE_JIUCUO_CONN.count(query));
query.put('price_lev', '偏低');
entity.put('price_low_count', (int) PRICE_JIUCUO_CONN.count(query));
/*if (entity.get('evaluation_count')) {由于估价记录数据量大,查询慢,先去掉这个,最后面在用多线程统计
entity.put('price_to_eva', roundDouble(entity.get('price_total_count'), entity.get('evaluation_count')));
}*/
if (entity.get('price_high_count')) {
entity.put('low_to_high', roundDouble(entity.get('price_low_count'), entity.get('price_high_count')));
}
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 估价完成]"
// query = new BasicDBObject();
// query.put('city', cityName);
// query.put('weekend', weekend);
@@ -710,7 +676,7 @@ class CityInfoAnaBatch extends BaseBatch {
// }
// }
// }
//
//
// if (total > 0) {
// BigDecimal bg = new BigDecimal(webZhunque * 1.0 / total);
// entity.put('web_average', bg.setScale(4, BigDecimal.ROUND_HALF_UP).doubleValue());
@@ -1284,7 +1250,7 @@ class CityInfoAnaBatch extends BaseBatch {
fields.put(key, 1)
}
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
def cur = SEAWEED.find(query, fields).batchSize(2000)
Iterator<DBObject> iter = cur.iterator()
def countMap = [:]
println cur.count()
@@ -1335,7 +1301,7 @@ class CityInfoAnaBatch extends BaseBatch {
fields.put(key, 1)
}
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
def cur = SEAWEED.find(query, fields).batchSize(2000)
Iterator<DBObject> iter = cur.iterator()
def countMap = [:]
@@ -1546,6 +1512,69 @@ class CityInfoAnaBatch extends BaseBatch {
}
return true;
}
def findOfficeRent(cityName){
def query = new BasicDBObject();
query.put('city', cityName);
// query.put('kind', '写字楼');
// query.put('weekend', weekend);
// def officeRents = RENT_STAT_CONN.distinct("name", query);
// if (officeRents) {
// detail.put('rentPrice', roundDouble(officeRents.size(), officeCount));
// }
query.put('cat', 'office');
def date1 = new Date()
date1.setDate(28)
if(new Date().getTime()> date1.getTime()){
query.put('date', Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")));
}else{
query.put('date', Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")));
}
def officeRents = OFFICE_RENT_STAT_CONN.distinct("name", query);
return officeRents
}
def findHalfYearOfficeRentCount(cityName){
def date1 = new Date()
date1.setDate(28)
def months = []
if(new Date().getTime()> date1.getTime()){
for(def i=1;i<=6;i++){
months << Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1*i, "yyyyMM"))
}
}else{
for(def i=2;i<=7;i++){
months << Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1*i, "yyyyMM"))
}
}
def map = [:]
for(def month : months){
def query = new BasicDBObject();
query.put('city', cityName);
query.put('cat', 'office');
query.put('date', month);
def officeRents = OFFICE_RENT_STAT_CONN.distinct("name", query);
officeRents.each{k->
def count = map.get(k)
if(!count){
map.put(k, 1)
}else{
map.put(k, count + 1)
}
}
}
def total = 0
map.each{k,v->
if(v==6){
total ++
}
}
return total
}
}