城市统计新增写字楼出租价格统计及加载小区换库
This commit is contained in:
+150
-121
@@ -1,8 +1,18 @@
|
||||
package com.fangjia.script.dataAna
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.Jms
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.google.common.collect.Lists
|
||||
import com.google.common.collect.Sets
|
||||
import com.mongodb.*
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
|
||||
import java.text.DecimalFormat
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -10,30 +20,6 @@ import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.Semaphore
|
||||
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.datastore.driver.FacetResult
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.Jms
|
||||
import com.google.common.collect.Lists
|
||||
import com.google.common.collect.Sets
|
||||
import com.mongodb.BasicDBList
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.Mongo
|
||||
|
||||
@BatchOwner(Watcher.LiXue)
|
||||
class CityInfoAnaBatch extends BaseBatch {
|
||||
|
||||
@@ -53,8 +39,10 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
final static DBCollection LOUDONG_CONN = MongoUtils.getCollectionByName("fangjia_base", "loudong", MongoUtils.MongoDB_Domain_mongo);
|
||||
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000);
|
||||
final static DBCollection FACILITY_CONN = MongoUtils.getCollectionByName("fangjia", "district_facilitys", MongoUtils.MongoDB_Domain_mongo)
|
||||
final static DBCollection OFFICE_RENT_STAT_CONN = MongoUtils.getCollectionByName("fangjia_stat", "office_rent_stat", MongoUtils.MongoDB_Domain_mongo);
|
||||
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616')
|
||||
final static DSCollection SEAWEED = DbUtils.seaweed
|
||||
// final static DSCollection SEAWEED = DbUtils.seaweed
|
||||
static final DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
final static def CITYS = [];
|
||||
@@ -125,7 +113,7 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); }
|
||||
|
||||
allCitys.each {k,v ->
|
||||
// if(["南京"].contains(k.toString())){
|
||||
// if(["北京"].contains(k.toString())){
|
||||
CITYS << k.toString();
|
||||
// }
|
||||
}
|
||||
@@ -140,72 +128,47 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
|
||||
// 城市小区数
|
||||
def allCityDisCount = [:];
|
||||
LinkedHashMap<String, Object> facet = new LinkedHashMap<String, Object>(2);
|
||||
facet.put("fields", ["city"]);
|
||||
facet.put('limit', 500);
|
||||
|
||||
DBObject disQuery = new BasicDBObject();
|
||||
disQuery.put('$facet', facet);
|
||||
disQuery.put("visible", 0);
|
||||
disQuery.put("cat", 'district');
|
||||
|
||||
DSCursor sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
|
||||
List<FacetResult> disFacetResult = sellDSCursor.getFacetResults();
|
||||
if (disFacetResult) {
|
||||
def obj = disFacetResult[0].values;
|
||||
obj.each {
|
||||
allCityDisCount.put(it.name, it.count.intValue());
|
||||
}
|
||||
BasicDBObject match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "district"));
|
||||
DBObject groupFields = new BasicDBObject();
|
||||
groupFields.put("_id", new BasicDBObject("city", '$city'))
|
||||
groupFields.put("count", new BasicDBObject('$sum', 1));
|
||||
DBObject group = new BasicDBObject('$group', groupFields);
|
||||
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
|
||||
AggregationOutput output = SEAWEED.aggregate(Arrays.asList(match, group));
|
||||
for (DBObject result : output.results()) {
|
||||
def obj = (DBObject ) result.get("_id");
|
||||
allCityDisCount.put(obj.get("city"), result.get("count"))
|
||||
}
|
||||
|
||||
// 城市新盘数
|
||||
def allCityNewDisCount = [:];
|
||||
disQuery = new BasicDBObject();
|
||||
disQuery.put('$facet', facet);
|
||||
disQuery.put("visible", 0);
|
||||
disQuery.put("cat", 'new_district');
|
||||
|
||||
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
|
||||
disFacetResult = sellDSCursor.getFacetResults();
|
||||
if (disFacetResult) {
|
||||
def obj = disFacetResult[0].values;
|
||||
obj.each {
|
||||
allCityNewDisCount.put(it.name, it.count.intValue());
|
||||
}
|
||||
match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "new_district"));
|
||||
output = SEAWEED.aggregate(Arrays.asList(match, group));
|
||||
for (DBObject result : output.results()) {
|
||||
def obj = (DBObject ) result.get("_id");
|
||||
allCityNewDisCount.put(obj.get("city"), result.get("count"))
|
||||
}
|
||||
|
||||
// 城市写字楼数
|
||||
def allCityOfficeCount = [:];
|
||||
disQuery = new BasicDBObject();
|
||||
disQuery.put('$facet', facet);
|
||||
disQuery.put("visible", 0);
|
||||
disQuery.put("cat", 'office');
|
||||
match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "office"));
|
||||
|
||||
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
|
||||
disFacetResult = sellDSCursor.getFacetResults();
|
||||
if (disFacetResult) {
|
||||
def obj = disFacetResult[0].values;
|
||||
obj.each {
|
||||
allCityOfficeCount.put(it.name, it.count.intValue());
|
||||
}
|
||||
output = SEAWEED.aggregate(Arrays.asList(match, group));
|
||||
for (DBObject result : output.results()) {
|
||||
def obj = (DBObject ) result.get("_id");
|
||||
allCityOfficeCount.put(obj.get("city"), result.get("count"))
|
||||
}
|
||||
|
||||
|
||||
// 城市商铺数
|
||||
def allCityShopCount = [:];
|
||||
disQuery = new BasicDBObject();
|
||||
disQuery.put('$facet', facet);
|
||||
disQuery.put("visible", 0);
|
||||
disQuery.put("cat", 'shop');
|
||||
match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "shop"));
|
||||
|
||||
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
|
||||
disFacetResult = sellDSCursor.getFacetResults();
|
||||
if (disFacetResult) {
|
||||
def obj = disFacetResult[0].values;
|
||||
obj.each {
|
||||
allCityShopCount.put(it.name, it.count.intValue());
|
||||
}
|
||||
output = SEAWEED.aggregate(Arrays.asList(match, group));
|
||||
for (DBObject result : output.results()) {
|
||||
def obj = (DBObject ) result.get("_id");
|
||||
allCityShopCount.put(obj.get("city"), result.get("count"))
|
||||
}
|
||||
|
||||
// 城市均价
|
||||
def allCityStats = [:];
|
||||
query = new BasicDBObject();
|
||||
@@ -220,10 +183,10 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
doProcess(allCitys.get(it), weekend, laseWeekend, allHouseStats, allCityDisCount, allCityNewDisCount, allCityOfficeCount, allCityShopCount, allCityStats);
|
||||
}
|
||||
|
||||
//统计有楼栋的小区数和有房号的小区数
|
||||
// countDistrictsHasLoudong(allCitys, weekend)
|
||||
//countDistrictsHasLoudongNew(allCitys, weekend)
|
||||
|
||||
// //统计有楼栋的小区数和有房号的小区数
|
||||
//// countDistrictsHasLoudong(allCitys, weekend)
|
||||
// //countDistrictsHasLoudongNew(allCitys, weekend)
|
||||
//
|
||||
final CountDownLatch latch = new CountDownLatch(CITYS.size())
|
||||
final Semaphore semaphore = new Semaphore(10)
|
||||
CITYS.each {
|
||||
@@ -239,13 +202,13 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
} finally {
|
||||
semaphore.release()
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
println "${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())}\t楼栋覆盖率统计完成"
|
||||
|
||||
|
||||
latch = new CountDownLatch(CITYS.size())
|
||||
semaphore = new Semaphore(10)
|
||||
CITYS.each {
|
||||
@@ -267,12 +230,12 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
}
|
||||
latch.await()
|
||||
println "${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())}\t房号覆盖率统计完成"
|
||||
|
||||
|
||||
|
||||
|
||||
//71个城市的覆盖率统计
|
||||
sendCountData(weekend)
|
||||
|
||||
|
||||
|
||||
|
||||
//最后统计之前因为太慢取消统计的本周估价次数
|
||||
latch = new CountDownLatch(CITYS.size())
|
||||
semaphore = new Semaphore(10)
|
||||
@@ -292,14 +255,14 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
query.put('city', cityName);
|
||||
query.put('c_date', new BasicDBObject('$gte', laseWeekend).append('$lte', weekend));
|
||||
entity.put('evaluation_count', (int) EVALUATION_CONN.count(query));
|
||||
|
||||
|
||||
if (entity.get('evaluation_count')) {
|
||||
entity.put('price_to_eva', DataUtil.roundDouble(entity.get('price_total_count'), entity.get('evaluation_count')));
|
||||
}
|
||||
CITY_INFO_CONN.save(entity)
|
||||
println cityName+"本周估价次数统计完毕"
|
||||
}
|
||||
|
||||
|
||||
} catch (Exception e) {
|
||||
println e.getMessage()
|
||||
} finally {
|
||||
@@ -585,8 +548,8 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 小区完成]"
|
||||
|
||||
|
||||
// if (allCityNewDisCount.get(cityName)) {
|
||||
// def newDisCount = allCityNewDisCount.get(cityName);
|
||||
// if (allCityNewDisCount.get(cityName)) {
|
||||
// def newDisCount = allCityNewDisCount.get(cityName);
|
||||
// entity.put('new_district_detail', newDistrictDetail(newDisCount, cityName));
|
||||
// } else {
|
||||
// entity.put('new_district_detail', [:]);
|
||||
@@ -604,23 +567,30 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
if (officePrices) {
|
||||
detail.put('price', roundDouble(officePrices.size(), officeCount));
|
||||
}
|
||||
|
||||
query = new BasicDBObject();
|
||||
query.put('city', cityName);
|
||||
query.put('kind', '写字楼');
|
||||
query.put('weekend', weekend);
|
||||
def officeRents = RENT_STAT_CONN.distinct("name", query);
|
||||
|
||||
def officeRents = findOfficeRent(cityName)
|
||||
if (officeRents) {
|
||||
detail.put('rentPrice', roundDouble(officeRents.size(), officeCount));
|
||||
}
|
||||
|
||||
def halfYearofficeRentCount = findHalfYearOfficeRentCount(cityName)
|
||||
if (halfYearofficeRentCount) {
|
||||
detail.put('halfYearRentPrice', roundDouble(halfYearofficeRentCount, officeCount));
|
||||
}
|
||||
|
||||
|
||||
entity.put('office_detail', detail);
|
||||
entity.put('existing_office_count', officeCount);
|
||||
} else {
|
||||
entity.put('office_detail', [:]);
|
||||
}
|
||||
|
||||
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 写字楼完成]"
|
||||
|
||||
// if(curInfo){
|
||||
// CITY_INFO_CONN.update(new BasicDBObject("_id", curInfo._id), new BasicDBObject('$set', entity))
|
||||
// print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 写字楼完成]"
|
||||
// }
|
||||
|
||||
|
||||
if (entity.get('official_population') && entity.get('district_avg_population')) {
|
||||
entity.put('prediction_district_count', (int) (entity.get('official_population') / entity.get('district_avg_population')));
|
||||
|
||||
@@ -628,12 +598,12 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
entity.put('district_coverage', roundDouble(entity.get('existing_district_count'), entity.get('prediction_district_count')));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
if (allCityShopCount.get(cityName)) {
|
||||
def shopCount = allCityShopCount.get(cityName);
|
||||
def detail = shopDetail(shopCount, cityName);
|
||||
|
||||
|
||||
DBObject query = new BasicDBObject();
|
||||
query.put('city', cityName);
|
||||
//query.put('estate_type', '商铺');
|
||||
@@ -642,7 +612,7 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
if (shopPrices) {
|
||||
detail.put('price', roundDouble(shopPrices.size(), shopCount));
|
||||
}
|
||||
|
||||
|
||||
query = new BasicDBObject();
|
||||
query.put('city', cityName);
|
||||
query.put('kind', '商铺');
|
||||
@@ -651,45 +621,41 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
if (shopRents) {
|
||||
detail.put('rentPrice', roundDouble(shopRents.size(), shopCount));
|
||||
}
|
||||
|
||||
|
||||
entity.put('shop_detail', detail);
|
||||
entity.put('existing_shop_count', shopCount);
|
||||
}else {
|
||||
entity.put('shop_detail', [:]);
|
||||
}
|
||||
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 商铺完成]"
|
||||
|
||||
|
||||
def cityStat = allCityStats.get(cityName);
|
||||
if (cityStat) {
|
||||
entity.put('avg_price', cityStat.avg_price);
|
||||
}
|
||||
|
||||
|
||||
DBObject query = new BasicDBObject();
|
||||
query.put('city', cityName);
|
||||
query.put('c_date', new BasicDBObject('$gte', lastWeekend).append('$lte', weekend));
|
||||
//entity.put('evaluation_count', (int) EVALUATION_CONN.count(query));由于估价记录数据量大,查询慢,先去掉这个,最后面在用多线程统计
|
||||
entity.put('price_total_count', (int) PRICE_JIUCUO_CONN.count(query));
|
||||
|
||||
|
||||
query.put('price_lev', '偏高');
|
||||
entity.put('price_high_count', (int) PRICE_JIUCUO_CONN.count(query));
|
||||
|
||||
|
||||
query.put('price_lev', '偏低');
|
||||
entity.put('price_low_count', (int) PRICE_JIUCUO_CONN.count(query));
|
||||
|
||||
|
||||
/*if (entity.get('evaluation_count')) {由于估价记录数据量大,查询慢,先去掉这个,最后面在用多线程统计
|
||||
entity.put('price_to_eva', roundDouble(entity.get('price_total_count'), entity.get('evaluation_count')));
|
||||
}*/
|
||||
|
||||
|
||||
if (entity.get('price_high_count')) {
|
||||
entity.put('low_to_high', roundDouble(entity.get('price_low_count'), entity.get('price_high_count')));
|
||||
}
|
||||
|
||||
|
||||
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 估价完成]"
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// query = new BasicDBObject();
|
||||
// query.put('city', cityName);
|
||||
// query.put('weekend', weekend);
|
||||
@@ -710,7 +676,7 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//
|
||||
// if (total > 0) {
|
||||
// BigDecimal bg = new BigDecimal(webZhunque * 1.0 / total);
|
||||
// entity.put('web_average', bg.setScale(4, BigDecimal.ROUND_HALF_UP).doubleValue());
|
||||
@@ -1284,7 +1250,7 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
fields.put(key, 1)
|
||||
}
|
||||
|
||||
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
|
||||
def cur = SEAWEED.find(query, fields).batchSize(2000)
|
||||
Iterator<DBObject> iter = cur.iterator()
|
||||
def countMap = [:]
|
||||
println cur.count()
|
||||
@@ -1335,7 +1301,7 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
fields.put(key, 1)
|
||||
}
|
||||
|
||||
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
|
||||
def cur = SEAWEED.find(query, fields).batchSize(2000)
|
||||
Iterator<DBObject> iter = cur.iterator()
|
||||
def countMap = [:]
|
||||
|
||||
@@ -1546,6 +1512,69 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
def findOfficeRent(cityName){
|
||||
def query = new BasicDBObject();
|
||||
query.put('city', cityName);
|
||||
// query.put('kind', '写字楼');
|
||||
// query.put('weekend', weekend);
|
||||
// def officeRents = RENT_STAT_CONN.distinct("name", query);
|
||||
// if (officeRents) {
|
||||
// detail.put('rentPrice', roundDouble(officeRents.size(), officeCount));
|
||||
// }
|
||||
query.put('cat', 'office');
|
||||
def date1 = new Date()
|
||||
date1.setDate(28)
|
||||
if(new Date().getTime()> date1.getTime()){
|
||||
query.put('date', Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")));
|
||||
}else{
|
||||
query.put('date', Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")));
|
||||
}
|
||||
def officeRents = OFFICE_RENT_STAT_CONN.distinct("name", query);
|
||||
return officeRents
|
||||
}
|
||||
|
||||
def findHalfYearOfficeRentCount(cityName){
|
||||
|
||||
def date1 = new Date()
|
||||
date1.setDate(28)
|
||||
def months = []
|
||||
if(new Date().getTime()> date1.getTime()){
|
||||
for(def i=1;i<=6;i++){
|
||||
months << Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1*i, "yyyyMM"))
|
||||
}
|
||||
}else{
|
||||
for(def i=2;i<=7;i++){
|
||||
months << Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1*i, "yyyyMM"))
|
||||
}
|
||||
}
|
||||
def map = [:]
|
||||
for(def month : months){
|
||||
def query = new BasicDBObject();
|
||||
query.put('city', cityName);
|
||||
query.put('cat', 'office');
|
||||
query.put('date', month);
|
||||
def officeRents = OFFICE_RENT_STAT_CONN.distinct("name", query);
|
||||
|
||||
officeRents.each{k->
|
||||
def count = map.get(k)
|
||||
if(!count){
|
||||
map.put(k, 1)
|
||||
}else{
|
||||
map.put(k, count + 1)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
def total = 0
|
||||
map.each{k,v->
|
||||
if(v==6){
|
||||
total ++
|
||||
}
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user