Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj
This commit is contained in:
+126
-97
@@ -1,8 +1,18 @@
|
|||||||
package com.fangjia.script.dataAna
|
package com.fangjia.script.dataAna
|
||||||
|
|
||||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
import com.fangjia.core.util.DateUtil
|
||||||
import com.fangjia.script.commData.CommonDatautils
|
import com.fangjia.datastore.driver.DSCursor
|
||||||
|
import com.fangjia.script.BaseBatch
|
||||||
|
import com.fangjia.script.BatchOwner
|
||||||
|
import com.fangjia.script.Watcher
|
||||||
|
import com.fangjia.script.util.DateUtils
|
||||||
|
import com.fangjia.script.util.DbUtils
|
||||||
|
import com.fangjia.script.util.Jms
|
||||||
import com.fangjia.script.util.MongoUtils
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.google.common.collect.Lists
|
||||||
|
import com.google.common.collect.Sets
|
||||||
|
import com.mongodb.*
|
||||||
|
import org.apache.commons.codec.digest.DigestUtils
|
||||||
|
|
||||||
import java.text.DecimalFormat
|
import java.text.DecimalFormat
|
||||||
import java.util.concurrent.CountDownLatch
|
import java.util.concurrent.CountDownLatch
|
||||||
@@ -10,30 +20,6 @@ import java.util.concurrent.ExecutorService
|
|||||||
import java.util.concurrent.Executors
|
import java.util.concurrent.Executors
|
||||||
import java.util.concurrent.Semaphore
|
import java.util.concurrent.Semaphore
|
||||||
|
|
||||||
import org.apache.commons.codec.digest.DigestUtils
|
|
||||||
import org.bson.types.ObjectId
|
|
||||||
|
|
||||||
import com.fangjia.core.util.DataUtil
|
|
||||||
import com.fangjia.core.util.DateUtil
|
|
||||||
import com.fangjia.datastore.driver.DSCollection
|
|
||||||
import com.fangjia.datastore.driver.DSCursor
|
|
||||||
import com.fangjia.datastore.driver.FacetResult
|
|
||||||
import com.fangjia.script.BaseBatch
|
|
||||||
import com.fangjia.script.BatchOwner
|
|
||||||
import com.fangjia.script.Watcher
|
|
||||||
import com.fangjia.script.util.DbUtils
|
|
||||||
import com.fangjia.script.util.ExcelUtils
|
|
||||||
import com.fangjia.script.util.Jms
|
|
||||||
import com.google.common.collect.Lists
|
|
||||||
import com.google.common.collect.Sets
|
|
||||||
import com.mongodb.BasicDBList
|
|
||||||
import com.mongodb.BasicDBObject
|
|
||||||
import com.mongodb.Bytes
|
|
||||||
import com.mongodb.DBCollection
|
|
||||||
import com.mongodb.DBCursor
|
|
||||||
import com.mongodb.DBObject
|
|
||||||
import com.mongodb.Mongo
|
|
||||||
|
|
||||||
@BatchOwner(Watcher.LiXue)
|
@BatchOwner(Watcher.LiXue)
|
||||||
class CityInfoAnaBatch extends BaseBatch {
|
class CityInfoAnaBatch extends BaseBatch {
|
||||||
|
|
||||||
@@ -53,8 +39,10 @@ class CityInfoAnaBatch extends BaseBatch {
|
|||||||
final static DBCollection LOUDONG_CONN = MongoUtils.getCollectionByName("fangjia_base", "loudong", MongoUtils.MongoDB_Domain_mongo);
|
final static DBCollection LOUDONG_CONN = MongoUtils.getCollectionByName("fangjia_base", "loudong", MongoUtils.MongoDB_Domain_mongo);
|
||||||
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000);
|
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000);
|
||||||
final static DBCollection FACILITY_CONN = MongoUtils.getCollectionByName("fangjia", "district_facilitys", MongoUtils.MongoDB_Domain_mongo)
|
final static DBCollection FACILITY_CONN = MongoUtils.getCollectionByName("fangjia", "district_facilitys", MongoUtils.MongoDB_Domain_mongo)
|
||||||
|
final static DBCollection OFFICE_RENT_STAT_CONN = MongoUtils.getCollectionByName("fangjia_stat", "office_rent_stat", MongoUtils.MongoDB_Domain_mongo);
|
||||||
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616')
|
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616')
|
||||||
final static DSCollection SEAWEED = DbUtils.seaweed
|
// final static DSCollection SEAWEED = DbUtils.seaweed
|
||||||
|
static final DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
|
||||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||||
|
|
||||||
final static def CITYS = [];
|
final static def CITYS = [];
|
||||||
@@ -125,7 +113,7 @@ class CityInfoAnaBatch extends BaseBatch {
|
|||||||
SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); }
|
SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); }
|
||||||
|
|
||||||
allCitys.each {k,v ->
|
allCitys.each {k,v ->
|
||||||
// if(["南京"].contains(k.toString())){
|
// if(["北京"].contains(k.toString())){
|
||||||
CITYS << k.toString();
|
CITYS << k.toString();
|
||||||
// }
|
// }
|
||||||
}
|
}
|
||||||
@@ -140,72 +128,47 @@ class CityInfoAnaBatch extends BaseBatch {
|
|||||||
|
|
||||||
// 城市小区数
|
// 城市小区数
|
||||||
def allCityDisCount = [:];
|
def allCityDisCount = [:];
|
||||||
LinkedHashMap<String, Object> facet = new LinkedHashMap<String, Object>(2);
|
BasicDBObject match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "district"));
|
||||||
facet.put("fields", ["city"]);
|
DBObject groupFields = new BasicDBObject();
|
||||||
facet.put('limit', 500);
|
groupFields.put("_id", new BasicDBObject("city", '$city'))
|
||||||
|
groupFields.put("count", new BasicDBObject('$sum', 1));
|
||||||
DBObject disQuery = new BasicDBObject();
|
DBObject group = new BasicDBObject('$group', groupFields);
|
||||||
disQuery.put('$facet', facet);
|
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
|
||||||
disQuery.put("visible", 0);
|
AggregationOutput output = SEAWEED.aggregate(Arrays.asList(match, group));
|
||||||
disQuery.put("cat", 'district');
|
for (DBObject result : output.results()) {
|
||||||
|
def obj = (DBObject ) result.get("_id");
|
||||||
DSCursor sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
|
allCityDisCount.put(obj.get("city"), result.get("count"))
|
||||||
List<FacetResult> disFacetResult = sellDSCursor.getFacetResults();
|
|
||||||
if (disFacetResult) {
|
|
||||||
def obj = disFacetResult[0].values;
|
|
||||||
obj.each {
|
|
||||||
allCityDisCount.put(it.name, it.count.intValue());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// 城市新盘数
|
// 城市新盘数
|
||||||
def allCityNewDisCount = [:];
|
def allCityNewDisCount = [:];
|
||||||
disQuery = new BasicDBObject();
|
match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "new_district"));
|
||||||
disQuery.put('$facet', facet);
|
output = SEAWEED.aggregate(Arrays.asList(match, group));
|
||||||
disQuery.put("visible", 0);
|
for (DBObject result : output.results()) {
|
||||||
disQuery.put("cat", 'new_district');
|
def obj = (DBObject ) result.get("_id");
|
||||||
|
allCityNewDisCount.put(obj.get("city"), result.get("count"))
|
||||||
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
|
|
||||||
disFacetResult = sellDSCursor.getFacetResults();
|
|
||||||
if (disFacetResult) {
|
|
||||||
def obj = disFacetResult[0].values;
|
|
||||||
obj.each {
|
|
||||||
allCityNewDisCount.put(it.name, it.count.intValue());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// 城市写字楼数
|
// 城市写字楼数
|
||||||
def allCityOfficeCount = [:];
|
def allCityOfficeCount = [:];
|
||||||
disQuery = new BasicDBObject();
|
match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "office"));
|
||||||
disQuery.put('$facet', facet);
|
|
||||||
disQuery.put("visible", 0);
|
|
||||||
disQuery.put("cat", 'office');
|
|
||||||
|
|
||||||
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
|
output = SEAWEED.aggregate(Arrays.asList(match, group));
|
||||||
disFacetResult = sellDSCursor.getFacetResults();
|
for (DBObject result : output.results()) {
|
||||||
if (disFacetResult) {
|
def obj = (DBObject ) result.get("_id");
|
||||||
def obj = disFacetResult[0].values;
|
allCityOfficeCount.put(obj.get("city"), result.get("count"))
|
||||||
obj.each {
|
|
||||||
allCityOfficeCount.put(it.name, it.count.intValue());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// 城市商铺数
|
// 城市商铺数
|
||||||
def allCityShopCount = [:];
|
def allCityShopCount = [:];
|
||||||
disQuery = new BasicDBObject();
|
match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "shop"));
|
||||||
disQuery.put('$facet', facet);
|
|
||||||
disQuery.put("visible", 0);
|
|
||||||
disQuery.put("cat", 'shop');
|
|
||||||
|
|
||||||
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
|
output = SEAWEED.aggregate(Arrays.asList(match, group));
|
||||||
disFacetResult = sellDSCursor.getFacetResults();
|
for (DBObject result : output.results()) {
|
||||||
if (disFacetResult) {
|
def obj = (DBObject ) result.get("_id");
|
||||||
def obj = disFacetResult[0].values;
|
allCityShopCount.put(obj.get("city"), result.get("count"))
|
||||||
obj.each {
|
|
||||||
allCityShopCount.put(it.name, it.count.intValue());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// 城市均价
|
// 城市均价
|
||||||
def allCityStats = [:];
|
def allCityStats = [:];
|
||||||
query = new BasicDBObject();
|
query = new BasicDBObject();
|
||||||
@@ -220,10 +183,10 @@ class CityInfoAnaBatch extends BaseBatch {
|
|||||||
doProcess(allCitys.get(it), weekend, laseWeekend, allHouseStats, allCityDisCount, allCityNewDisCount, allCityOfficeCount, allCityShopCount, allCityStats);
|
doProcess(allCitys.get(it), weekend, laseWeekend, allHouseStats, allCityDisCount, allCityNewDisCount, allCityOfficeCount, allCityShopCount, allCityStats);
|
||||||
}
|
}
|
||||||
|
|
||||||
//统计有楼栋的小区数和有房号的小区数
|
// //统计有楼栋的小区数和有房号的小区数
|
||||||
// countDistrictsHasLoudong(allCitys, weekend)
|
//// countDistrictsHasLoudong(allCitys, weekend)
|
||||||
//countDistrictsHasLoudongNew(allCitys, weekend)
|
// //countDistrictsHasLoudongNew(allCitys, weekend)
|
||||||
|
//
|
||||||
final CountDownLatch latch = new CountDownLatch(CITYS.size())
|
final CountDownLatch latch = new CountDownLatch(CITYS.size())
|
||||||
final Semaphore semaphore = new Semaphore(10)
|
final Semaphore semaphore = new Semaphore(10)
|
||||||
CITYS.each {
|
CITYS.each {
|
||||||
@@ -585,8 +548,8 @@ class CityInfoAnaBatch extends BaseBatch {
|
|||||||
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 小区完成]"
|
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 小区完成]"
|
||||||
|
|
||||||
|
|
||||||
// if (allCityNewDisCount.get(cityName)) {
|
// if (allCityNewDisCount.get(cityName)) {
|
||||||
// def newDisCount = allCityNewDisCount.get(cityName);
|
// def newDisCount = allCityNewDisCount.get(cityName);
|
||||||
// entity.put('new_district_detail', newDistrictDetail(newDisCount, cityName));
|
// entity.put('new_district_detail', newDistrictDetail(newDisCount, cityName));
|
||||||
// } else {
|
// } else {
|
||||||
// entity.put('new_district_detail', [:]);
|
// entity.put('new_district_detail', [:]);
|
||||||
@@ -605,22 +568,29 @@ class CityInfoAnaBatch extends BaseBatch {
|
|||||||
detail.put('price', roundDouble(officePrices.size(), officeCount));
|
detail.put('price', roundDouble(officePrices.size(), officeCount));
|
||||||
}
|
}
|
||||||
|
|
||||||
query = new BasicDBObject();
|
def officeRents = findOfficeRent(cityName)
|
||||||
query.put('city', cityName);
|
|
||||||
query.put('kind', '写字楼');
|
|
||||||
query.put('weekend', weekend);
|
|
||||||
def officeRents = RENT_STAT_CONN.distinct("name", query);
|
|
||||||
if (officeRents) {
|
if (officeRents) {
|
||||||
detail.put('rentPrice', roundDouble(officeRents.size(), officeCount));
|
detail.put('rentPrice', roundDouble(officeRents.size(), officeCount));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def halfYearofficeRentCount = findHalfYearOfficeRentCount(cityName)
|
||||||
|
if (halfYearofficeRentCount) {
|
||||||
|
detail.put('halfYearRentPrice', roundDouble(halfYearofficeRentCount, officeCount));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
entity.put('office_detail', detail);
|
entity.put('office_detail', detail);
|
||||||
entity.put('existing_office_count', officeCount);
|
entity.put('existing_office_count', officeCount);
|
||||||
} else {
|
} else {
|
||||||
entity.put('office_detail', [:]);
|
entity.put('office_detail', [:]);
|
||||||
}
|
}
|
||||||
|
|
||||||
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 写字楼完成]"
|
// if(curInfo){
|
||||||
|
// CITY_INFO_CONN.update(new BasicDBObject("_id", curInfo._id), new BasicDBObject('$set', entity))
|
||||||
|
// print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 写字楼完成]"
|
||||||
|
// }
|
||||||
|
|
||||||
|
|
||||||
if (entity.get('official_population') && entity.get('district_avg_population')) {
|
if (entity.get('official_population') && entity.get('district_avg_population')) {
|
||||||
entity.put('prediction_district_count', (int) (entity.get('official_population') / entity.get('district_avg_population')));
|
entity.put('prediction_district_count', (int) (entity.get('official_population') / entity.get('district_avg_population')));
|
||||||
|
|
||||||
@@ -686,10 +656,6 @@ class CityInfoAnaBatch extends BaseBatch {
|
|||||||
|
|
||||||
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 估价完成]"
|
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 估价完成]"
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// query = new BasicDBObject();
|
// query = new BasicDBObject();
|
||||||
// query.put('city', cityName);
|
// query.put('city', cityName);
|
||||||
// query.put('weekend', weekend);
|
// query.put('weekend', weekend);
|
||||||
@@ -1284,7 +1250,7 @@ class CityInfoAnaBatch extends BaseBatch {
|
|||||||
fields.put(key, 1)
|
fields.put(key, 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
|
def cur = SEAWEED.find(query, fields).batchSize(2000)
|
||||||
Iterator<DBObject> iter = cur.iterator()
|
Iterator<DBObject> iter = cur.iterator()
|
||||||
def countMap = [:]
|
def countMap = [:]
|
||||||
println cur.count()
|
println cur.count()
|
||||||
@@ -1335,7 +1301,7 @@ class CityInfoAnaBatch extends BaseBatch {
|
|||||||
fields.put(key, 1)
|
fields.put(key, 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
|
def cur = SEAWEED.find(query, fields).batchSize(2000)
|
||||||
Iterator<DBObject> iter = cur.iterator()
|
Iterator<DBObject> iter = cur.iterator()
|
||||||
def countMap = [:]
|
def countMap = [:]
|
||||||
|
|
||||||
@@ -1547,5 +1513,68 @@ class CityInfoAnaBatch extends BaseBatch {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def findOfficeRent(cityName){
|
||||||
|
def query = new BasicDBObject();
|
||||||
|
query.put('city', cityName);
|
||||||
|
// query.put('kind', '写字楼');
|
||||||
|
// query.put('weekend', weekend);
|
||||||
|
// def officeRents = RENT_STAT_CONN.distinct("name", query);
|
||||||
|
// if (officeRents) {
|
||||||
|
// detail.put('rentPrice', roundDouble(officeRents.size(), officeCount));
|
||||||
|
// }
|
||||||
|
query.put('cat', 'office');
|
||||||
|
def date1 = new Date()
|
||||||
|
date1.setDate(28)
|
||||||
|
if(new Date().getTime()> date1.getTime()){
|
||||||
|
query.put('date', Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")));
|
||||||
|
}else{
|
||||||
|
query.put('date', Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")));
|
||||||
|
}
|
||||||
|
def officeRents = OFFICE_RENT_STAT_CONN.distinct("name", query);
|
||||||
|
return officeRents
|
||||||
|
}
|
||||||
|
|
||||||
|
def findHalfYearOfficeRentCount(cityName){
|
||||||
|
|
||||||
|
def date1 = new Date()
|
||||||
|
date1.setDate(28)
|
||||||
|
def months = []
|
||||||
|
if(new Date().getTime()> date1.getTime()){
|
||||||
|
for(def i=1;i<=6;i++){
|
||||||
|
months << Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1*i, "yyyyMM"))
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
for(def i=2;i<=7;i++){
|
||||||
|
months << Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1*i, "yyyyMM"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def map = [:]
|
||||||
|
for(def month : months){
|
||||||
|
def query = new BasicDBObject();
|
||||||
|
query.put('city', cityName);
|
||||||
|
query.put('cat', 'office');
|
||||||
|
query.put('date', month);
|
||||||
|
def officeRents = OFFICE_RENT_STAT_CONN.distinct("name", query);
|
||||||
|
|
||||||
|
officeRents.each{k->
|
||||||
|
def count = map.get(k)
|
||||||
|
if(!count){
|
||||||
|
map.put(k, 1)
|
||||||
|
}else{
|
||||||
|
map.put(k, count + 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def total = 0
|
||||||
|
map.each{k,v->
|
||||||
|
if(v==6){
|
||||||
|
total ++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return total
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user