Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj

This commit is contained in:
2025-03-20 09:34:49 +08:00
@@ -1,8 +1,18 @@
package com.fangjia.script.dataAna package com.fangjia.script.dataAna
import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils import com.fangjia.datastore.driver.DSCursor
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.Jms
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.google.common.collect.Lists
import com.google.common.collect.Sets
import com.mongodb.*
import org.apache.commons.codec.digest.DigestUtils
import java.text.DecimalFormat import java.text.DecimalFormat
import java.util.concurrent.CountDownLatch import java.util.concurrent.CountDownLatch
@@ -10,30 +20,6 @@ import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors import java.util.concurrent.Executors
import java.util.concurrent.Semaphore import java.util.concurrent.Semaphore
import org.apache.commons.codec.digest.DigestUtils
import org.bson.types.ObjectId
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.Jms
import com.google.common.collect.Lists
import com.google.common.collect.Sets
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.mongodb.Mongo
@BatchOwner(Watcher.LiXue) @BatchOwner(Watcher.LiXue)
class CityInfoAnaBatch extends BaseBatch { class CityInfoAnaBatch extends BaseBatch {
@@ -53,8 +39,10 @@ class CityInfoAnaBatch extends BaseBatch {
final static DBCollection LOUDONG_CONN = MongoUtils.getCollectionByName("fangjia_base", "loudong", MongoUtils.MongoDB_Domain_mongo); final static DBCollection LOUDONG_CONN = MongoUtils.getCollectionByName("fangjia_base", "loudong", MongoUtils.MongoDB_Domain_mongo);
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000); final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000);
final static DBCollection FACILITY_CONN = MongoUtils.getCollectionByName("fangjia", "district_facilitys", MongoUtils.MongoDB_Domain_mongo) final static DBCollection FACILITY_CONN = MongoUtils.getCollectionByName("fangjia", "district_facilitys", MongoUtils.MongoDB_Domain_mongo)
final static DBCollection OFFICE_RENT_STAT_CONN = MongoUtils.getCollectionByName("fangjia_stat", "office_rent_stat", MongoUtils.MongoDB_Domain_mongo);
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616') static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616')
final static DSCollection SEAWEED = DbUtils.seaweed // final static DSCollection SEAWEED = DbUtils.seaweed
static final DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
final static def CITYS = []; final static def CITYS = [];
@@ -125,7 +113,7 @@ class CityInfoAnaBatch extends BaseBatch {
SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); } SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); }
allCitys.each {k,v -> allCitys.each {k,v ->
// if(["南京"].contains(k.toString())){ // if(["北京"].contains(k.toString())){
CITYS << k.toString(); CITYS << k.toString();
// } // }
} }
@@ -140,72 +128,47 @@ class CityInfoAnaBatch extends BaseBatch {
// 城市小区数 // 城市小区数
def allCityDisCount = [:]; def allCityDisCount = [:];
LinkedHashMap<String, Object> facet = new LinkedHashMap<String, Object>(2); BasicDBObject match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "district"));
facet.put("fields", ["city"]); DBObject groupFields = new BasicDBObject();
facet.put('limit', 500); groupFields.put("_id", new BasicDBObject("city", '$city'))
groupFields.put("count", new BasicDBObject('$sum', 1));
DBObject disQuery = new BasicDBObject(); DBObject group = new BasicDBObject('$group', groupFields);
disQuery.put('$facet', facet); // 定义聚合管道,按 city 字段分组,并计算每个城市的数量
disQuery.put("visible", 0); AggregationOutput output = SEAWEED.aggregate(Arrays.asList(match, group));
disQuery.put("cat", 'district'); for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
DSCursor sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0); allCityDisCount.put(obj.get("city"), result.get("count"))
List<FacetResult> disFacetResult = sellDSCursor.getFacetResults();
if (disFacetResult) {
def obj = disFacetResult[0].values;
obj.each {
allCityDisCount.put(it.name, it.count.intValue());
}
} }
// 城市新盘数 // 城市新盘数
def allCityNewDisCount = [:]; def allCityNewDisCount = [:];
disQuery = new BasicDBObject(); match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "new_district"));
disQuery.put('$facet', facet); output = SEAWEED.aggregate(Arrays.asList(match, group));
disQuery.put("visible", 0); for (DBObject result : output.results()) {
disQuery.put("cat", 'new_district'); def obj = (DBObject ) result.get("_id");
allCityNewDisCount.put(obj.get("city"), result.get("count"))
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
disFacetResult = sellDSCursor.getFacetResults();
if (disFacetResult) {
def obj = disFacetResult[0].values;
obj.each {
allCityNewDisCount.put(it.name, it.count.intValue());
}
} }
// 城市写字楼数 // 城市写字楼数
def allCityOfficeCount = [:]; def allCityOfficeCount = [:];
disQuery = new BasicDBObject(); match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "office"));
disQuery.put('$facet', facet);
disQuery.put("visible", 0);
disQuery.put("cat", 'office');
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0); output = SEAWEED.aggregate(Arrays.asList(match, group));
disFacetResult = sellDSCursor.getFacetResults(); for (DBObject result : output.results()) {
if (disFacetResult) { def obj = (DBObject ) result.get("_id");
def obj = disFacetResult[0].values; allCityOfficeCount.put(obj.get("city"), result.get("count"))
obj.each {
allCityOfficeCount.put(it.name, it.count.intValue());
}
} }
// 城市商铺数 // 城市商铺数
def allCityShopCount = [:]; def allCityShopCount = [:];
disQuery = new BasicDBObject(); match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "shop"));
disQuery.put('$facet', facet);
disQuery.put("visible", 0);
disQuery.put("cat", 'shop');
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0); output = SEAWEED.aggregate(Arrays.asList(match, group));
disFacetResult = sellDSCursor.getFacetResults(); for (DBObject result : output.results()) {
if (disFacetResult) { def obj = (DBObject ) result.get("_id");
def obj = disFacetResult[0].values; allCityShopCount.put(obj.get("city"), result.get("count"))
obj.each {
allCityShopCount.put(it.name, it.count.intValue());
}
} }
// 城市均价 // 城市均价
def allCityStats = [:]; def allCityStats = [:];
query = new BasicDBObject(); query = new BasicDBObject();
@@ -220,10 +183,10 @@ class CityInfoAnaBatch extends BaseBatch {
doProcess(allCitys.get(it), weekend, laseWeekend, allHouseStats, allCityDisCount, allCityNewDisCount, allCityOfficeCount, allCityShopCount, allCityStats); doProcess(allCitys.get(it), weekend, laseWeekend, allHouseStats, allCityDisCount, allCityNewDisCount, allCityOfficeCount, allCityShopCount, allCityStats);
} }
//统计有楼栋的小区数和有房号的小区数 // //统计有楼栋的小区数和有房号的小区数
// countDistrictsHasLoudong(allCitys, weekend) //// countDistrictsHasLoudong(allCitys, weekend)
//countDistrictsHasLoudongNew(allCitys, weekend) // //countDistrictsHasLoudongNew(allCitys, weekend)
//
final CountDownLatch latch = new CountDownLatch(CITYS.size()) final CountDownLatch latch = new CountDownLatch(CITYS.size())
final Semaphore semaphore = new Semaphore(10) final Semaphore semaphore = new Semaphore(10)
CITYS.each { CITYS.each {
@@ -585,8 +548,8 @@ class CityInfoAnaBatch extends BaseBatch {
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 小区完成]" print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 小区完成]"
// if (allCityNewDisCount.get(cityName)) { // if (allCityNewDisCount.get(cityName)) {
// def newDisCount = allCityNewDisCount.get(cityName); // def newDisCount = allCityNewDisCount.get(cityName);
// entity.put('new_district_detail', newDistrictDetail(newDisCount, cityName)); // entity.put('new_district_detail', newDistrictDetail(newDisCount, cityName));
// } else { // } else {
// entity.put('new_district_detail', [:]); // entity.put('new_district_detail', [:]);
@@ -605,22 +568,29 @@ class CityInfoAnaBatch extends BaseBatch {
detail.put('price', roundDouble(officePrices.size(), officeCount)); detail.put('price', roundDouble(officePrices.size(), officeCount));
} }
query = new BasicDBObject(); def officeRents = findOfficeRent(cityName)
query.put('city', cityName);
query.put('kind', '写字楼');
query.put('weekend', weekend);
def officeRents = RENT_STAT_CONN.distinct("name", query);
if (officeRents) { if (officeRents) {
detail.put('rentPrice', roundDouble(officeRents.size(), officeCount)); detail.put('rentPrice', roundDouble(officeRents.size(), officeCount));
} }
def halfYearofficeRentCount = findHalfYearOfficeRentCount(cityName)
if (halfYearofficeRentCount) {
detail.put('halfYearRentPrice', roundDouble(halfYearofficeRentCount, officeCount));
}
entity.put('office_detail', detail); entity.put('office_detail', detail);
entity.put('existing_office_count', officeCount); entity.put('existing_office_count', officeCount);
} else { } else {
entity.put('office_detail', [:]); entity.put('office_detail', [:]);
} }
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 写字楼完成]" // if(curInfo){
// CITY_INFO_CONN.update(new BasicDBObject("_id", curInfo._id), new BasicDBObject('$set', entity))
// print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 写字楼完成]"
// }
if (entity.get('official_population') && entity.get('district_avg_population')) { if (entity.get('official_population') && entity.get('district_avg_population')) {
entity.put('prediction_district_count', (int) (entity.get('official_population') / entity.get('district_avg_population'))); entity.put('prediction_district_count', (int) (entity.get('official_population') / entity.get('district_avg_population')));
@@ -686,10 +656,6 @@ class CityInfoAnaBatch extends BaseBatch {
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 估价完成]" print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 估价完成]"
// query = new BasicDBObject(); // query = new BasicDBObject();
// query.put('city', cityName); // query.put('city', cityName);
// query.put('weekend', weekend); // query.put('weekend', weekend);
@@ -1284,7 +1250,7 @@ class CityInfoAnaBatch extends BaseBatch {
fields.put(key, 1) fields.put(key, 1)
} }
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000) def cur = SEAWEED.find(query, fields).batchSize(2000)
Iterator<DBObject> iter = cur.iterator() Iterator<DBObject> iter = cur.iterator()
def countMap = [:] def countMap = [:]
println cur.count() println cur.count()
@@ -1335,7 +1301,7 @@ class CityInfoAnaBatch extends BaseBatch {
fields.put(key, 1) fields.put(key, 1)
} }
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000) def cur = SEAWEED.find(query, fields).batchSize(2000)
Iterator<DBObject> iter = cur.iterator() Iterator<DBObject> iter = cur.iterator()
def countMap = [:] def countMap = [:]
@@ -1547,5 +1513,68 @@ class CityInfoAnaBatch extends BaseBatch {
return true; return true;
} }
def findOfficeRent(cityName){
def query = new BasicDBObject();
query.put('city', cityName);
// query.put('kind', '写字楼');
// query.put('weekend', weekend);
// def officeRents = RENT_STAT_CONN.distinct("name", query);
// if (officeRents) {
// detail.put('rentPrice', roundDouble(officeRents.size(), officeCount));
// }
query.put('cat', 'office');
def date1 = new Date()
date1.setDate(28)
if(new Date().getTime()> date1.getTime()){
query.put('date', Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")));
}else{
query.put('date', Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")));
}
def officeRents = OFFICE_RENT_STAT_CONN.distinct("name", query);
return officeRents
}
def findHalfYearOfficeRentCount(cityName){
def date1 = new Date()
date1.setDate(28)
def months = []
if(new Date().getTime()> date1.getTime()){
for(def i=1;i<=6;i++){
months << Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1*i, "yyyyMM"))
}
}else{
for(def i=2;i<=7;i++){
months << Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1*i, "yyyyMM"))
}
}
def map = [:]
for(def month : months){
def query = new BasicDBObject();
query.put('city', cityName);
query.put('cat', 'office');
query.put('date', month);
def officeRents = OFFICE_RENT_STAT_CONN.distinct("name", query);
officeRents.each{k->
def count = map.get(k)
if(!count){
map.put(k, 1)
}else{
map.put(k, count + 1)
}
}
}
def total = 0
map.each{k,v->
if(v==6){
total ++
}
}
return total
}
} }