Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj

This commit is contained in:
2025-03-20 09:34:49 +08:00
@@ -1,8 +1,18 @@
package com.fangjia.script.dataAna package com.fangjia.script.dataAna
import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils import com.fangjia.datastore.driver.DSCursor
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.Jms
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.google.common.collect.Lists
import com.google.common.collect.Sets
import com.mongodb.*
import org.apache.commons.codec.digest.DigestUtils
import java.text.DecimalFormat import java.text.DecimalFormat
import java.util.concurrent.CountDownLatch import java.util.concurrent.CountDownLatch
@@ -10,30 +20,6 @@ import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors import java.util.concurrent.Executors
import java.util.concurrent.Semaphore import java.util.concurrent.Semaphore
import org.apache.commons.codec.digest.DigestUtils
import org.bson.types.ObjectId
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.Jms
import com.google.common.collect.Lists
import com.google.common.collect.Sets
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.mongodb.Mongo
@BatchOwner(Watcher.LiXue) @BatchOwner(Watcher.LiXue)
class CityInfoAnaBatch extends BaseBatch { class CityInfoAnaBatch extends BaseBatch {
@@ -53,8 +39,10 @@ class CityInfoAnaBatch extends BaseBatch {
final static DBCollection LOUDONG_CONN = MongoUtils.getCollectionByName("fangjia_base", "loudong", MongoUtils.MongoDB_Domain_mongo); final static DBCollection LOUDONG_CONN = MongoUtils.getCollectionByName("fangjia_base", "loudong", MongoUtils.MongoDB_Domain_mongo);
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000); final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000);
final static DBCollection FACILITY_CONN = MongoUtils.getCollectionByName("fangjia", "district_facilitys", MongoUtils.MongoDB_Domain_mongo) final static DBCollection FACILITY_CONN = MongoUtils.getCollectionByName("fangjia", "district_facilitys", MongoUtils.MongoDB_Domain_mongo)
final static DBCollection OFFICE_RENT_STAT_CONN = MongoUtils.getCollectionByName("fangjia_stat", "office_rent_stat", MongoUtils.MongoDB_Domain_mongo);
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616') static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616')
final static DSCollection SEAWEED = DbUtils.seaweed // final static DSCollection SEAWEED = DbUtils.seaweed
static final DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
final static def CITYS = []; final static def CITYS = [];
@@ -125,7 +113,7 @@ class CityInfoAnaBatch extends BaseBatch {
SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); } SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); }
allCitys.each {k,v -> allCitys.each {k,v ->
// if(["南京"].contains(k.toString())){ // if(["北京"].contains(k.toString())){
CITYS << k.toString(); CITYS << k.toString();
// } // }
} }
@@ -140,72 +128,47 @@ class CityInfoAnaBatch extends BaseBatch {
// 城市小区数 // 城市小区数
def allCityDisCount = [:]; def allCityDisCount = [:];
LinkedHashMap<String, Object> facet = new LinkedHashMap<String, Object>(2); BasicDBObject match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "district"));
facet.put("fields", ["city"]); DBObject groupFields = new BasicDBObject();
facet.put('limit', 500); groupFields.put("_id", new BasicDBObject("city", '$city'))
groupFields.put("count", new BasicDBObject('$sum', 1));
DBObject disQuery = new BasicDBObject(); DBObject group = new BasicDBObject('$group', groupFields);
disQuery.put('$facet', facet); // 定义聚合管道,按 city 字段分组,并计算每个城市的数量
disQuery.put("visible", 0); AggregationOutput output = SEAWEED.aggregate(Arrays.asList(match, group));
disQuery.put("cat", 'district'); for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
DSCursor sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0); allCityDisCount.put(obj.get("city"), result.get("count"))
List<FacetResult> disFacetResult = sellDSCursor.getFacetResults();
if (disFacetResult) {
def obj = disFacetResult[0].values;
obj.each {
allCityDisCount.put(it.name, it.count.intValue());
}
} }
// 城市新盘数 // 城市新盘数
def allCityNewDisCount = [:]; def allCityNewDisCount = [:];
disQuery = new BasicDBObject(); match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "new_district"));
disQuery.put('$facet', facet); output = SEAWEED.aggregate(Arrays.asList(match, group));
disQuery.put("visible", 0); for (DBObject result : output.results()) {
disQuery.put("cat", 'new_district'); def obj = (DBObject ) result.get("_id");
allCityNewDisCount.put(obj.get("city"), result.get("count"))
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
disFacetResult = sellDSCursor.getFacetResults();
if (disFacetResult) {
def obj = disFacetResult[0].values;
obj.each {
allCityNewDisCount.put(it.name, it.count.intValue());
}
} }
// 城市写字楼数 // 城市写字楼数
def allCityOfficeCount = [:]; def allCityOfficeCount = [:];
disQuery = new BasicDBObject(); match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "office"));
disQuery.put('$facet', facet);
disQuery.put("visible", 0);
disQuery.put("cat", 'office');
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0); output = SEAWEED.aggregate(Arrays.asList(match, group));
disFacetResult = sellDSCursor.getFacetResults(); for (DBObject result : output.results()) {
if (disFacetResult) { def obj = (DBObject ) result.get("_id");
def obj = disFacetResult[0].values; allCityOfficeCount.put(obj.get("city"), result.get("count"))
obj.each {
allCityOfficeCount.put(it.name, it.count.intValue());
}
} }
// 城市商铺数 // 城市商铺数
def allCityShopCount = [:]; def allCityShopCount = [:];
disQuery = new BasicDBObject(); match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "shop"));
disQuery.put('$facet', facet);
disQuery.put("visible", 0);
disQuery.put("cat", 'shop');
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0); output = SEAWEED.aggregate(Arrays.asList(match, group));
disFacetResult = sellDSCursor.getFacetResults(); for (DBObject result : output.results()) {
if (disFacetResult) { def obj = (DBObject ) result.get("_id");
def obj = disFacetResult[0].values; allCityShopCount.put(obj.get("city"), result.get("count"))
obj.each {
allCityShopCount.put(it.name, it.count.intValue());
}
} }
// 城市均价 // 城市均价
def allCityStats = [:]; def allCityStats = [:];
query = new BasicDBObject(); query = new BasicDBObject();
@@ -220,10 +183,10 @@ class CityInfoAnaBatch extends BaseBatch {
doProcess(allCitys.get(it), weekend, laseWeekend, allHouseStats, allCityDisCount, allCityNewDisCount, allCityOfficeCount, allCityShopCount, allCityStats); doProcess(allCitys.get(it), weekend, laseWeekend, allHouseStats, allCityDisCount, allCityNewDisCount, allCityOfficeCount, allCityShopCount, allCityStats);
} }
//统计有楼栋的小区数和有房号的小区数 // //统计有楼栋的小区数和有房号的小区数
// countDistrictsHasLoudong(allCitys, weekend) //// countDistrictsHasLoudong(allCitys, weekend)
//countDistrictsHasLoudongNew(allCitys, weekend) // //countDistrictsHasLoudongNew(allCitys, weekend)
//
final CountDownLatch latch = new CountDownLatch(CITYS.size()) final CountDownLatch latch = new CountDownLatch(CITYS.size())
final Semaphore semaphore = new Semaphore(10) final Semaphore semaphore = new Semaphore(10)
CITYS.each { CITYS.each {
@@ -239,13 +202,13 @@ class CityInfoAnaBatch extends BaseBatch {
} finally { } finally {
semaphore.release() semaphore.release()
latch.countDown() latch.countDown()
} }
}; };
}) })
} }
latch.await() latch.await()
println "${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())}\t楼栋覆盖率统计完成" println "${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())}\t楼栋覆盖率统计完成"
latch = new CountDownLatch(CITYS.size()) latch = new CountDownLatch(CITYS.size())
semaphore = new Semaphore(10) semaphore = new Semaphore(10)
CITYS.each { CITYS.each {
@@ -267,12 +230,12 @@ class CityInfoAnaBatch extends BaseBatch {
} }
latch.await() latch.await()
println "${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())}\t房号覆盖率统计完成" println "${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())}\t房号覆盖率统计完成"
//71个城市的覆盖率统计 //71个城市的覆盖率统计
sendCountData(weekend) sendCountData(weekend)
//最后统计之前因为太慢取消统计的本周估价次数 //最后统计之前因为太慢取消统计的本周估价次数
latch = new CountDownLatch(CITYS.size()) latch = new CountDownLatch(CITYS.size())
semaphore = new Semaphore(10) semaphore = new Semaphore(10)
@@ -292,14 +255,14 @@ class CityInfoAnaBatch extends BaseBatch {
query.put('city', cityName); query.put('city', cityName);
query.put('c_date', new BasicDBObject('$gte', laseWeekend).append('$lte', weekend)); query.put('c_date', new BasicDBObject('$gte', laseWeekend).append('$lte', weekend));
entity.put('evaluation_count', (int) EVALUATION_CONN.count(query)); entity.put('evaluation_count', (int) EVALUATION_CONN.count(query));
if (entity.get('evaluation_count')) { if (entity.get('evaluation_count')) {
entity.put('price_to_eva', DataUtil.roundDouble(entity.get('price_total_count'), entity.get('evaluation_count'))); entity.put('price_to_eva', DataUtil.roundDouble(entity.get('price_total_count'), entity.get('evaluation_count')));
} }
CITY_INFO_CONN.save(entity) CITY_INFO_CONN.save(entity)
println cityName+"本周估价次数统计完毕" println cityName+"本周估价次数统计完毕"
} }
} catch (Exception e) { } catch (Exception e) {
println e.getMessage() println e.getMessage()
} finally { } finally {
@@ -585,8 +548,8 @@ class CityInfoAnaBatch extends BaseBatch {
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 小区完成]" print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 小区完成]"
// if (allCityNewDisCount.get(cityName)) { // if (allCityNewDisCount.get(cityName)) {
// def newDisCount = allCityNewDisCount.get(cityName); // def newDisCount = allCityNewDisCount.get(cityName);
// entity.put('new_district_detail', newDistrictDetail(newDisCount, cityName)); // entity.put('new_district_detail', newDistrictDetail(newDisCount, cityName));
// } else { // } else {
// entity.put('new_district_detail', [:]); // entity.put('new_district_detail', [:]);
@@ -604,23 +567,30 @@ class CityInfoAnaBatch extends BaseBatch {
if (officePrices) { if (officePrices) {
detail.put('price', roundDouble(officePrices.size(), officeCount)); detail.put('price', roundDouble(officePrices.size(), officeCount));
} }
query = new BasicDBObject(); def officeRents = findOfficeRent(cityName)
query.put('city', cityName);
query.put('kind', '写字楼');
query.put('weekend', weekend);
def officeRents = RENT_STAT_CONN.distinct("name", query);
if (officeRents) { if (officeRents) {
detail.put('rentPrice', roundDouble(officeRents.size(), officeCount)); detail.put('rentPrice', roundDouble(officeRents.size(), officeCount));
} }
def halfYearofficeRentCount = findHalfYearOfficeRentCount(cityName)
if (halfYearofficeRentCount) {
detail.put('halfYearRentPrice', roundDouble(halfYearofficeRentCount, officeCount));
}
entity.put('office_detail', detail); entity.put('office_detail', detail);
entity.put('existing_office_count', officeCount); entity.put('existing_office_count', officeCount);
} else { } else {
entity.put('office_detail', [:]); entity.put('office_detail', [:]);
} }
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 写字楼完成]" // if(curInfo){
// CITY_INFO_CONN.update(new BasicDBObject("_id", curInfo._id), new BasicDBObject('$set', entity))
// print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 写字楼完成]"
// }
if (entity.get('official_population') && entity.get('district_avg_population')) { if (entity.get('official_population') && entity.get('district_avg_population')) {
entity.put('prediction_district_count', (int) (entity.get('official_population') / entity.get('district_avg_population'))); entity.put('prediction_district_count', (int) (entity.get('official_population') / entity.get('district_avg_population')));
@@ -628,12 +598,12 @@ class CityInfoAnaBatch extends BaseBatch {
entity.put('district_coverage', roundDouble(entity.get('existing_district_count'), entity.get('prediction_district_count'))); entity.put('district_coverage', roundDouble(entity.get('existing_district_count'), entity.get('prediction_district_count')));
} }
} }
if (allCityShopCount.get(cityName)) { if (allCityShopCount.get(cityName)) {
def shopCount = allCityShopCount.get(cityName); def shopCount = allCityShopCount.get(cityName);
def detail = shopDetail(shopCount, cityName); def detail = shopDetail(shopCount, cityName);
DBObject query = new BasicDBObject(); DBObject query = new BasicDBObject();
query.put('city', cityName); query.put('city', cityName);
//query.put('estate_type', '商铺'); //query.put('estate_type', '商铺');
@@ -642,7 +612,7 @@ class CityInfoAnaBatch extends BaseBatch {
if (shopPrices) { if (shopPrices) {
detail.put('price', roundDouble(shopPrices.size(), shopCount)); detail.put('price', roundDouble(shopPrices.size(), shopCount));
} }
query = new BasicDBObject(); query = new BasicDBObject();
query.put('city', cityName); query.put('city', cityName);
query.put('kind', '商铺'); query.put('kind', '商铺');
@@ -651,45 +621,41 @@ class CityInfoAnaBatch extends BaseBatch {
if (shopRents) { if (shopRents) {
detail.put('rentPrice', roundDouble(shopRents.size(), shopCount)); detail.put('rentPrice', roundDouble(shopRents.size(), shopCount));
} }
entity.put('shop_detail', detail); entity.put('shop_detail', detail);
entity.put('existing_shop_count', shopCount); entity.put('existing_shop_count', shopCount);
}else { }else {
entity.put('shop_detail', [:]); entity.put('shop_detail', [:]);
} }
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 商铺完成]" print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 商铺完成]"
def cityStat = allCityStats.get(cityName); def cityStat = allCityStats.get(cityName);
if (cityStat) { if (cityStat) {
entity.put('avg_price', cityStat.avg_price); entity.put('avg_price', cityStat.avg_price);
} }
DBObject query = new BasicDBObject(); DBObject query = new BasicDBObject();
query.put('city', cityName); query.put('city', cityName);
query.put('c_date', new BasicDBObject('$gte', lastWeekend).append('$lte', weekend)); query.put('c_date', new BasicDBObject('$gte', lastWeekend).append('$lte', weekend));
//entity.put('evaluation_count', (int) EVALUATION_CONN.count(query));由于估价记录数据量大,查询慢,先去掉这个,最后面在用多线程统计 //entity.put('evaluation_count', (int) EVALUATION_CONN.count(query));由于估价记录数据量大,查询慢,先去掉这个,最后面在用多线程统计
entity.put('price_total_count', (int) PRICE_JIUCUO_CONN.count(query)); entity.put('price_total_count', (int) PRICE_JIUCUO_CONN.count(query));
query.put('price_lev', '偏高'); query.put('price_lev', '偏高');
entity.put('price_high_count', (int) PRICE_JIUCUO_CONN.count(query)); entity.put('price_high_count', (int) PRICE_JIUCUO_CONN.count(query));
query.put('price_lev', '偏低'); query.put('price_lev', '偏低');
entity.put('price_low_count', (int) PRICE_JIUCUO_CONN.count(query)); entity.put('price_low_count', (int) PRICE_JIUCUO_CONN.count(query));
/*if (entity.get('evaluation_count')) {由于估价记录数据量大,查询慢,先去掉这个,最后面在用多线程统计 /*if (entity.get('evaluation_count')) {由于估价记录数据量大,查询慢,先去掉这个,最后面在用多线程统计
entity.put('price_to_eva', roundDouble(entity.get('price_total_count'), entity.get('evaluation_count'))); entity.put('price_to_eva', roundDouble(entity.get('price_total_count'), entity.get('evaluation_count')));
}*/ }*/
if (entity.get('price_high_count')) { if (entity.get('price_high_count')) {
entity.put('low_to_high', roundDouble(entity.get('price_low_count'), entity.get('price_high_count'))); entity.put('low_to_high', roundDouble(entity.get('price_low_count'), entity.get('price_high_count')));
} }
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 估价完成]" print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 估价完成]"
// query = new BasicDBObject(); // query = new BasicDBObject();
// query.put('city', cityName); // query.put('city', cityName);
// query.put('weekend', weekend); // query.put('weekend', weekend);
@@ -710,7 +676,7 @@ class CityInfoAnaBatch extends BaseBatch {
// } // }
// } // }
// } // }
// //
// if (total > 0) { // if (total > 0) {
// BigDecimal bg = new BigDecimal(webZhunque * 1.0 / total); // BigDecimal bg = new BigDecimal(webZhunque * 1.0 / total);
// entity.put('web_average', bg.setScale(4, BigDecimal.ROUND_HALF_UP).doubleValue()); // entity.put('web_average', bg.setScale(4, BigDecimal.ROUND_HALF_UP).doubleValue());
@@ -1284,7 +1250,7 @@ class CityInfoAnaBatch extends BaseBatch {
fields.put(key, 1) fields.put(key, 1)
} }
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000) def cur = SEAWEED.find(query, fields).batchSize(2000)
Iterator<DBObject> iter = cur.iterator() Iterator<DBObject> iter = cur.iterator()
def countMap = [:] def countMap = [:]
println cur.count() println cur.count()
@@ -1335,7 +1301,7 @@ class CityInfoAnaBatch extends BaseBatch {
fields.put(key, 1) fields.put(key, 1)
} }
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000) def cur = SEAWEED.find(query, fields).batchSize(2000)
Iterator<DBObject> iter = cur.iterator() Iterator<DBObject> iter = cur.iterator()
def countMap = [:] def countMap = [:]
@@ -1546,6 +1512,69 @@ class CityInfoAnaBatch extends BaseBatch {
} }
return true; return true;
} }
def findOfficeRent(cityName){
def query = new BasicDBObject();
query.put('city', cityName);
// query.put('kind', '写字楼');
// query.put('weekend', weekend);
// def officeRents = RENT_STAT_CONN.distinct("name", query);
// if (officeRents) {
// detail.put('rentPrice', roundDouble(officeRents.size(), officeCount));
// }
query.put('cat', 'office');
def date1 = new Date()
date1.setDate(28)
if(new Date().getTime()> date1.getTime()){
query.put('date', Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")));
}else{
query.put('date', Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")));
}
def officeRents = OFFICE_RENT_STAT_CONN.distinct("name", query);
return officeRents
}
def findHalfYearOfficeRentCount(cityName){
def date1 = new Date()
date1.setDate(28)
def months = []
if(new Date().getTime()> date1.getTime()){
for(def i=1;i<=6;i++){
months << Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1*i, "yyyyMM"))
}
}else{
for(def i=2;i<=7;i++){
months << Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1*i, "yyyyMM"))
}
}
def map = [:]
for(def month : months){
def query = new BasicDBObject();
query.put('city', cityName);
query.put('cat', 'office');
query.put('date', month);
def officeRents = OFFICE_RENT_STAT_CONN.distinct("name", query);
officeRents.each{k->
def count = map.get(k)
if(!count){
map.put(k, 1)
}else{
map.put(k, count + 1)
}
}
}
def total = 0
map.each{k,v->
if(v==6){
total ++
}
}
return total
}
} }