From e1ecabc222ca1d1613646354c83b28d85a045b28 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 19 Mar 2025 17:32:18 +0800 Subject: [PATCH] =?UTF-8?q?=E5=9F=8E=E5=B8=82=E7=BB=9F=E8=AE=A1=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E5=86=99=E5=AD=97=E6=A5=BC=E5=87=BA=E7=A7=9F=E4=BB=B7?= =?UTF-8?q?=E6=A0=BC=E7=BB=9F=E8=AE=A1=E5=8F=8A=E5=8A=A0=E8=BD=BD=E5=B0=8F?= =?UTF-8?q?=E5=8C=BA=E6=8D=A2=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/dataAna/CityInfoAnaBatch.groovy | 271 ++++++++++-------- 1 file changed, 150 insertions(+), 121 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy index 8511662a1..004acafbe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy @@ -1,8 +1,18 @@ package com.fangjia.script.dataAna -import com.fangjia.script.ccbevaluation.CcbCommonUtils -import com.fangjia.script.commData.CommonDatautils +import com.fangjia.core.util.DateUtil +import com.fangjia.datastore.driver.DSCursor +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.Jms import com.fangjia.script.util.MongoUtils +import com.google.common.collect.Lists +import com.google.common.collect.Sets +import com.mongodb.* +import org.apache.commons.codec.digest.DigestUtils import java.text.DecimalFormat import java.util.concurrent.CountDownLatch @@ -10,30 +20,6 @@ import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.Semaphore -import org.apache.commons.codec.digest.DigestUtils -import org.bson.types.ObjectId - -import com.fangjia.core.util.DataUtil -import com.fangjia.core.util.DateUtil -import com.fangjia.datastore.driver.DSCollection -import com.fangjia.datastore.driver.DSCursor -import com.fangjia.datastore.driver.FacetResult -import com.fangjia.script.BaseBatch -import com.fangjia.script.BatchOwner -import com.fangjia.script.Watcher -import com.fangjia.script.util.DbUtils -import com.fangjia.script.util.ExcelUtils -import com.fangjia.script.util.Jms -import com.google.common.collect.Lists -import com.google.common.collect.Sets -import com.mongodb.BasicDBList -import com.mongodb.BasicDBObject -import com.mongodb.Bytes -import com.mongodb.DBCollection -import com.mongodb.DBCursor -import com.mongodb.DBObject -import com.mongodb.Mongo - @BatchOwner(Watcher.LiXue) class CityInfoAnaBatch extends BaseBatch { @@ -53,8 +39,10 @@ class CityInfoAnaBatch extends BaseBatch { final static DBCollection LOUDONG_CONN = MongoUtils.getCollectionByName("fangjia_base", "loudong", MongoUtils.MongoDB_Domain_mongo); final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000); final static DBCollection FACILITY_CONN = MongoUtils.getCollectionByName("fangjia", "district_facilitys", MongoUtils.MongoDB_Domain_mongo) + final static DBCollection OFFICE_RENT_STAT_CONN = MongoUtils.getCollectionByName("fangjia_stat", "office_rent_stat", MongoUtils.MongoDB_Domain_mongo); static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616') - final static DSCollection SEAWEED = DbUtils.seaweed +// final static DSCollection SEAWEED = DbUtils.seaweed + static final DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com") public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) final static def CITYS = []; @@ -125,7 +113,7 @@ class CityInfoAnaBatch extends BaseBatch { SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); } allCitys.each {k,v -> -// if(["南京"].contains(k.toString())){ +// if(["北京"].contains(k.toString())){ CITYS << k.toString(); // } } @@ -140,72 +128,47 @@ class CityInfoAnaBatch extends BaseBatch { // 城市小区数 def allCityDisCount = [:]; - LinkedHashMap facet = new LinkedHashMap(2); - facet.put("fields", ["city"]); - facet.put('limit', 500); - - DBObject disQuery = new BasicDBObject(); - disQuery.put('$facet', facet); - disQuery.put("visible", 0); - disQuery.put("cat", 'district'); - - DSCursor sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0); - List disFacetResult = sellDSCursor.getFacetResults(); - if (disFacetResult) { - def obj = disFacetResult[0].values; - obj.each { - allCityDisCount.put(it.name, it.count.intValue()); - } + BasicDBObject match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "district")); + DBObject groupFields = new BasicDBObject(); + groupFields.put("_id", new BasicDBObject("city", '$city')) + groupFields.put("count", new BasicDBObject('$sum', 1)); + DBObject group = new BasicDBObject('$group', groupFields); + // 定义聚合管道,按 city 字段分组,并计算每个城市的数量 + AggregationOutput output = SEAWEED.aggregate(Arrays.asList(match, group)); + for (DBObject result : output.results()) { + def obj = (DBObject ) result.get("_id"); + allCityDisCount.put(obj.get("city"), result.get("count")) } // 城市新盘数 def allCityNewDisCount = [:]; - disQuery = new BasicDBObject(); - disQuery.put('$facet', facet); - disQuery.put("visible", 0); - disQuery.put("cat", 'new_district'); - - sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0); - disFacetResult = sellDSCursor.getFacetResults(); - if (disFacetResult) { - def obj = disFacetResult[0].values; - obj.each { - allCityNewDisCount.put(it.name, it.count.intValue()); - } + match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "new_district")); + output = SEAWEED.aggregate(Arrays.asList(match, group)); + for (DBObject result : output.results()) { + def obj = (DBObject ) result.get("_id"); + allCityNewDisCount.put(obj.get("city"), result.get("count")) } // 城市写字楼数 def allCityOfficeCount = [:]; - disQuery = new BasicDBObject(); - disQuery.put('$facet', facet); - disQuery.put("visible", 0); - disQuery.put("cat", 'office'); + match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "office")); - sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0); - disFacetResult = sellDSCursor.getFacetResults(); - if (disFacetResult) { - def obj = disFacetResult[0].values; - obj.each { - allCityOfficeCount.put(it.name, it.count.intValue()); - } + output = SEAWEED.aggregate(Arrays.asList(match, group)); + for (DBObject result : output.results()) { + def obj = (DBObject ) result.get("_id"); + allCityOfficeCount.put(obj.get("city"), result.get("count")) } - // 城市商铺数 def allCityShopCount = [:]; - disQuery = new BasicDBObject(); - disQuery.put('$facet', facet); - disQuery.put("visible", 0); - disQuery.put("cat", 'shop'); + match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "shop")); - sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0); - disFacetResult = sellDSCursor.getFacetResults(); - if (disFacetResult) { - def obj = disFacetResult[0].values; - obj.each { - allCityShopCount.put(it.name, it.count.intValue()); - } + output = SEAWEED.aggregate(Arrays.asList(match, group)); + for (DBObject result : output.results()) { + def obj = (DBObject ) result.get("_id"); + allCityShopCount.put(obj.get("city"), result.get("count")) } + // 城市均价 def allCityStats = [:]; query = new BasicDBObject(); @@ -220,10 +183,10 @@ class CityInfoAnaBatch extends BaseBatch { doProcess(allCitys.get(it), weekend, laseWeekend, allHouseStats, allCityDisCount, allCityNewDisCount, allCityOfficeCount, allCityShopCount, allCityStats); } - //统计有楼栋的小区数和有房号的小区数 -// countDistrictsHasLoudong(allCitys, weekend) - //countDistrictsHasLoudongNew(allCitys, weekend) - +// //统计有楼栋的小区数和有房号的小区数 +//// countDistrictsHasLoudong(allCitys, weekend) +// //countDistrictsHasLoudongNew(allCitys, weekend) +// final CountDownLatch latch = new CountDownLatch(CITYS.size()) final Semaphore semaphore = new Semaphore(10) CITYS.each { @@ -239,13 +202,13 @@ class CityInfoAnaBatch extends BaseBatch { } finally { semaphore.release() latch.countDown() - } + } }; }) } latch.await() println "${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())}\t楼栋覆盖率统计完成" - + latch = new CountDownLatch(CITYS.size()) semaphore = new Semaphore(10) CITYS.each { @@ -267,12 +230,12 @@ class CityInfoAnaBatch extends BaseBatch { } latch.await() println "${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())}\t房号覆盖率统计完成" - - + + //71个城市的覆盖率统计 sendCountData(weekend) - - + + //最后统计之前因为太慢取消统计的本周估价次数 latch = new CountDownLatch(CITYS.size()) semaphore = new Semaphore(10) @@ -292,14 +255,14 @@ class CityInfoAnaBatch extends BaseBatch { query.put('city', cityName); query.put('c_date', new BasicDBObject('$gte', laseWeekend).append('$lte', weekend)); entity.put('evaluation_count', (int) EVALUATION_CONN.count(query)); - + if (entity.get('evaluation_count')) { entity.put('price_to_eva', DataUtil.roundDouble(entity.get('price_total_count'), entity.get('evaluation_count'))); } CITY_INFO_CONN.save(entity) println cityName+"本周估价次数统计完毕" } - + } catch (Exception e) { println e.getMessage() } finally { @@ -585,8 +548,8 @@ class CityInfoAnaBatch extends BaseBatch { print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 小区完成]" -// if (allCityNewDisCount.get(cityName)) { -// def newDisCount = allCityNewDisCount.get(cityName); +// if (allCityNewDisCount.get(cityName)) { +// def newDisCount = allCityNewDisCount.get(cityName); // entity.put('new_district_detail', newDistrictDetail(newDisCount, cityName)); // } else { // entity.put('new_district_detail', [:]); @@ -604,23 +567,30 @@ class CityInfoAnaBatch extends BaseBatch { if (officePrices) { detail.put('price', roundDouble(officePrices.size(), officeCount)); } - - query = new BasicDBObject(); - query.put('city', cityName); - query.put('kind', '写字楼'); - query.put('weekend', weekend); - def officeRents = RENT_STAT_CONN.distinct("name", query); + + def officeRents = findOfficeRent(cityName) if (officeRents) { detail.put('rentPrice', roundDouble(officeRents.size(), officeCount)); } + + def halfYearofficeRentCount = findHalfYearOfficeRentCount(cityName) + if (halfYearofficeRentCount) { + detail.put('halfYearRentPrice', roundDouble(halfYearofficeRentCount, officeCount)); + } + entity.put('office_detail', detail); entity.put('existing_office_count', officeCount); } else { entity.put('office_detail', [:]); } - - print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 写字楼完成]" + +// if(curInfo){ +// CITY_INFO_CONN.update(new BasicDBObject("_id", curInfo._id), new BasicDBObject('$set', entity)) +// print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 写字楼完成]" +// } + + if (entity.get('official_population') && entity.get('district_avg_population')) { entity.put('prediction_district_count', (int) (entity.get('official_population') / entity.get('district_avg_population'))); @@ -628,12 +598,12 @@ class CityInfoAnaBatch extends BaseBatch { entity.put('district_coverage', roundDouble(entity.get('existing_district_count'), entity.get('prediction_district_count'))); } } - - + + if (allCityShopCount.get(cityName)) { def shopCount = allCityShopCount.get(cityName); def detail = shopDetail(shopCount, cityName); - + DBObject query = new BasicDBObject(); query.put('city', cityName); //query.put('estate_type', '商铺'); @@ -642,7 +612,7 @@ class CityInfoAnaBatch extends BaseBatch { if (shopPrices) { detail.put('price', roundDouble(shopPrices.size(), shopCount)); } - + query = new BasicDBObject(); query.put('city', cityName); query.put('kind', '商铺'); @@ -651,45 +621,41 @@ class CityInfoAnaBatch extends BaseBatch { if (shopRents) { detail.put('rentPrice', roundDouble(shopRents.size(), shopCount)); } - + entity.put('shop_detail', detail); entity.put('existing_shop_count', shopCount); }else { entity.put('shop_detail', [:]); } print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 商铺完成]" - + def cityStat = allCityStats.get(cityName); if (cityStat) { entity.put('avg_price', cityStat.avg_price); } - + DBObject query = new BasicDBObject(); query.put('city', cityName); query.put('c_date', new BasicDBObject('$gte', lastWeekend).append('$lte', weekend)); //entity.put('evaluation_count', (int) EVALUATION_CONN.count(query));由于估价记录数据量大,查询慢,先去掉这个,最后面在用多线程统计 entity.put('price_total_count', (int) PRICE_JIUCUO_CONN.count(query)); - + query.put('price_lev', '偏高'); entity.put('price_high_count', (int) PRICE_JIUCUO_CONN.count(query)); - + query.put('price_lev', '偏低'); entity.put('price_low_count', (int) PRICE_JIUCUO_CONN.count(query)); - + /*if (entity.get('evaluation_count')) {由于估价记录数据量大,查询慢,先去掉这个,最后面在用多线程统计 entity.put('price_to_eva', roundDouble(entity.get('price_total_count'), entity.get('evaluation_count'))); }*/ - + if (entity.get('price_high_count')) { entity.put('low_to_high', roundDouble(entity.get('price_low_count'), entity.get('price_high_count'))); } - + print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 估价完成]" - - - - - + // query = new BasicDBObject(); // query.put('city', cityName); // query.put('weekend', weekend); @@ -710,7 +676,7 @@ class CityInfoAnaBatch extends BaseBatch { // } // } // } -// +// // if (total > 0) { // BigDecimal bg = new BigDecimal(webZhunque * 1.0 / total); // entity.put('web_average', bg.setScale(4, BigDecimal.ROUND_HALF_UP).doubleValue()); @@ -1284,7 +1250,7 @@ class CityInfoAnaBatch extends BaseBatch { fields.put(key, 1) } - DSCursor cur = SEAWEED.find(query, fields).batchSize(2000) + def cur = SEAWEED.find(query, fields).batchSize(2000) Iterator iter = cur.iterator() def countMap = [:] println cur.count() @@ -1335,7 +1301,7 @@ class CityInfoAnaBatch extends BaseBatch { fields.put(key, 1) } - DSCursor cur = SEAWEED.find(query, fields).batchSize(2000) + def cur = SEAWEED.find(query, fields).batchSize(2000) Iterator iter = cur.iterator() def countMap = [:] @@ -1546,6 +1512,69 @@ class CityInfoAnaBatch extends BaseBatch { } return true; } + + def findOfficeRent(cityName){ + def query = new BasicDBObject(); + query.put('city', cityName); +// query.put('kind', '写字楼'); +// query.put('weekend', weekend); +// def officeRents = RENT_STAT_CONN.distinct("name", query); +// if (officeRents) { +// detail.put('rentPrice', roundDouble(officeRents.size(), officeCount)); +// } + query.put('cat', 'office'); + def date1 = new Date() + date1.setDate(28) + if(new Date().getTime()> date1.getTime()){ + query.put('date', Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"))); + }else{ + query.put('date', Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"))); + } + def officeRents = OFFICE_RENT_STAT_CONN.distinct("name", query); + return officeRents + } + + def findHalfYearOfficeRentCount(cityName){ + + def date1 = new Date() + date1.setDate(28) + def months = [] + if(new Date().getTime()> date1.getTime()){ + for(def i=1;i<=6;i++){ + months << Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1*i, "yyyyMM")) + } + }else{ + for(def i=2;i<=7;i++){ + months << Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1*i, "yyyyMM")) + } + } + def map = [:] + for(def month : months){ + def query = new BasicDBObject(); + query.put('city', cityName); + query.put('cat', 'office'); + query.put('date', month); + def officeRents = OFFICE_RENT_STAT_CONN.distinct("name", query); + + officeRents.each{k-> + def count = map.get(k) + if(!count){ + map.put(k, 1) + }else{ + map.put(k, count + 1) + } + + } + } + def total = 0 + map.each{k,v-> + if(v==6){ + total ++ + } + } + + return total + } }