From d8e781617c663862bd8e502471ee2eb85c8e5674 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 21 Mar 2025 16:51:14 +0800 Subject: [PATCH] =?UTF-8?q?=E8=80=81=E8=84=9A=E6=9C=AC=E6=9F=A5=E6=88=BF?= =?UTF-8?q?=E6=BA=90=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DistrictDealSellRentRankBatch.groovy | 255 +++++++++++++----- 1 file changed, 187 insertions(+), 68 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictDealSellRentRankBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictDealSellRentRankBatch.groovy index 5b9e5d8b5..b670962db 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictDealSellRentRankBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictDealSellRentRankBatch.groovy @@ -1,6 +1,5 @@ package com.fangjia.script.district -import com.fangjia.core.util.DataUtil import com.fangjia.core.util.DateUtil import com.fangjia.datastore.driver.DSCollection import com.fangjia.datastore.driver.DSCursor @@ -9,11 +8,20 @@ import com.fangjia.datastore.driver.FacetResult import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong /** * Created by dxy on 2016/12/15. */ @@ -26,6 +34,10 @@ class DistrictDealSellRentRankBatch extends BaseBatch { private static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") private static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName("fangjia", "house_loudong", "mongo.db.fangjia.com") private static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com") + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) + // 最大重试次数 + private static final int MAX_TRY_COUNT = 5; + public void process(Object args) { def cities = [ @@ -47,7 +59,7 @@ class DistrictDealSellRentRankBatch extends BaseBatch { "图木舒克","五家渠","济源","仙桃","潜江","天门","景德镇", "神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水", "保亭","琼中","北屯","铁门关","双河","可克达拉" ] -// cities = ["无锡"] + cities = ["鹤岗"] println cities for(String city : cities) { List districts = findDistrictByCity(city) @@ -63,14 +75,14 @@ class DistrictDealSellRentRankBatch extends BaseBatch { for(DBObject district : districts) { String key = district.get("name").toString().trim() + "_" + district.get("region") + "_" + district.get("city") long sellCount = sellCountMap.get(key) ? sellCountMap.get(key) : 0 - if(district.get('alias')) { - for (String ali : district.get('alias')) { - if(ali) { - String skey = ali.toString().trim() + "_" + district.get("region") + "_" + district.get("city") - sellCount += (sellCountMap.get(skey) ? sellCountMap.get(skey) : 0) - } - } - } +// if(district.get('alias')) { +// for (String ali : district.get('alias')) { +// if(ali) { +// String skey = ali.toString().trim() + "_" + district.get("region") + "_" + district.get("city") +// sellCount += (sellCountMap.get(skey) ? sellCountMap.get(skey) : 0) +// } +// } +// } sellCountsList << (int)sellCount } sellCountsList = sellCountsList.unique() @@ -219,67 +231,74 @@ class DistrictDealSellRentRankBatch extends BaseBatch { } private def findDistrictHouseCount(String city, List districts, Date date) { - def regionDistrictMap = [:] - for(DBObject dbo : districts) { - def names = [] - String n = DataUtil.escapeSolr(dbo.get('name')) - if(n) { - names << n - } - if(dbo.get("alias")) { - for(String ali : dbo.get("alias")) { - ali = DataUtil.escapeSolr(ali) - if(ali) { - names << ali - } - } - } - String region = dbo.get('region') - if(!regionDistrictMap.containsKey(region)) { - regionDistrictMap.put(region, names) - }else { - def list = regionDistrictMap.get(region) - list.addAll(names) - regionDistrictMap.put(region, list) - } - } + def startDate = DateUtil.formatByPattern(DateUtil.getFirstDayOfMonth(DateUtil.addMonthToDate(date, -1)),"yyyyMMdd") + def endDate = DateUtil.formatByPattern(DateUtil.getFirstDayOfMonth(date),"yyyyMMdd") + Map monthMap = findAllCommHouse(city, districts, startDate.toInteger(), endDate.toInteger()) - Map halfMap = new HashMap(); - Map monthMap = new HashMap(); - for(String region : regionDistrictMap.keySet()) { - def nameGroups = [] - int len = 0 - def nameList = [] - for(String name : regionDistrictMap.get(region)) { - if(len >= 600 || nameList.size() >= 100) { - nameGroups << new ArrayList(nameList); - nameList.clear() - len = 0 - } - len += name.length() - nameList << name - } - if(nameList.size() > 0) { - nameGroups << new ArrayList(nameList); - } + def startDate2 = DateUtil.formatByPattern(DateUtil.getFirstDayOfMonth(DateUtil.addMonthToDate(date, -6)),"yyyyMMdd") + Map halfMap = findAllCommHouse(city, districts, startDate2.toInteger(), endDate.toInteger()) - println city + "\t" + region + "\t" + regionDistrictMap.get(region).size() + "/" + nameGroups.size() + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date()) - for(List names : nameGroups) { - def halfRegionResult = [:] - def monthRegionResult = [:] - try { - halfRegionResult = facetDistrictCount(city, region, names, date, 7) - monthRegionResult = facetDistrictCount(city, region, names, date, 2) - }catch (e) { - sleep(10 * 1000) - halfRegionResult = facetDistrictCount(city, region, names, date, 7) - monthRegionResult = facetDistrictCount(city, region, names, date, 2) - } - halfMap.putAll(halfRegionResult) - monthMap.putAll(monthRegionResult) - } - } +// def regionDistrictMap = [:] +// for(DBObject dbo : districts) { +// def names = [] +// String n = DataUtil.escapeSolr(dbo.get('name')) +// if(n) { +// names << n +// } +// if(dbo.get("alias")) { +// for(String ali : dbo.get("alias")) { +// ali = DataUtil.escapeSolr(ali) +// if(ali) { +// names << ali +// } +// } +// } +// +// String region = dbo.get('region') +// if(!regionDistrictMap.containsKey(region)) { +// regionDistrictMap.put(region, names) +// }else { +// def list = regionDistrictMap.get(region) +// list.addAll(names) +// regionDistrictMap.put(region, list) +// } +// } +// +// +// for(String region : regionDistrictMap.keySet()) { +// def nameGroups = [] +// int len = 0 +// def nameList = [] +// for(String name : regionDistrictMap.get(region)) { +// if(len >= 600 || nameList.size() >= 100) { +// nameGroups << new ArrayList(nameList); +// nameList.clear() +// len = 0 +// } +// len += name.length() +// nameList << name +// } +// if(nameList.size() > 0) { +// nameGroups << new ArrayList(nameList); +// } +// +// println city + "\t" + region + "\t" + regionDistrictMap.get(region).size() + "/" + nameGroups.size() + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date()) +// for(List names : nameGroups) { +// def halfRegionResult = [:] +// def monthRegionResult = [:] +// try { +// halfRegionResult = facetDistrictCount(city, region, names, date, 7) +// monthRegionResult = facetDistrictCount(city, region, names, date, 2) +// }catch (e) { +// sleep(10 * 1000) +// halfRegionResult = facetDistrictCount(city, region, names, date, 7) +// monthRegionResult = facetDistrictCount(city, region, names, date, 2) +// } +// halfMap.putAll(halfRegionResult) +// monthMap.putAll(monthRegionResult) +// } +// } return [halfMap, monthMap]; } @@ -352,6 +371,106 @@ class DistrictDealSellRentRankBatch extends BaseBatch { return ret } + def findAllCommHouse(city, districts, startDate, endDate){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println city +" 小区量:"+districts.size() + + def retMap = new ConcurrentHashMap() + for(def i=0;i0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(6); + + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); + return houses.size(); + } + + + static main(args) { runBatch(new DistrictDealSellRentRankBatch(), args) }