老脚本查房源修改
This commit is contained in:
+187
-68
@@ -1,6 +1,5 @@
|
||||
package com.fangjia.script.district
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
@@ -9,11 +8,20 @@ import com.fangjia.datastore.driver.FacetResult
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
/**
|
||||
* Created by dxy on 2016/12/15.
|
||||
*/
|
||||
@@ -26,6 +34,10 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
|
||||
private static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
private static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName("fangjia", "house_loudong", "mongo.db.fangjia.com")
|
||||
private static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
|
||||
// 最大重试次数
|
||||
private static final int MAX_TRY_COUNT = 5;
|
||||
|
||||
|
||||
public void process(Object args) {
|
||||
def cities = [
|
||||
@@ -47,7 +59,7 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
|
||||
"图木舒克","五家渠","济源","仙桃","潜江","天门","景德镇", "神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水",
|
||||
"保亭","琼中","北屯","铁门关","双河","可克达拉"
|
||||
]
|
||||
// cities = ["无锡"]
|
||||
cities = ["鹤岗"]
|
||||
println cities
|
||||
for(String city : cities) {
|
||||
List<DBObject> districts = findDistrictByCity(city)
|
||||
@@ -63,14 +75,14 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
|
||||
for(DBObject district : districts) {
|
||||
String key = district.get("name").toString().trim() + "_" + district.get("region") + "_" + district.get("city")
|
||||
long sellCount = sellCountMap.get(key) ? sellCountMap.get(key) : 0
|
||||
if(district.get('alias')) {
|
||||
for (String ali : district.get('alias')) {
|
||||
if(ali) {
|
||||
String skey = ali.toString().trim() + "_" + district.get("region") + "_" + district.get("city")
|
||||
sellCount += (sellCountMap.get(skey) ? sellCountMap.get(skey) : 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
// if(district.get('alias')) {
|
||||
// for (String ali : district.get('alias')) {
|
||||
// if(ali) {
|
||||
// String skey = ali.toString().trim() + "_" + district.get("region") + "_" + district.get("city")
|
||||
// sellCount += (sellCountMap.get(skey) ? sellCountMap.get(skey) : 0)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
sellCountsList << (int)sellCount
|
||||
}
|
||||
sellCountsList = sellCountsList.unique()
|
||||
@@ -219,67 +231,74 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
private def findDistrictHouseCount(String city, List<DBObject> districts, Date date) {
|
||||
def regionDistrictMap = [:]
|
||||
for(DBObject dbo : districts) {
|
||||
def names = []
|
||||
String n = DataUtil.escapeSolr(dbo.get('name'))
|
||||
if(n) {
|
||||
names << n
|
||||
}
|
||||
if(dbo.get("alias")) {
|
||||
for(String ali : dbo.get("alias")) {
|
||||
ali = DataUtil.escapeSolr(ali)
|
||||
if(ali) {
|
||||
names << ali
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
String region = dbo.get('region')
|
||||
if(!regionDistrictMap.containsKey(region)) {
|
||||
regionDistrictMap.put(region, names)
|
||||
}else {
|
||||
def list = regionDistrictMap.get(region)
|
||||
list.addAll(names)
|
||||
regionDistrictMap.put(region, list)
|
||||
}
|
||||
}
|
||||
def startDate = DateUtil.formatByPattern(DateUtil.getFirstDayOfMonth(DateUtil.addMonthToDate(date, -1)),"yyyyMMdd")
|
||||
def endDate = DateUtil.formatByPattern(DateUtil.getFirstDayOfMonth(date),"yyyyMMdd")
|
||||
Map<String, Long> monthMap = findAllCommHouse(city, districts, startDate.toInteger(), endDate.toInteger())
|
||||
|
||||
Map<String, Long> halfMap = new HashMap<String, Long>();
|
||||
Map<String, Long> monthMap = new HashMap<String, Long>();
|
||||
for(String region : regionDistrictMap.keySet()) {
|
||||
def nameGroups = []
|
||||
int len = 0
|
||||
def nameList = []
|
||||
for(String name : regionDistrictMap.get(region)) {
|
||||
if(len >= 600 || nameList.size() >= 100) {
|
||||
nameGroups << new ArrayList(nameList);
|
||||
nameList.clear()
|
||||
len = 0
|
||||
}
|
||||
len += name.length()
|
||||
nameList << name
|
||||
}
|
||||
if(nameList.size() > 0) {
|
||||
nameGroups << new ArrayList(nameList);
|
||||
}
|
||||
def startDate2 = DateUtil.formatByPattern(DateUtil.getFirstDayOfMonth(DateUtil.addMonthToDate(date, -6)),"yyyyMMdd")
|
||||
Map<String, Long> halfMap = findAllCommHouse(city, districts, startDate2.toInteger(), endDate.toInteger())
|
||||
|
||||
println city + "\t" + region + "\t" + regionDistrictMap.get(region).size() + "/" + nameGroups.size() + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date())
|
||||
for(List<String> names : nameGroups) {
|
||||
def halfRegionResult = [:]
|
||||
def monthRegionResult = [:]
|
||||
try {
|
||||
halfRegionResult = facetDistrictCount(city, region, names, date, 7)
|
||||
monthRegionResult = facetDistrictCount(city, region, names, date, 2)
|
||||
}catch (e) {
|
||||
sleep(10 * 1000)
|
||||
halfRegionResult = facetDistrictCount(city, region, names, date, 7)
|
||||
monthRegionResult = facetDistrictCount(city, region, names, date, 2)
|
||||
}
|
||||
halfMap.putAll(halfRegionResult)
|
||||
monthMap.putAll(monthRegionResult)
|
||||
}
|
||||
}
|
||||
// def regionDistrictMap = [:]
|
||||
// for(DBObject dbo : districts) {
|
||||
// def names = []
|
||||
// String n = DataUtil.escapeSolr(dbo.get('name'))
|
||||
// if(n) {
|
||||
// names << n
|
||||
// }
|
||||
// if(dbo.get("alias")) {
|
||||
// for(String ali : dbo.get("alias")) {
|
||||
// ali = DataUtil.escapeSolr(ali)
|
||||
// if(ali) {
|
||||
// names << ali
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// String region = dbo.get('region')
|
||||
// if(!regionDistrictMap.containsKey(region)) {
|
||||
// regionDistrictMap.put(region, names)
|
||||
// }else {
|
||||
// def list = regionDistrictMap.get(region)
|
||||
// list.addAll(names)
|
||||
// regionDistrictMap.put(region, list)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//
|
||||
// for(String region : regionDistrictMap.keySet()) {
|
||||
// def nameGroups = []
|
||||
// int len = 0
|
||||
// def nameList = []
|
||||
// for(String name : regionDistrictMap.get(region)) {
|
||||
// if(len >= 600 || nameList.size() >= 100) {
|
||||
// nameGroups << new ArrayList(nameList);
|
||||
// nameList.clear()
|
||||
// len = 0
|
||||
// }
|
||||
// len += name.length()
|
||||
// nameList << name
|
||||
// }
|
||||
// if(nameList.size() > 0) {
|
||||
// nameGroups << new ArrayList(nameList);
|
||||
// }
|
||||
//
|
||||
// println city + "\t" + region + "\t" + regionDistrictMap.get(region).size() + "/" + nameGroups.size() + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date())
|
||||
// for(List<String> names : nameGroups) {
|
||||
// def halfRegionResult = [:]
|
||||
// def monthRegionResult = [:]
|
||||
// try {
|
||||
// halfRegionResult = facetDistrictCount(city, region, names, date, 7)
|
||||
// monthRegionResult = facetDistrictCount(city, region, names, date, 2)
|
||||
// }catch (e) {
|
||||
// sleep(10 * 1000)
|
||||
// halfRegionResult = facetDistrictCount(city, region, names, date, 7)
|
||||
// monthRegionResult = facetDistrictCount(city, region, names, date, 2)
|
||||
// }
|
||||
// halfMap.putAll(halfRegionResult)
|
||||
// monthMap.putAll(monthRegionResult)
|
||||
// }
|
||||
// }
|
||||
|
||||
return [halfMap, monthMap];
|
||||
}
|
||||
@@ -352,6 +371,106 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
|
||||
return ret
|
||||
}
|
||||
|
||||
def findAllCommHouse(city, districts, startDate, endDate){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println city +" 小区量:"+districts.size()
|
||||
|
||||
def retMap = new ConcurrentHashMap()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def count = findHousesByDistrict(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
|
||||
if(count){
|
||||
String key = rMap.get("name") + "_" + rMap.get("region") + "_" + rMap.get("city")
|
||||
retMap.put(key, count)
|
||||
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" 原始:"+count
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
println totalCount.get() +" 查房源Error: "+city+" "+rMap.region+" "+rMap.name+" "+e.getMessage()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return retMap
|
||||
}
|
||||
|
||||
def findHousesByDistrict(city, region, cat, startDate, endDate, name, alias) {
|
||||
int retryCount = 0;
|
||||
boolean success = false;
|
||||
def count = 0
|
||||
|
||||
while (retryCount < MAX_TRY_COUNT && !success) {
|
||||
try {
|
||||
count = findHousesByMdate(city, region, cat, startDate, endDate, name, alias)
|
||||
// 如果操作成功,设置success为true
|
||||
success = true;
|
||||
} catch (Exception e) {
|
||||
retryCount++;
|
||||
println "查询重试 " + retryCount + "/" + MAX_TRY_COUNT
|
||||
// 这里可以选择在重试之前等待一段时间
|
||||
Thread.sleep(3000); // 例如等待1秒
|
||||
}
|
||||
}
|
||||
|
||||
if (!success) {
|
||||
println "重试5次后依然失败: " + MAX_TRY_COUNT + " "+city +" "+region+" "+name
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
|
||||
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
|
||||
def index = new Random().nextInt(6);
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
|
||||
return houses.size();
|
||||
}
|
||||
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictDealSellRentRankBatch(), args)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user