Merge branch 'script_gj' into 'master'

老脚本查房源修改

See merge request !1094
This commit is contained in:
2025-03-21 16:54:32 +08:00
@@ -1,6 +1,5 @@
package com.fangjia.script.district
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DSCursor
@@ -9,11 +8,20 @@ import com.fangjia.datastore.driver.FacetResult
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* Created by dxy on 2016/12/15.
*/
@@ -26,6 +34,10 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
private static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
private static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName("fangjia", "house_loudong", "mongo.db.fangjia.com")
private static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
// 最大重试次数
private static final int MAX_TRY_COUNT = 5;
public void process(Object args) {
def cities = [
@@ -47,7 +59,7 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
"图木舒克","五家渠","济源","仙桃","潜江","天门","景德镇", "神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水",
"保亭","琼中","北屯","铁门关","双河","可克达拉"
]
// cities = ["无锡"]
cities = ["鹤岗"]
println cities
for(String city : cities) {
List<DBObject> districts = findDistrictByCity(city)
@@ -63,14 +75,14 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
for(DBObject district : districts) {
String key = district.get("name").toString().trim() + "_" + district.get("region") + "_" + district.get("city")
long sellCount = sellCountMap.get(key) ? sellCountMap.get(key) : 0
if(district.get('alias')) {
for (String ali : district.get('alias')) {
if(ali) {
String skey = ali.toString().trim() + "_" + district.get("region") + "_" + district.get("city")
sellCount += (sellCountMap.get(skey) ? sellCountMap.get(skey) : 0)
}
}
}
// if(district.get('alias')) {
// for (String ali : district.get('alias')) {
// if(ali) {
// String skey = ali.toString().trim() + "_" + district.get("region") + "_" + district.get("city")
// sellCount += (sellCountMap.get(skey) ? sellCountMap.get(skey) : 0)
// }
// }
// }
sellCountsList << (int)sellCount
}
sellCountsList = sellCountsList.unique()
@@ -219,67 +231,74 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
}
private def findDistrictHouseCount(String city, List<DBObject> districts, Date date) {
def regionDistrictMap = [:]
for(DBObject dbo : districts) {
def names = []
String n = DataUtil.escapeSolr(dbo.get('name'))
if(n) {
names << n
}
if(dbo.get("alias")) {
for(String ali : dbo.get("alias")) {
ali = DataUtil.escapeSolr(ali)
if(ali) {
names << ali
}
}
}
String region = dbo.get('region')
if(!regionDistrictMap.containsKey(region)) {
regionDistrictMap.put(region, names)
}else {
def list = regionDistrictMap.get(region)
list.addAll(names)
regionDistrictMap.put(region, list)
}
}
def startDate = DateUtil.formatByPattern(DateUtil.getFirstDayOfMonth(DateUtil.addMonthToDate(date, -1)),"yyyyMMdd")
def endDate = DateUtil.formatByPattern(DateUtil.getFirstDayOfMonth(date),"yyyyMMdd")
Map<String, Long> monthMap = findAllCommHouse(city, districts, startDate.toInteger(), endDate.toInteger())
Map<String, Long> halfMap = new HashMap<String, Long>();
Map<String, Long> monthMap = new HashMap<String, Long>();
for(String region : regionDistrictMap.keySet()) {
def nameGroups = []
int len = 0
def nameList = []
for(String name : regionDistrictMap.get(region)) {
if(len >= 600 || nameList.size() >= 100) {
nameGroups << new ArrayList(nameList);
nameList.clear()
len = 0
}
len += name.length()
nameList << name
}
if(nameList.size() > 0) {
nameGroups << new ArrayList(nameList);
}
def startDate2 = DateUtil.formatByPattern(DateUtil.getFirstDayOfMonth(DateUtil.addMonthToDate(date, -6)),"yyyyMMdd")
Map<String, Long> halfMap = findAllCommHouse(city, districts, startDate2.toInteger(), endDate.toInteger())
println city + "\t" + region + "\t" + regionDistrictMap.get(region).size() + "/" + nameGroups.size() + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date())
for(List<String> names : nameGroups) {
def halfRegionResult = [:]
def monthRegionResult = [:]
try {
halfRegionResult = facetDistrictCount(city, region, names, date, 7)
monthRegionResult = facetDistrictCount(city, region, names, date, 2)
}catch (e) {
sleep(10 * 1000)
halfRegionResult = facetDistrictCount(city, region, names, date, 7)
monthRegionResult = facetDistrictCount(city, region, names, date, 2)
}
halfMap.putAll(halfRegionResult)
monthMap.putAll(monthRegionResult)
}
}
// def regionDistrictMap = [:]
// for(DBObject dbo : districts) {
// def names = []
// String n = DataUtil.escapeSolr(dbo.get('name'))
// if(n) {
// names << n
// }
// if(dbo.get("alias")) {
// for(String ali : dbo.get("alias")) {
// ali = DataUtil.escapeSolr(ali)
// if(ali) {
// names << ali
// }
// }
// }
//
// String region = dbo.get('region')
// if(!regionDistrictMap.containsKey(region)) {
// regionDistrictMap.put(region, names)
// }else {
// def list = regionDistrictMap.get(region)
// list.addAll(names)
// regionDistrictMap.put(region, list)
// }
// }
//
//
// for(String region : regionDistrictMap.keySet()) {
// def nameGroups = []
// int len = 0
// def nameList = []
// for(String name : regionDistrictMap.get(region)) {
// if(len >= 600 || nameList.size() >= 100) {
// nameGroups << new ArrayList(nameList);
// nameList.clear()
// len = 0
// }
// len += name.length()
// nameList << name
// }
// if(nameList.size() > 0) {
// nameGroups << new ArrayList(nameList);
// }
//
// println city + "\t" + region + "\t" + regionDistrictMap.get(region).size() + "/" + nameGroups.size() + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date())
// for(List<String> names : nameGroups) {
// def halfRegionResult = [:]
// def monthRegionResult = [:]
// try {
// halfRegionResult = facetDistrictCount(city, region, names, date, 7)
// monthRegionResult = facetDistrictCount(city, region, names, date, 2)
// }catch (e) {
// sleep(10 * 1000)
// halfRegionResult = facetDistrictCount(city, region, names, date, 7)
// monthRegionResult = facetDistrictCount(city, region, names, date, 2)
// }
// halfMap.putAll(halfRegionResult)
// monthMap.putAll(monthRegionResult)
// }
// }
return [halfMap, monthMap];
}
@@ -352,6 +371,106 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
return ret
}
def findAllCommHouse(city, districts, startDate, endDate){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println city +" 小区量:"+districts.size()
def retMap = new ConcurrentHashMap()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def count = findHousesByDistrict(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
if(count){
String key = rMap.get("name") + "_" + rMap.get("region") + "_" + rMap.get("city")
retMap.put(key, count)
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" 原始:"+count
}
} catch (Exception e) {
println totalCount.get() +" 查房源Error: "+city+" "+rMap.region+" "+rMap.name+" "+e.getMessage()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return retMap
}
def findHousesByDistrict(city, region, cat, startDate, endDate, name, alias) {
int retryCount = 0;
boolean success = false;
def count = 0
while (retryCount < MAX_TRY_COUNT && !success) {
try {
count = findHousesByMdate(city, region, cat, startDate, endDate, name, alias)
// 如果操作成功,设置success为true
success = true;
} catch (Exception e) {
retryCount++;
println "查询重试 " + retryCount + "/" + MAX_TRY_COUNT
// 这里可以选择在重试之前等待一段时间
Thread.sleep(3000); // 例如等待1秒
}
}
if (!success) {
println "重试5次后依然失败: " + MAX_TRY_COUNT + " "+city +" "+region+" "+name
}
return count;
}
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(6);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
return houses.size();
}
static main(args) {
runBatch(new DistrictDealSellRentRankBatch(), args)
}