Merge branch 'script_gj' into 'master'

多线程

See merge request !961
This commit is contained in:
2024-09-29 11:06:50 +08:00
@@ -8,6 +8,7 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
@@ -19,6 +20,10 @@ import org.apache.commons.lang.math.RandomUtils
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
@BatchOwner(Watcher.GaoJuan)
class ImportIndexStats extends BaseBatch{
@@ -33,6 +38,7 @@ class ImportIndexStats extends BaseBatch{
final DBCollection indices_price_house_newColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_new", "192.168.0.206", 27020)
final DBCollection indices_price_house_new_typeColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_new_type", "192.168.0.206", 27020)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
//DSCollection seawater = new DataStore("192.168.0.65", 27777).getCollection("fangjia.seawater")
private static def cityList=["北京", "天津", "石家庄", "太原", "呼和浩特", "沈阳", "大连", "长春", "哈尔滨", "上海", "南京", "杭州", "宁波", "合肥", "福州", "厦门",
@@ -74,80 +80,112 @@ class ImportIndexStats extends BaseBatch{
def s = "201601"
def startDate = null;
def stop = false;
def runDates = []
while(!stop){
startDate = DateUtils.parseFromTightYYYYMM(s);
def year = startDate.getYear()+1900;
def month = startDate.month+1;
def date = createDateString(year, month, 0, "");
def map = [:]
map.year = year
map.month = month
runDates << map
s = DateUtils.addMonth(date, 1, "yyyyMM")
if(year==2024 && month==9){
if(year==2024 && month==8){
stop = true;
}
calculatePriceIndex(year, month);
println "运行......"+date+" 完成"
}
for(def map : runDates){
calculatePriceIndex(map.year, map.month);
}
}
//计算统计局没有发布的城市的价格指数
def calculatePriceIndex(def year, def month){
def citys =CommonDatautils.findAllCity();
def date = createDateString(year, month, 0, "");
def haveIndexCitys = findPriceIndexCitys(date.toInteger());
def index = 0;
if(citys){
for(def city : citys){
if(haveIndexCitys.contains(city)){
continue;
}
def id = findPriceIndex(city, date.toInteger());
if(id){//已计算
continue;
}
def result = [:];
calculateAvgPriceIndex(city, year, month, result);
calculateAreaPriceInfex(result, city, year, month, 90);//0-90
calculateAreaPriceInfex(result, city, year, month, 144);//90-144
calculateAreaPriceInfex(result, city, year, month, 1000);//144-1000
def totalCount = new AtomicLong()
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(citys.size())
// checkPriceIndex(result);//涨跌幅控制
index++;
if(result){
result.city = city;
result.yearmonth = date.toInteger();
result.source = 2;
result.remark = "均价计算";
result.cat = "district";
result.type = "城市"
savePriceIndex(result);
println "${index}"+ result+" ";
}else{
println "${index}"+ result+" "+city +" 无数据";
}
def newCitys = []
for(def i=0;i<citys.size();i++) {
def city = citys[i]
if (!haveIndexCitys.contains(city)) {
newCitys << city
}
}
def total = newCitys.size()
for(def i=0;i<newCitys.size();i++){
final def city = newCitys[i]
def id = findPriceIndex(city, date.toInteger());
if(id){//已计算
continue;
}
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def result = calculating(city, year, month);
println result
println "${city}[${totalCount.get()}/${total}]运行......"+year+"-"+month+" 完成"
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return addCount.get();
}
def calculating(city, year, month){
def result = [:];
calculateAvgPriceIndex(city, year, month, result);
calculateAreaPriceInfex(result, city, year, month, 90);//0-90
calculateAreaPriceInfex(result, city, year, month, 144);//90-144
calculateAreaPriceInfex(result, city, year, month, 1000);//144-1000
checkPriceIndex(result);//涨跌幅控制
if(result){
result.city = city;
result.yearmonth = (year.toString()+month.toString()).toInteger();
result.source = 2;
result.remark = "均价计算";
result.cat = "district";
result.type = "城市"
savePriceIndex(result);
}
return result
}
//计算住房的价格指数
def calculateAvgPriceIndex(def city, def year, def month, def result){
// def curMonthPrice = findCityByDates(city, createDateString(year, month, 1, "-"), createDateString(year, month + 1, 1, "-"));
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString())),"yyyyMMdd")
def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+(month.toString()))),"yyyyMMdd")
def curMonthPrice = findHouse3(city, startDate.toInteger(), endDate.toInteger(), null)
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString())),"yyyy-MM-dd")
def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+(month.toString()))),"yyyy-MM-dd")
def curMonthPrice = findCityByDates(city, startDate, endDate);
// def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString())),"yyyyMMdd")
// def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+(month.toString()))),"yyyyMMdd")
// def curMonthPrice = findHouse3(city, startDate.toInteger(), endDate.toInteger(), null)
float markup = 0f;
float yearMarkup = 0f;
float dingJiMarkup = 0f;
//环比涨跌幅
// def lastMonthPrice = findCityByDates(city, createDateString(year, month - 1, 1, "-"), createDateString(year, month, 1, "-"));
startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString()),-1),"yyyyMMdd")
endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString()),-1),"yyyyMMdd")
def lastMonthPrice = findHouse3(city, startDate.toInteger(), endDate.toInteger(), null)
startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString()),-1),"yyyy-MM-dd")
endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString()),-1),"yyyy-MM-dd")
def lastMonthPrice = findCityByDates(city, startDate, endDate);
// startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString()),-1),"yyyyMMdd")
// endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString()),-1),"yyyyMMdd")
// def lastMonthPrice = findHouse3(city, startDate.toInteger(), endDate.toInteger(), null)
if(curMonthPrice && lastMonthPrice){
markup = curMonthPrice *100/lastMonthPrice
markup = DataUtil.getDoubleRound(1,markup);
@@ -157,10 +195,12 @@ class ImportIndexStats extends BaseBatch{
// markup = v;
// }
//同比涨跌幅
// def lastYearPrice = findCityByDates(city, createDateString(year-1, month, 1, "-"), createDateString(year-1, month+1 , 1, "-"));
startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM((year-1).toString()+month.toString())),"yyyyMMdd")
endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM((year-1).toString()+(month.toString()))),"yyyyMMdd")
def lastYearPrice = findHouse3(city, startDate.toInteger(), endDate.toInteger(), null)
startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM((year-1).toString()+month.toString())),"yyyy-MM-dd")
endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM((year-1).toString()+(month.toString()))),"yyyy-MM-dd")
def lastYearPrice = findCityByDates(city, startDate, endDate);
// startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM((year-1).toString()+month.toString())),"yyyyMMdd")
// endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM((year-1).toString()+(month.toString()))),"yyyyMMdd")
// def lastYearPrice = findHouse3(city, startDate.toInteger(), endDate.toInteger(), null)
if(curMonthPrice && lastYearPrice){
yearMarkup = curMonthPrice * 100/lastYearPrice;
yearMarkup = DataUtil.getRound(1,yearMarkup);
@@ -170,10 +210,12 @@ class ImportIndexStats extends BaseBatch{
// yearMarkup = v;
// }
//定基涨跌幅
// def dingJiPrice = findCityByDates(city, createDateString(dingjiYear, month, 1, "-"), createDateString(dingjiYear, month+1 , 1, "-"));
startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD2(dingjiYear.toString()+"0101")),"yyyyMMdd")
endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD2(dingjiYear.toInteger()+"1231")),"yyyyMMdd")
def dingJiPrice = findHouse3(city, startDate, endDate, null)
startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD2(dingjiYear.toString()+"0101")),"yyyy-MM-dd")
endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD2(dingjiYear.toInteger()+"1231")),"yyyy-MM-dd")
def dingJiPrice = findCityByDates(city, startDate, endDate);
// startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD2(dingjiYear.toString()+"0101")),"yyyyMMdd")
// endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD2(dingjiYear.toInteger()+"1231")),"yyyyMMdd")
// def dingJiPrice = findHouse3(city, startDate, endDate, null)
if(curMonthPrice && dingJiPrice){
dingJiMarkup = curMonthPrice*100/dingJiPrice;
dingJiMarkup = DataUtil.getRound(1,dingJiMarkup);
@@ -269,18 +311,18 @@ class ImportIndexStats extends BaseBatch{
result.each {k,v->
def temp = v - 100;
if(k == "markup" || k == "areaMarkup90" || k == "areaMarkup144" || k == "areaMarkup1000"){
if(temp < -2 || temp > 1){
v = getNewMarkup(temp, -2, 1) + 100;
if(temp < -3 || temp > 3){
v = getNewMarkup(temp, -3, 3) + 100;
detailFlag = 2;
}
}else if(k == "yearMarkup" || k == "areaYearMarkup90" || k == "areaYearMarkup144" || k == "areaYearMarkup1000"){
if(temp < -5 || temp > 7){
v = getNewMarkup(temp, -5, 7) + 100;
if(temp < -20 || temp > 30){
v = getNewMarkup(temp, -20, 30) + 100;
detailFlag = 2;
}
}else{
if(temp < -6 || temp > 29){
v = getNewMarkup(temp, -6, 29) + 100;
if(temp < -50 || temp > 100){
v = getNewMarkup(temp, -50, 100) + 100;
detailFlag = 2;
}
}