指数脚本修改

This commit is contained in:
2024-11-01 14:37:38 +08:00
parent 9fcc8b3857
commit fac47d1faf
@@ -1,16 +1,14 @@
package com.fangjia.script.trade package com.fangjia.script.trade
import com.fangjia.core.util.DataUtil import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
///导入价格指数 ///导入价格指数
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
@@ -19,10 +17,8 @@ import com.mongodb.DBCollection
import com.mongodb.DBObject import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.apache.commons.lang.math.RandomUtils import org.apache.commons.lang.math.RandomUtils
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.EsDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors import java.util.concurrent.Executors
@@ -74,13 +70,19 @@ class ImportIndexStats extends BaseBatch{
// updateDataType() // updateDataType()
// synch() // synch()
// synchTjjIndex()//同步统计局抓取过来的指数 // synchTjjIndex()//批量同步统计局抓取过来的指数
Calendar calendar=Calendar.getInstance(); Calendar calendar=Calendar.getInstance();
def month= calendar.get(Calendar.MONTH); def month= calendar.get(Calendar.MONTH);
def year= calendar.get(Calendar.YEAR); def year= calendar.get(Calendar.YEAR);
def date = createDateString(year, month, 0, ""); def date = createDateString(year, month, 0, "");
calculatePriceIndex(year, month, date);
def day = calendar.get(Calendar.DATE)
if(day == 5){
calculatePriceIndex(year, month, date);
}
createTjjData(date)//同步单期统计局数据
// addLostIndex() // addLostIndex()
// addHistoryIndex() // addHistoryIndex()
@@ -199,7 +201,7 @@ class ImportIndexStats extends BaseBatch{
runDates << map runDates << map
s = DateUtils.addMonth(date, 1, "yyyyMM") s = DateUtils.addMonth(date, 1, "yyyyMM")
if(year==2024 && month==8){ if(year==2024 && month==10){
stop = true; stop = true;
} }
} }
@@ -210,7 +212,7 @@ class ImportIndexStats extends BaseBatch{
//计算统计局没有发布的城市的价格指数 //计算统计局没有发布的城市的价格指数
def calculatePriceIndex(def year, def month, def date){ def calculatePriceIndex(def year, def month, def date){
// def citys = ["五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]// // def citys = ["定安","五指山","琼海","儋州","文昌","万宁","东方","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]//
def citys = CommonDatautils.findAllCity(); def citys = CommonDatautils.findAllCity();
// def date = createDateString(year, month, 0, ""); // def date = createDateString(year, month, 0, "");
def haveIndexCitys = findPriceIndexCitys(date.toInteger()); def haveIndexCitys = findPriceIndexCitys(date.toInteger());
@@ -233,14 +235,14 @@ class ImportIndexStats extends BaseBatch{
if(id){//已计算 if(id){//已计算
continue; continue;
} }
if(date.toInteger()>=201601){ if(date.toInteger()>=203101){
dingjiYear = 2015 dingjiYear = 2030
}else if(date.toInteger()>=202101){
dingjiYear = 2020
}else if(date.toInteger()>=202601){ }else if(date.toInteger()>=202601){
dingjiYear = 2025 dingjiYear = 2025
}else if(date.toInteger()>=203101){ }else if(date.toInteger()>=202101){
dingjiYear = 2030 dingjiYear = 2020
}else if(date.toInteger()>=201601){
dingjiYear = 2015
} }
fixedThreadPool.execute(new Runnable() { fixedThreadPool.execute(new Runnable() {
void run() { void run() {
@@ -259,6 +261,7 @@ class ImportIndexStats extends BaseBatch{
}); });
} }
latch.await(1, TimeUnit.MINUTES); latch.await(1, TimeUnit.MINUTES);
// latch.await(1, TimeUnit.MINUTES);
} }
def calculating(city, year, month, date){ def calculating(city, year, month, date){
@@ -826,69 +829,76 @@ class ImportIndexStats extends BaseBatch{
} }
def createTjjData(date){ def createTjjData(date){
def queryStartDate = DateUtils.parseFromTightByPattern(date+"01", "yyyyMMdd")
def queryEndDate = DateUtils.parseFromTightByPattern(date+"05", "yyyyMMdd")
def sellIndexMap = findTjjIndex(indices_price_house_sellColl, queryStartDate, queryEndDate)
def sellIndexAreaMap = findTjjAreaIndex(indices_price_house_sell_typeColl, queryStartDate, queryEndDate)
def newIndexMap = findTjjIndex(indices_price_house_newColl, queryStartDate, queryEndDate) def count = STATS_INDEX_COLL.count(new BasicDBObject("type","城市").append("source", 1).append("yearmonth", date.toInteger()))
def newAreaIndexAreaMap = findTjjAreaIndex(indices_price_house_new_typeColl, queryStartDate, queryEndDate) if(count == 0){
def rets = [] def queryStartDate = DateUtils.parseFromTightByPattern(date+"01", "yyyyMMdd")
sellIndexMap.each{city, d-> def queryEndDate = DateUtils.parseFromTightByPattern(date+"05", "yyyyMMdd")
def dbo = new BasicDBObject() def sellIndexMap = findTjjIndex(indices_price_house_sellColl, queryStartDate, queryEndDate)
dbo.put("city",d.city) def sellIndexAreaMap = findTjjAreaIndex(indices_price_house_sell_typeColl, queryStartDate, queryEndDate)
dbo.put("yearmonth",date.toInteger())
dbo.put("source" ,1) def newIndexMap = findTjjIndex(indices_price_house_newColl, queryStartDate, queryEndDate)
dbo.put("remark" , "统计局数据导入") def newAreaIndexAreaMap = findTjjAreaIndex(indices_price_house_new_typeColl, queryStartDate, queryEndDate)
dbo.put("cat" , "district") def rets = []
dbo.put("c_date" , new Date()) sellIndexMap.each{city, d->
dbo.put("m_date" , new Date()) def dbo = new BasicDBObject()
dbo.put("type" , "城市") dbo.put("city",d.city)
dbo.put("chain" ,d.mom) dbo.put("yearmonth",date.toInteger())
dbo.put("yeartoyear" , d.yoy) dbo.put("source" ,1)
dbo.put("fixedbase" , d.fixedbase) dbo.put("remark" , "统计局数据导入")
dbo.put("cat" , "district")
dbo.put("c_date" , new Date())
dbo.put("m_date" , new Date())
dbo.put("type" , "城市")
dbo.put("chain" ,d.mom)
dbo.put("yeartoyear" , d.yoy)
dbo.put("fixedbase" , d.fixedbase)
def areaMap = sellIndexAreaMap.get(d.city) def areaMap = sellIndexAreaMap.get(d.city)
def areaObj1 = areaMap.get("90") def areaObj1 = areaMap.get("90")
dbo.put("below_chain" , areaObj1.mom) dbo.put("below_chain" , areaObj1.mom)
dbo.put("below_yeartoyear" , areaObj1.yoy) dbo.put("below_yeartoyear" , areaObj1.yoy)
dbo.put("below_fixedbase" , areaObj1.fixedbase) dbo.put("below_fixedbase" , areaObj1.fixedbase)
def areaObj2 = areaMap.get("90-144") def areaObj2 = areaMap.get("90-144")
dbo.put("among_chain" , areaObj2.mom) dbo.put("among_chain" , areaObj2.mom)
dbo.put("among_yeartoyear" , areaObj2.yoy) dbo.put("among_yeartoyear" , areaObj2.yoy)
dbo.put("among_fixedbase" , areaObj2.fixedbase) dbo.put("among_fixedbase" , areaObj2.fixedbase)
def areaObj3 = areaMap.get("144") def areaObj3 = areaMap.get("144")
dbo.put("above_chain" , areaObj3.mom) dbo.put("above_chain" , areaObj3.mom)
dbo.put("above_yeartoyear" , areaObj3.yoy) dbo.put("above_yeartoyear" , areaObj3.yoy)
dbo.put("above_fixedbase" , areaObj3.fixedbase) dbo.put("above_fixedbase" , areaObj3.fixedbase)
def newObj = newIndexMap.get(d.city) def newObj = newIndexMap.get(d.city)
dbo.put("new_chain" , newObj.mom) dbo.put("new_chain" , newObj.mom)
dbo.put("new_yeartoyear" , newObj.yoy) dbo.put("new_yeartoyear" , newObj.yoy)
dbo.put("new_fixedbase" , newObj.fixedbase) dbo.put("new_fixedbase" , newObj.fixedbase)
def newAreaMap = newAreaIndexAreaMap.get(d.city) def newAreaMap = newAreaIndexAreaMap.get(d.city)
def areaNewObj1 = newAreaMap.get("90") def areaNewObj1 = newAreaMap.get("90")
dbo.put("new_below_chain" , areaNewObj1.mom) dbo.put("new_below_chain" , areaNewObj1.mom)
dbo.put("new_below_yeartoyear" , areaNewObj1.yoy) dbo.put("new_below_yeartoyear" , areaNewObj1.yoy)
dbo.put("new_below_fixedbase" , areaNewObj1.fixedbase) dbo.put("new_below_fixedbase" , areaNewObj1.fixedbase)
def areaNewObj2 = newAreaMap.get("90-144") def areaNewObj2 = newAreaMap.get("90-144")
dbo.put("new_among_chain" , areaNewObj2.mom) dbo.put("new_among_chain" , areaNewObj2.mom)
dbo.put("new_among_yeartoyear" , areaNewObj2.yoy) dbo.put("new_among_yeartoyear" , areaNewObj2.yoy)
dbo.put("new_among_fixedbase" , areaNewObj2.fixedbase) dbo.put("new_among_fixedbase" , areaNewObj2.fixedbase)
def areaNewObj3 = newAreaMap.get("144") def areaNewObj3 = newAreaMap.get("144")
dbo.put("new_above_chain" , areaNewObj3.mom) dbo.put("new_above_chain" , areaNewObj3.mom)
dbo.put("new_above_yeartoyear" , areaNewObj3.yoy) dbo.put("new_above_yeartoyear" , areaNewObj3.yoy)
dbo.put("new_above_fixedbase" , areaNewObj3.fixedbase) dbo.put("new_above_fixedbase" , areaNewObj3.fixedbase)
rets << dbo
}
if(rets){
STATS_INDEX_COLL.insert(rets)
}
rets << dbo
} }
STATS_INDEX_COLL.insert(rets)
} }
def findTjjIndex(coll, queryStartDate, queryEndDate){ def findTjjIndex(coll, queryStartDate, queryEndDate){