Merge branch 'script_gj' into 'master'
Script gj See merge request !958
This commit is contained in:
+293
-98
@@ -9,6 +9,7 @@ import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
@@ -23,16 +24,25 @@ import java.sql.ResultSet
|
||||
class ImportIndexStats extends BaseBatch{
|
||||
//public final static DBCollection under_line = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('stats_index');
|
||||
public static final DBCollection on_line = MongoUtils.getCollectionByName("fangjia", "stats_index", "mongo.db.fangjia.com")
|
||||
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
|
||||
public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
|
||||
//public static final DSCollection SEAWATER_COLL = new DataStore("192.168.0.65", 27777).getCollection("fangjia.seawater")
|
||||
|
||||
|
||||
final DBCollection indices_price_house_sellColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_sell", "192.168.0.206", 27020)
|
||||
final DBCollection indices_price_house_sell_typeColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_sell_type", "192.168.0.206", 27020)
|
||||
|
||||
final DBCollection indices_price_house_newColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_new", "192.168.0.206", 27020)
|
||||
final DBCollection indices_price_house_new_typeColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_new_type", "192.168.0.206", 27020)
|
||||
|
||||
|
||||
//DSCollection seawater = new DataStore("192.168.0.65", 27777).getCollection("fangjia.seawater")
|
||||
private static def cityList=["北京", "天津", "石家庄", "太原", "呼和浩特", "沈阳", "大连", "长春", "哈尔滨", "上海", "南京", "杭州", "宁波", "合肥", "福州", "厦门",
|
||||
"南昌", "济南", "青岛", "郑州", "武汉", "长沙", "广州", "深圳", "南宁", "海口", "重庆", "成都", "贵阳", "昆明", "西安", "兰州", "西宁", "银川", "乌鲁木齐", "唐山",
|
||||
"秦皇岛", "包头", "丹东", "锦州", "吉林", "牡丹江", "无锡", "扬州", "徐州", "温州", "金华", "蚌埠", "安庆", "泉州", "九江", "赣州", "烟台", "济宁", "洛阳", "平顶山",
|
||||
"宜昌", "襄阳", "岳阳", "常德", "惠州", "湛江", "韶关", "桂林", "北海", "三亚", "泸州", "南充", "遵义", "大理"];
|
||||
|
||||
public static final def dingjiYear = 2016;
|
||||
|
||||
|
||||
public static final def dingjiYear = 2015;
|
||||
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
private static final EsDataSource dataSource = new EsDataSource()
|
||||
@@ -50,14 +60,18 @@ class ImportIndexStats extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
// updateDataType()
|
||||
// synch()
|
||||
// synchTjjIndex()//同步统计局抓取过来的指数
|
||||
|
||||
// Calendar calendar=Calendar.getInstance();
|
||||
// def month= calendar.get(Calendar.MONTH)+1;
|
||||
// def year= calendar.get(Calendar.YEAR);
|
||||
// def date = createDateString(year, month, 0, "");
|
||||
// calculatePriceIndex(year, month);
|
||||
|
||||
Calendar calendar=Calendar.getInstance();
|
||||
def month= calendar.get(Calendar.MONTH)+1;
|
||||
def year= calendar.get(Calendar.YEAR);
|
||||
def date = createDateString(year, month, 0, "");
|
||||
calculatePriceIndex(year, month);
|
||||
|
||||
/*def s = "201601"
|
||||
def s = "201601"
|
||||
def startDate = null;
|
||||
def stop = false;
|
||||
while(!stop){
|
||||
@@ -67,12 +81,12 @@ class ImportIndexStats extends BaseBatch{
|
||||
def date = createDateString(year, month, 0, "");
|
||||
s = DateUtils.addMonth(date, 1, "yyyyMM")
|
||||
|
||||
if(year==2017 && month==7){
|
||||
if(year==2024 && month==9){
|
||||
stop = true;
|
||||
}
|
||||
calculatePriceIndex(year, month);
|
||||
println "运行......"+date
|
||||
}*/
|
||||
println "运行......"+date+" 完成"
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -80,83 +94,94 @@ class ImportIndexStats extends BaseBatch{
|
||||
def calculatePriceIndex(def year, def month){
|
||||
def citys =CommonDatautils.findAllCity();
|
||||
def date = createDateString(year, month, 0, "");
|
||||
def haveIndexCitys = findPriceIndexCitys(date);
|
||||
def haveIndexCitys = findPriceIndexCitys(date.toInteger());
|
||||
def index = 0;
|
||||
if(citys){
|
||||
for(def city : citys){
|
||||
if(cityList.contains(city)){
|
||||
if(haveIndexCitys.contains(city)){
|
||||
continue;
|
||||
}
|
||||
|
||||
def id = findPriceIndex(city, date);
|
||||
def id = findPriceIndex(city, date.toInteger());
|
||||
if(id){//已计算
|
||||
continue;
|
||||
}
|
||||
|
||||
def result = [:];
|
||||
calculateAvgPriceIndex(city, year, month, result);
|
||||
|
||||
def curMonthPriceMap = findHouse2(city, createDateString(year, month, 1, "-"), createDateString(year, month + 1, 1, "-"));
|
||||
def lastMonthPriceMap = findHouse2(city, createDateString(year, month-1, 1, "-"), createDateString(year, month , 1, "-"));
|
||||
def lastYearPriceMap = findHouse2(city, createDateString(year-1, month, 1, "-"), createDateString(year-1, month + 1, 1, "-"));
|
||||
def fixPriceMap = findHouse2(city, createDateString(dingjiYear, month, 1, "-"), createDateString(dingjiYear, month+1 , 1, "-"));
|
||||
|
||||
calculateAreaPriceInfex(result, 90, curMonthPriceMap.get("0_90"),lastMonthPriceMap.get("0_90"),lastYearPriceMap.get("0_90"), fixPriceMap.get("0_90"));//0-90
|
||||
calculateAreaPriceInfex(result, 144, curMonthPriceMap.get("90-144"),lastMonthPriceMap.get("90-144"),lastYearPriceMap.get("90-144"), fixPriceMap.get("90-144"));//90-144
|
||||
calculateAreaPriceInfex(result, 1000, curMonthPriceMap.get("144_500"),lastMonthPriceMap.get("144_500"),lastYearPriceMap.get("144_500"), fixPriceMap.get("144_500"));//144-1000
|
||||
|
||||
// calculateAreaPriceInfex(city, year, month, result, 0, 90);//0-90
|
||||
// calculateAreaPriceInfex(city, year, month, result, 90, 144);//90-144
|
||||
// calculateAreaPriceInfex(city, year, month, result, 144, 1000);//144-1000
|
||||
calculateAreaPriceInfex(result, city, year, month, 90);//0-90
|
||||
calculateAreaPriceInfex(result, city, year, month, 144);//90-144
|
||||
calculateAreaPriceInfex(result, city, year, month, 1000);//144-1000
|
||||
|
||||
|
||||
checkPriceIndex(result);
|
||||
// checkPriceIndex(result);//涨跌幅控制
|
||||
index++;
|
||||
if(result){
|
||||
result.city = city;
|
||||
result.yearmonth = date;
|
||||
result.yearmonth = date.toInteger();
|
||||
result.source = 2;
|
||||
result.remark = "均价计算";
|
||||
result.cat = "district";
|
||||
result.type = "城市"
|
||||
savePriceIndex(result);
|
||||
println "${index}"+ result+" ";
|
||||
}else{
|
||||
println "${index}"+ result+" "+city +" 无数据";
|
||||
}
|
||||
println "${index}"+ result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//计算新建住房的价格指数
|
||||
//计算住房的价格指数
|
||||
def calculateAvgPriceIndex(def city, def year, def month, def result){
|
||||
def curMonthPrice = findCityByDates(city, createDateString(year, month, 1, "-"), createDateString(year, month + 1, 1, "-"));
|
||||
// def curMonthPrice = findCityByDates(city, createDateString(year, month, 1, "-"), createDateString(year, month + 1, 1, "-"));
|
||||
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString())),"yyyyMMdd")
|
||||
def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+(month.toString()))),"yyyyMMdd")
|
||||
def curMonthPrice = findHouse3(city, startDate.toInteger(), endDate.toInteger(), null)
|
||||
float markup = 0f;
|
||||
float yearMarkup = 0f;
|
||||
float dingJiMarkup = 0f;
|
||||
//环比涨跌幅
|
||||
def lastMonthPrice = findCityByDates(city, createDateString(year, month - 1, 1, "-"), createDateString(year, month, 1, "-"));
|
||||
|
||||
// def lastMonthPrice = findCityByDates(city, createDateString(year, month - 1, 1, "-"), createDateString(year, month, 1, "-"));
|
||||
startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString()),-1),"yyyyMMdd")
|
||||
endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString()),-1),"yyyyMMdd")
|
||||
def lastMonthPrice = findHouse3(city, startDate.toInteger(), endDate.toInteger(), null)
|
||||
if(lastMonthPrice){
|
||||
markup = curMonthPrice *100/lastMonthPrice
|
||||
markup = DataUtil.getDoubleRound(1,markup);
|
||||
}else{
|
||||
def v = random(-3, 3) + 100;
|
||||
markup = v;
|
||||
}
|
||||
// else{
|
||||
// def v = random(-3, 3) + 100;
|
||||
// markup = v;
|
||||
// }
|
||||
//同比涨跌幅
|
||||
def lastYearPrice = findCityByDates(city, createDateString(year-1, month, 1, "-"), createDateString(year-1, month+1 , 1, "-"));
|
||||
// def lastYearPrice = findCityByDates(city, createDateString(year-1, month, 1, "-"), createDateString(year-1, month+1 , 1, "-"));
|
||||
startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM((year-1).toString()+month.toString())),"yyyyMMdd")
|
||||
endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM((year-1).toString()+(month.toString()))),"yyyyMMdd")
|
||||
def lastYearPrice = findHouse3(city, startDate.toInteger(), endDate.toInteger(), null)
|
||||
if(lastYearPrice){
|
||||
yearMarkup = curMonthPrice * 100/lastYearPrice;
|
||||
yearMarkup = DataUtil.getRound(1,yearMarkup);
|
||||
}else{
|
||||
def v = random(-5, 5) + 100;
|
||||
yearMarkup = v;
|
||||
}
|
||||
// else{
|
||||
// def v = random(-5, 5) + 100;
|
||||
// yearMarkup = v;
|
||||
// }
|
||||
//定基涨跌幅
|
||||
def dingJiPrice = findCityByDates(city, createDateString(dingjiYear, month, 1, "-"), createDateString(dingjiYear, month+1 , 1, "-"));
|
||||
// def dingJiPrice = findCityByDates(city, createDateString(dingjiYear, month, 1, "-"), createDateString(dingjiYear, month+1 , 1, "-"));
|
||||
startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD2(dingjiYear.toString()+"0101")),"yyyyMMdd")
|
||||
endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD2(dingjiYear.toInteger()+"1231")),"yyyyMMdd")
|
||||
def dingJiPrice = findHouse3(city, startDate, endDate, null)
|
||||
if(dingJiPrice){
|
||||
dingJiMarkup = curMonthPrice*100/dingJiPrice;
|
||||
dingJiMarkup = DataUtil.getRound(1,dingJiMarkup);
|
||||
}else{
|
||||
def v = random(-8, 8) + 100;
|
||||
dingJiMarkup = v;
|
||||
}
|
||||
// else{
|
||||
// def v = random(-8, 8) + 100;
|
||||
// dingJiMarkup = v;
|
||||
// }
|
||||
|
||||
if(markup){
|
||||
result.markup = markup;
|
||||
@@ -173,45 +198,53 @@ class ImportIndexStats extends BaseBatch{
|
||||
return result;
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
//计算住宅按平米分类的价格指数
|
||||
def calculateAreaPriceInfex(def result, def endArea, def curHousePrices1, def lastMonthHousePrices1, def lastYearHousePrice2, def dingjiHousePrices3){
|
||||
//def curHousePrices1 = findHouse(city, startArea, endArea, createDateString(year, month, 1, "-"), createDateString(year, month + 1, 1, "-"));
|
||||
def cur_price = getAvgPrice(curHousePrices1);
|
||||
def calculateAreaPriceInfex(def result, def city, def year, def month, def endArea){
|
||||
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString())),"yyyyMMdd")
|
||||
def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+(month.toString()))),"yyyyMMdd")
|
||||
def cur_price = findHouse3(city, startDate, endDate, endArea);
|
||||
def areaMarkup = 0;
|
||||
def areaYearMarkup = 0;
|
||||
def areaDingJiMarkup = 0;
|
||||
//环比涨跌幅
|
||||
//def lastMonthHousePrices1 = findHouse(city, startArea, endArea, createDateString(year, month-1, 1, "-"), createDateString(year, month , 1, "-"));
|
||||
def lastMonthlt90_price = getAvgPrice(lastMonthHousePrices1);
|
||||
startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString()),-1),"yyyyMMdd")
|
||||
endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString()),-1),"yyyyMMdd")
|
||||
def lastMonthlt90_price = findHouse3(city, startDate, endDate, endArea);
|
||||
if(cur_price && lastMonthlt90_price){
|
||||
areaMarkup = cur_price*100/lastMonthlt90_price
|
||||
areaMarkup = DataUtil.getRound(1,areaMarkup);
|
||||
}else{
|
||||
def v = random(-3, 3) + 100;
|
||||
areaMarkup = v;
|
||||
}
|
||||
// else{
|
||||
// def v = random(-3, 3) + 100;
|
||||
// areaMarkup = v;
|
||||
// }
|
||||
|
||||
//def lastYearHousePrice2 = findHouse(city, startArea, endArea, createDateString(year-1, month, 1, "-"), createDateString(year-1, month + 1, 1, "-"));
|
||||
def lastYearprice = getAvgPrice(lastYearHousePrice2);
|
||||
startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM((year-1).toString()+month.toString())),"yyyyMMdd")
|
||||
endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM((year-1).toString()+(month.toString()))),"yyyyMMdd")
|
||||
def lastYearprice = findHouse3(city, startDate, endDate, endArea);
|
||||
//同比涨跌幅
|
||||
if(cur_price && lastYearprice){
|
||||
areaYearMarkup = cur_price *100/lastYearprice
|
||||
areaYearMarkup = DataUtil.getRound(1,areaYearMarkup);
|
||||
}else{
|
||||
def v = random(-5, 5) + 100;
|
||||
areaYearMarkup = v;
|
||||
}
|
||||
// else{
|
||||
// def v = random(-5, 5) + 100;
|
||||
// areaYearMarkup = v;
|
||||
// }
|
||||
//定基涨跌幅
|
||||
//def dingjiHousePrices3 = findHouse(city, startArea, endArea, createDateString(2010, month, 1, "-"), createDateString(2010, month+1 , 1, "-"));
|
||||
def dingJiPrice = getAvgPrice(dingjiHousePrices3);
|
||||
startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD2(dingjiYear.toString()+"0101")),"yyyyMMdd")
|
||||
endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD2(dingjiYear.toInteger()+"1231")),"yyyyMMdd")
|
||||
def dingJiPrice = findHouse3(city, startDate, endDate, endArea);
|
||||
if(cur_price && dingJiPrice){
|
||||
areaDingJiMarkup = cur_price *100/dingJiPrice
|
||||
areaDingJiMarkup = DataUtil.getRound(1,areaDingJiMarkup);
|
||||
}else{
|
||||
def v = random(-8, 8) + 100;
|
||||
areaDingJiMarkup = v;
|
||||
}
|
||||
// else{
|
||||
// def v = random(-8, 8) + 100;
|
||||
// areaDingJiMarkup = v;
|
||||
// }
|
||||
|
||||
|
||||
if(areaMarkup){
|
||||
@@ -320,27 +353,56 @@ class ImportIndexStats extends BaseBatch{
|
||||
// def dbCursor = seawater.find(q, keys).batchSize(5000);
|
||||
// Iterator<BasicDBObject> it = dbCursor.iterator()
|
||||
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
println city +" : "+startDate+" "+endDate +" "+startArea+"-"+endArea
|
||||
def prices = [];
|
||||
while (rs.next()) {
|
||||
ElasticUtils.executeSqlQuery(sqlBuffer.toString(),{ rs->
|
||||
def area = (Double) rs.getObject("area")
|
||||
def totalPrice = (Integer) rs.getObject("total_price")
|
||||
if(!totalPrice){
|
||||
continue;
|
||||
}
|
||||
if(area <= 25 && area>=500){
|
||||
continue;
|
||||
}
|
||||
|
||||
def price = totalPrice.toInteger()/area.toDouble();
|
||||
if(price > 1000 && price < 200000){
|
||||
prices << price.toInteger()
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return prices;
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findHouse3(def city,def startDate, def endDate, def endArea){
|
||||
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT avg(avg_price) as avg_price FROM \"seawater-20*\" WHERE 1=1")
|
||||
sqlBuffer.append(" and city = '").append(city).append("'")
|
||||
sqlBuffer.append(" and cat = 'sell'")
|
||||
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <="+endDate);
|
||||
sqlBuffer.append(" and avg_price < 300000")
|
||||
if(endArea){
|
||||
if(endArea ==90){
|
||||
sqlBuffer.append(" and area>=25 and area<90 and total_price>0")
|
||||
}else if (endArea == 144){
|
||||
sqlBuffer.append(" and area>=90 and area<144 and total_price>0 ")
|
||||
}else if (endArea > 144){
|
||||
sqlBuffer.append(" and area>=144 and total_price>0 ")
|
||||
}
|
||||
}
|
||||
|
||||
def avg_price = null;
|
||||
try{
|
||||
ElasticUtils.executeSqlQuery(sqlBuffer.toString(),{ rs->
|
||||
avg_price = rs.avg_price
|
||||
})
|
||||
}catch(Exception e){
|
||||
avg_price = findHouse3(city,startDate, endDate, endArea);
|
||||
println e.getMessage()
|
||||
}
|
||||
if(avg_price){
|
||||
return (int)avg_price;
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -367,21 +429,14 @@ class ImportIndexStats extends BaseBatch{
|
||||
}
|
||||
sqlBuffer.append(" and cat = 'sell'")
|
||||
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <="+endDate);
|
||||
sqlBuffer.append(" ORDER BY p_date desc limit 5000")
|
||||
sqlBuffer.append(" and area>=25 and area<=500 and total_price>0 ORDER BY p_date desc limit 5000")
|
||||
|
||||
def priceMap = [:];
|
||||
try{
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
while (rs.next()) {
|
||||
def area = (Double) rs.getObject("area")
|
||||
def totalPrice = (Integer) rs.getObject("total_price")
|
||||
if(!totalPrice){
|
||||
continue;
|
||||
}
|
||||
if(area <= 25 && area>=500){
|
||||
continue;
|
||||
}
|
||||
|
||||
ElasticUtils.executeSqlQuery(sqlBuffer.toString(),{ rs->
|
||||
def area = rs.area
|
||||
def totalPrice = rs.total_price
|
||||
|
||||
def areaKey = "";
|
||||
if(area>0 && area<90){
|
||||
areaKey = "0_90";
|
||||
@@ -403,7 +458,7 @@ class ImportIndexStats extends BaseBatch{
|
||||
prices << price.toInteger()
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}catch(Exception e){
|
||||
findHouse2(city,startDate, endDate);
|
||||
}
|
||||
@@ -471,10 +526,11 @@ class ImportIndexStats extends BaseBatch{
|
||||
def findPriceIndexCitys(def date){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("yearmonth", date);
|
||||
|
||||
q.put("cat", "district")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
keys.put("city", 1)
|
||||
def cursor = on_line.find(q, keys);
|
||||
def cursor = STATS_INDEX_COLL.find(q, keys);
|
||||
def citys = [];
|
||||
for(def dbo : cursor){
|
||||
if(!citys.contains(dbo.city)){
|
||||
@@ -489,11 +545,11 @@ class ImportIndexStats extends BaseBatch{
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("yearmonth", date);
|
||||
q.put("city", city);
|
||||
q.put("cat", "district");
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
keys.put("city", 1)
|
||||
def cursor = on_line.find(q, keys);
|
||||
def citys = [];
|
||||
def cursor = STATS_INDEX_COLL.find(q, keys);
|
||||
for(def dbo : cursor){
|
||||
def id = dbo.get("_id");
|
||||
return id;
|
||||
@@ -510,13 +566,13 @@ class ImportIndexStats extends BaseBatch{
|
||||
dbo.put("cat",result.cat);
|
||||
dbo.put("c_date",new Date());
|
||||
if(result.markup){
|
||||
dbo.put("new_chain", result.markup.toDouble());
|
||||
dbo.put("chain", result.markup.toDouble());
|
||||
}
|
||||
if(result.yearMarkup){
|
||||
dbo.put("new_yeartoyear", result.yearMarkup.toDouble());
|
||||
dbo.put("yeartoyear", result.yearMarkup.toDouble());
|
||||
}
|
||||
if(result.dingJiMarkup){
|
||||
dbo.put("new_fixedbase", result.dingJiMarkup.toDouble());
|
||||
dbo.put("fixedbase", result.dingJiMarkup.toDouble());
|
||||
}
|
||||
|
||||
if(result.areaMarkup90){
|
||||
@@ -560,12 +616,151 @@ class ImportIndexStats extends BaseBatch{
|
||||
if(id){
|
||||
DBObject q = new BasicDBObject()
|
||||
q.put("_id", id)
|
||||
on_line.update(q, new BasicDBObject('$set', dbo));
|
||||
STATS_INDEX_COLL.update(q, new BasicDBObject('$set', dbo));
|
||||
}else{
|
||||
on_line.save(dbo);
|
||||
STATS_INDEX_COLL.save(dbo);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
def updateDataType(){
|
||||
def cursor = on_line.find(new BasicDBObject()).batchSize(5000)
|
||||
def index = 0
|
||||
for(def d : cursor){
|
||||
index++
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("yearmonth", d.yearmonth.toInteger())
|
||||
on_line.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
println index
|
||||
}
|
||||
}
|
||||
|
||||
def synch(){
|
||||
def cursor = on_line.find(new BasicDBObject()).batchSize(5000)
|
||||
for(def d : cursor){
|
||||
STATS_INDEX_COLL.save(d)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def synchTjjIndex(){
|
||||
def s = "202401"
|
||||
def startDate = null;
|
||||
def stop = false;
|
||||
while(!stop){
|
||||
startDate = DateUtils.parseFromTightYYYYMM(s);
|
||||
def year = startDate.getYear()+1900;
|
||||
def month = startDate.month+1;
|
||||
|
||||
def date = createDateString(year, month, 0, "");
|
||||
createTjjData(date)
|
||||
println "运行......"+date+".......保存完成"
|
||||
|
||||
s = DateUtils.addMonth(date, 1, "yyyyMM")
|
||||
|
||||
if(year==2024 && month==8){
|
||||
stop = true;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def createTjjData(date){
|
||||
def queryStartDate = DateUtils.parseFromTightByPattern(date+"01", "yyyyMMdd")
|
||||
def queryEndDate = DateUtils.parseFromTightByPattern(date+"05", "yyyyMMdd")
|
||||
def sellIndexMap = findTjjIndex(indices_price_house_sellColl, queryStartDate, queryEndDate)
|
||||
def sellIndexAreaMap = findTjjAreaIndex(indices_price_house_sell_typeColl, queryStartDate, queryEndDate)
|
||||
|
||||
def newIndexMap = findTjjIndex(indices_price_house_newColl, queryStartDate, queryEndDate)
|
||||
def newAreaIndexAreaMap = findTjjAreaIndex(indices_price_house_new_typeColl, queryStartDate, queryEndDate)
|
||||
def rets = []
|
||||
sellIndexMap.each{city, d->
|
||||
def dbo = new BasicDBObject()
|
||||
dbo.put("city",d.city)
|
||||
dbo.put("yearmonth",date.toInteger())
|
||||
dbo.put("source" ,1)
|
||||
dbo.put("remark" , "统计局数据导入")
|
||||
dbo.put("cat" , "district")
|
||||
dbo.put("c_date" , new Date())
|
||||
dbo.put("m_date" , new Date())
|
||||
dbo.put("type" , "城市")
|
||||
dbo.put("chain" ,d.mom)
|
||||
dbo.put("yeartoyear" , d.yoy)
|
||||
dbo.put("fixedbase" , d.fixedbase)
|
||||
|
||||
|
||||
def areaMap = sellIndexAreaMap.get(d.city)
|
||||
def areaObj1 = areaMap.get("90")
|
||||
dbo.put("below_chain" , areaObj1.mom)
|
||||
dbo.put("below_yeartoyear" , areaObj1.yoy)
|
||||
dbo.put("below_fixedbase" , areaObj1.fixedbase)
|
||||
|
||||
def areaObj2 = areaMap.get("90-144")
|
||||
dbo.put("among_chain" , areaObj2.mom)
|
||||
dbo.put("among_yeartoyear" , areaObj2.yoy)
|
||||
dbo.put("among_fixedbase" , areaObj2.fixedbase)
|
||||
|
||||
def areaObj3 = areaMap.get("144")
|
||||
dbo.put("above_chain" , areaObj3.mom)
|
||||
dbo.put("above_yeartoyear" , areaObj3.yoy)
|
||||
dbo.put("above_fixedbase" , areaObj3.fixedbase)
|
||||
|
||||
def newObj = newIndexMap.get(d.city)
|
||||
dbo.put("new_chain" , newObj.mom)
|
||||
dbo.put("new_yeartoyear" , newObj.yoy)
|
||||
dbo.put("new_fixedbase" , newObj.fixedbase)
|
||||
|
||||
def newAreaMap = newAreaIndexAreaMap.get(d.city)
|
||||
def areaNewObj1 = newAreaMap.get("90")
|
||||
dbo.put("new_below_chain" , areaNewObj1.mom)
|
||||
dbo.put("new_below_yeartoyear" , areaNewObj1.yoy)
|
||||
dbo.put("new_below_fixedbase" , areaNewObj1.fixedbase)
|
||||
|
||||
def areaNewObj2 = newAreaMap.get("90-144")
|
||||
dbo.put("new_among_chain" , areaNewObj2.mom)
|
||||
dbo.put("new_among_yeartoyear" , areaNewObj2.yoy)
|
||||
dbo.put("new_among_fixedbase" , areaNewObj2.fixedbase)
|
||||
|
||||
def areaNewObj3 = newAreaMap.get("144")
|
||||
dbo.put("new_above_chain" , areaNewObj3.mom)
|
||||
dbo.put("new_above_yeartoyear" , areaNewObj3.yoy)
|
||||
dbo.put("new_above_fixedbase" , areaNewObj3.fixedbase)
|
||||
|
||||
rets << dbo
|
||||
}
|
||||
STATS_INDEX_COLL.insert(rets)
|
||||
}
|
||||
|
||||
def findTjjIndex(coll, queryStartDate, queryEndDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("date", new BasicDBObject('$gte',queryStartDate).append('$lte', queryEndDate))
|
||||
def dbcursor = coll.find(q)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def o = map.get(d.city)
|
||||
if(!o){
|
||||
map.put(d.city,d)
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
def findTjjAreaIndex(coll,queryStartDate, queryEndDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("date", new BasicDBObject('$gte',queryStartDate).append('$lte', queryEndDate))
|
||||
def dbcursor = coll.find(q)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def areaMap = map.get(d.city)
|
||||
if(!areaMap){
|
||||
areaMap = [:]
|
||||
map.put(d.city, areaMap)
|
||||
}
|
||||
def o = areaMap.get(d.area)
|
||||
if(!o){
|
||||
areaMap.put(d.area, d)
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user