新增成交同步脚本
This commit is contained in:
+40
-28
@@ -1,6 +1,9 @@
|
||||
package com.fangjia.script.dealPrice
|
||||
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
@@ -15,43 +18,52 @@ import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
class SynchDealPriceToEs {
|
||||
/**
|
||||
* 成交同步206,按日同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class SynchDealPriceToEs extends BaseBatch{
|
||||
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
|
||||
|
||||
static main(args) {
|
||||
def task = new SynchDealPriceToEs()
|
||||
task.run()
|
||||
runBatch(new SynchDealPriceToEs(), args)
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def batchsize = 2000
|
||||
def citys = getCityCount()
|
||||
citys.each{city,count->
|
||||
if(city instanceof String){
|
||||
for(def year = 1999;year<=2025;year++){
|
||||
def q = new BasicDBObject("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern(year+"-01-01", "yyyy-MM-dd")).append('$lte', DateUtils.parseFromTightByPattern(year+1+"-01-01", "yyyy-MM-dd")))
|
||||
def dbcursor = dealPriceConn.find(q).batchSize(batchsize)
|
||||
println city +" "+ dbcursor.count()
|
||||
def ret = []
|
||||
for(def o : dbcursor){
|
||||
ret << o
|
||||
}
|
||||
save(ret)
|
||||
}
|
||||
}
|
||||
def startDate = DateUtils.formatDateByPattern(new Date()-1, "yyyy-MM-dd")
|
||||
def q = new BasicDBObject()
|
||||
q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
|
||||
def dbcursor = dealPriceConn.find(q)
|
||||
println DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd")+" 更新量:"+ dbcursor.count()
|
||||
def ret = []
|
||||
for(def o : dbcursor){
|
||||
ret << o
|
||||
}
|
||||
save(ret)
|
||||
|
||||
// def index = 0
|
||||
// for(def d : dbcursor){
|
||||
// ret << d
|
||||
// if(ret.size()>=2000){
|
||||
// def batchsize = 2000
|
||||
// def citys = getCityCount()
|
||||
// citys.each{city,count->
|
||||
// if(city instanceof String){
|
||||
// for(def year = 1999;year<=2025;year++){
|
||||
// def q = new BasicDBObject("city", city)
|
||||
// q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern(year+"-01-01", "yyyy-MM-dd")).append('$lte', DateUtils.parseFromTightByPattern(year+1+"-01-01", "yyyy-MM-dd")))
|
||||
// def dbcursor = dealPriceConn.find(q).batchSize(batchsize)
|
||||
// println city +" "+ dbcursor.count()
|
||||
// def ret = []
|
||||
// for(def o : dbcursor){
|
||||
// ret << o
|
||||
// }
|
||||
// save(ret)
|
||||
// ret.clear()
|
||||
// index ++
|
||||
// println index
|
||||
// }
|
||||
// }
|
||||
}
|
||||
// }
|
||||
}
|
||||
|
||||
|
||||
@@ -93,7 +105,7 @@ class SynchDealPriceToEs {
|
||||
// def map = rows.get(key)
|
||||
def data_id = map._id.toString()
|
||||
map.remove("_id")
|
||||
request.add(new IndexRequest("deal_price", "_doc", data_id).source(map, XContentType.JSON));
|
||||
request.add(new IndexRequest("trades", "_doc", data_id).source(map, XContentType.JSON));
|
||||
|
||||
// 批量提交数据
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
|
||||
+32
-32
@@ -67,39 +67,39 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
]
|
||||
|
||||
static main(args) {
|
||||
// runBatch(new DistrictFieldAddBatch(), args)
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\上海别墅.xlsx",{d->
|
||||
def dist = CommonDatautils.findDistrict(d['B'], d['G'], d['C'], "district")
|
||||
// if(dist){
|
||||
// d['L'] = dist.loudong_count
|
||||
// d['M'] = dist.household_count
|
||||
// d['Q'] = dist.estate_type2
|
||||
runBatch(new DistrictFieldAddBatch(), args)
|
||||
// ExcelUtils.multiThreadingModify("D:\\数据\\小区\\上海别墅.xlsx",{d->
|
||||
// def dist = CommonDatautils.findDistrict(d['B'], d['G'], d['C'], "district")
|
||||
//// if(dist){
|
||||
//// d['L'] = dist.loudong_count
|
||||
//// d['M'] = dist.household_count
|
||||
//// d['Q'] = dist.estate_type2
|
||||
//// }
|
||||
// def list = findEsDatasByCityAndName(d['B'], d['G'], d['C'], "搜房")
|
||||
// if(!list && dist){
|
||||
// def alias = dist.alias
|
||||
// for(def name : alias){
|
||||
// list = findEsDatasByCityAndName(d['B'], d['G'], name, "搜房")
|
||||
// if(list){
|
||||
// d['Z'] = "别名"
|
||||
// }
|
||||
// }
|
||||
// }else{
|
||||
// d['Z'] = "名称"
|
||||
// }
|
||||
def list = findEsDatasByCityAndName(d['B'], d['G'], d['C'], "搜房")
|
||||
if(!list && dist){
|
||||
def alias = dist.alias
|
||||
for(def name : alias){
|
||||
list = findEsDatasByCityAndName(d['B'], d['G'], name, "搜房")
|
||||
if(list){
|
||||
d['Z'] = "别名"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
d['Z'] = "名称"
|
||||
}
|
||||
if(list){
|
||||
def obj = list[0]
|
||||
if(obj.loudong_cnt){
|
||||
d['W'] = obj.loudong_cnt
|
||||
}
|
||||
|
||||
if(obj.households){
|
||||
d['X'] = obj.households
|
||||
}
|
||||
d['Y'] = obj.url
|
||||
}
|
||||
|
||||
})
|
||||
// if(list){
|
||||
// def obj = list[0]
|
||||
// if(obj.loudong_cnt){
|
||||
// d['W'] = obj.loudong_cnt
|
||||
// }
|
||||
//
|
||||
// if(obj.households){
|
||||
// d['X'] = obj.households
|
||||
// }
|
||||
// d['Y'] = obj.url
|
||||
// }
|
||||
//
|
||||
// })
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
Reference in New Issue
Block a user