新增成交同步脚本

This commit is contained in:
2025-01-16 17:59:59 +08:00
parent 0153072098
commit 82ddcbc4ed
2 changed files with 72 additions and 60 deletions
@@ -1,6 +1,9 @@
package com.fangjia.script.dealPrice
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
@@ -15,43 +18,52 @@ import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
class SynchDealPriceToEs {
/**
* 成交同步206,按日同步
*/
@BatchOwner(Watcher.GaoJuan)
class SynchDealPriceToEs extends BaseBatch{
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
static main(args) {
def task = new SynchDealPriceToEs()
task.run()
runBatch(new SynchDealPriceToEs(), args)
}
@Override
void process(Object args) {
run();
}
def run(){
def batchsize = 2000
def citys = getCityCount()
citys.each{city,count->
if(city instanceof String){
for(def year = 1999;year<=2025;year++){
def q = new BasicDBObject("city", city)
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern(year+"-01-01", "yyyy-MM-dd")).append('$lte', DateUtils.parseFromTightByPattern(year+1+"-01-01", "yyyy-MM-dd")))
def dbcursor = dealPriceConn.find(q).batchSize(batchsize)
println city +" "+ dbcursor.count()
def ret = []
for(def o : dbcursor){
ret << o
}
save(ret)
}
}
def startDate = DateUtils.formatDateByPattern(new Date()-1, "yyyy-MM-dd")
def q = new BasicDBObject()
q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
def dbcursor = dealPriceConn.find(q)
println DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd")+" 更新量:"+ dbcursor.count()
def ret = []
for(def o : dbcursor){
ret << o
}
save(ret)
// def index = 0
// for(def d : dbcursor){
// ret << d
// if(ret.size()>=2000){
// def batchsize = 2000
// def citys = getCityCount()
// citys.each{city,count->
// if(city instanceof String){
// for(def year = 1999;year<=2025;year++){
// def q = new BasicDBObject("city", city)
// q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern(year+"-01-01", "yyyy-MM-dd")).append('$lte', DateUtils.parseFromTightByPattern(year+1+"-01-01", "yyyy-MM-dd")))
// def dbcursor = dealPriceConn.find(q).batchSize(batchsize)
// println city +" "+ dbcursor.count()
// def ret = []
// for(def o : dbcursor){
// ret << o
// }
// save(ret)
// ret.clear()
// index ++
// println index
// }
// }
}
// }
}
@@ -93,7 +105,7 @@ class SynchDealPriceToEs {
// def map = rows.get(key)
def data_id = map._id.toString()
map.remove("_id")
request.add(new IndexRequest("deal_price", "_doc", data_id).source(map, XContentType.JSON));
request.add(new IndexRequest("trades", "_doc", data_id).source(map, XContentType.JSON));
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
@@ -67,39 +67,39 @@ class DistrictFieldAddBatch extends BaseBatch{
]
static main(args) {
// runBatch(new DistrictFieldAddBatch(), args)
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\上海别墅.xlsx",{d->
def dist = CommonDatautils.findDistrict(d['B'], d['G'], d['C'], "district")
// if(dist){
// d['L'] = dist.loudong_count
// d['M'] = dist.household_count
// d['Q'] = dist.estate_type2
runBatch(new DistrictFieldAddBatch(), args)
// ExcelUtils.multiThreadingModify("D:\\数据\\小区\\上海别墅.xlsx",{d->
// def dist = CommonDatautils.findDistrict(d['B'], d['G'], d['C'], "district")
//// if(dist){
//// d['L'] = dist.loudong_count
//// d['M'] = dist.household_count
//// d['Q'] = dist.estate_type2
//// }
// def list = findEsDatasByCityAndName(d['B'], d['G'], d['C'], "搜房")
// if(!list && dist){
// def alias = dist.alias
// for(def name : alias){
// list = findEsDatasByCityAndName(d['B'], d['G'], name, "搜房")
// if(list){
// d['Z'] = "别名"
// }
// }
// }else{
// d['Z'] = "名称"
// }
def list = findEsDatasByCityAndName(d['B'], d['G'], d['C'], "搜房")
if(!list && dist){
def alias = dist.alias
for(def name : alias){
list = findEsDatasByCityAndName(d['B'], d['G'], name, "搜房")
if(list){
d['Z'] = "别名"
}
}
}else{
d['Z'] = "名称"
}
if(list){
def obj = list[0]
if(obj.loudong_cnt){
d['W'] = obj.loudong_cnt
}
if(obj.households){
d['X'] = obj.households
}
d['Y'] = obj.url
}
})
// if(list){
// def obj = list[0]
// if(obj.loudong_cnt){
// d['W'] = obj.loudong_cnt
// }
//
// if(obj.households){
// d['X'] = obj.households
// }
// d['Y'] = obj.url
// }
//
// })
}
@Override