Merge branch 'script_gj' into 'master'

新增成交同步脚本

See merge request !1068
This commit is contained in:
2025-01-16 18:02:18 +08:00
2 changed files with 72 additions and 60 deletions
@@ -1,6 +1,9 @@
package com.fangjia.script.dealPrice package com.fangjia.script.dealPrice
import com.fangjia.core.util.MongoUtils import com.fangjia.core.util.MongoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.gj.xiamen.ElasticUtils89 import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
@@ -15,43 +18,52 @@ import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.common.xcontent.XContentType
class SynchDealPriceToEs { /**
* 成交同步206,按日同步
*/
@BatchOwner(Watcher.GaoJuan)
class SynchDealPriceToEs extends BaseBatch{
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017) static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
static main(args) { static main(args) {
def task = new SynchDealPriceToEs() runBatch(new SynchDealPriceToEs(), args)
task.run() }
@Override
void process(Object args) {
run();
} }
def run(){ def run(){
def batchsize = 2000 def startDate = DateUtils.formatDateByPattern(new Date()-1, "yyyy-MM-dd")
def citys = getCityCount() def q = new BasicDBObject()
citys.each{city,count-> q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
if(city instanceof String){ def dbcursor = dealPriceConn.find(q)
for(def year = 1999;year<=2025;year++){ println DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd")+" 更新量:"+ dbcursor.count()
def q = new BasicDBObject("city", city) def ret = []
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern(year+"-01-01", "yyyy-MM-dd")).append('$lte', DateUtils.parseFromTightByPattern(year+1+"-01-01", "yyyy-MM-dd"))) for(def o : dbcursor){
def dbcursor = dealPriceConn.find(q).batchSize(batchsize) ret << o
println city +" "+ dbcursor.count() }
def ret = [] save(ret)
for(def o : dbcursor){
ret << o
}
save(ret)
}
}
// def index = 0 // def batchsize = 2000
// for(def d : dbcursor){ // def citys = getCityCount()
// ret << d // citys.each{city,count->
// if(ret.size()>=2000){ // if(city instanceof String){
// for(def year = 1999;year<=2025;year++){
// def q = new BasicDBObject("city", city)
// q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern(year+"-01-01", "yyyy-MM-dd")).append('$lte', DateUtils.parseFromTightByPattern(year+1+"-01-01", "yyyy-MM-dd")))
// def dbcursor = dealPriceConn.find(q).batchSize(batchsize)
// println city +" "+ dbcursor.count()
// def ret = []
// for(def o : dbcursor){
// ret << o
// }
// save(ret) // save(ret)
// ret.clear()
// index ++
// println index
// } // }
// } // }
} // }
} }
@@ -93,7 +105,7 @@ class SynchDealPriceToEs {
// def map = rows.get(key) // def map = rows.get(key)
def data_id = map._id.toString() def data_id = map._id.toString()
map.remove("_id") map.remove("_id")
request.add(new IndexRequest("deal_price", "_doc", data_id).source(map, XContentType.JSON)); request.add(new IndexRequest("trades", "_doc", data_id).source(map, XContentType.JSON));
// 批量提交数据 // 批量提交数据
if (request.numberOfActions() >= batchSize) { if (request.numberOfActions() >= batchSize) {
@@ -67,39 +67,39 @@ class DistrictFieldAddBatch extends BaseBatch{
] ]
static main(args) { static main(args) {
// runBatch(new DistrictFieldAddBatch(), args) runBatch(new DistrictFieldAddBatch(), args)
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\上海别墅.xlsx",{d-> // ExcelUtils.multiThreadingModify("D:\\数据\\小区\\上海别墅.xlsx",{d->
def dist = CommonDatautils.findDistrict(d['B'], d['G'], d['C'], "district") // def dist = CommonDatautils.findDistrict(d['B'], d['G'], d['C'], "district")
// if(dist){ //// if(dist){
// d['L'] = dist.loudong_count //// d['L'] = dist.loudong_count
// d['M'] = dist.household_count //// d['M'] = dist.household_count
// d['Q'] = dist.estate_type2 //// d['Q'] = dist.estate_type2
//// }
// def list = findEsDatasByCityAndName(d['B'], d['G'], d['C'], "搜房")
// if(!list && dist){
// def alias = dist.alias
// for(def name : alias){
// list = findEsDatasByCityAndName(d['B'], d['G'], name, "搜房")
// if(list){
// d['Z'] = "别名"
// }
// }
// }else{
// d['Z'] = "名称"
// } // }
def list = findEsDatasByCityAndName(d['B'], d['G'], d['C'], "搜房") // if(list){
if(!list && dist){ // def obj = list[0]
def alias = dist.alias // if(obj.loudong_cnt){
for(def name : alias){ // d['W'] = obj.loudong_cnt
list = findEsDatasByCityAndName(d['B'], d['G'], name, "搜房") // }
if(list){ //
d['Z'] = "别名" // if(obj.households){
} // d['X'] = obj.households
} // }
}else{ // d['Y'] = obj.url
d['Z'] = "名称" // }
} //
if(list){ // })
def obj = list[0]
if(obj.loudong_cnt){
d['W'] = obj.loudong_cnt
}
if(obj.households){
d['X'] = obj.households
}
d['Y'] = obj.url
}
})
} }
@Override @Override