Merge branch 'script_gj' into 'master'
新增成交同步脚本 See merge request !1068
This commit is contained in:
+40
-28
@@ -1,6 +1,9 @@
|
|||||||
package com.fangjia.script.dealPrice
|
package com.fangjia.script.dealPrice
|
||||||
|
|
||||||
import com.fangjia.core.util.MongoUtils
|
import com.fangjia.core.util.MongoUtils
|
||||||
|
import com.fangjia.script.BaseBatch
|
||||||
|
import com.fangjia.script.BatchOwner
|
||||||
|
import com.fangjia.script.Watcher
|
||||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||||
import com.fangjia.script.util.DateUtils
|
import com.fangjia.script.util.DateUtils
|
||||||
import com.fangjia.script.util.ElasticUtils
|
import com.fangjia.script.util.ElasticUtils
|
||||||
@@ -15,43 +18,52 @@ import org.elasticsearch.action.support.WriteRequest
|
|||||||
import org.elasticsearch.client.RequestOptions
|
import org.elasticsearch.client.RequestOptions
|
||||||
import org.elasticsearch.common.xcontent.XContentType
|
import org.elasticsearch.common.xcontent.XContentType
|
||||||
|
|
||||||
class SynchDealPriceToEs {
|
/**
|
||||||
|
* 成交同步206,按日同步
|
||||||
|
*/
|
||||||
|
@BatchOwner(Watcher.GaoJuan)
|
||||||
|
class SynchDealPriceToEs extends BaseBatch{
|
||||||
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
|
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new SynchDealPriceToEs()
|
runBatch(new SynchDealPriceToEs(), args)
|
||||||
task.run()
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
void process(Object args) {
|
||||||
|
run();
|
||||||
}
|
}
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
def batchsize = 2000
|
def startDate = DateUtils.formatDateByPattern(new Date()-1, "yyyy-MM-dd")
|
||||||
def citys = getCityCount()
|
def q = new BasicDBObject()
|
||||||
citys.each{city,count->
|
q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
|
||||||
if(city instanceof String){
|
def dbcursor = dealPriceConn.find(q)
|
||||||
for(def year = 1999;year<=2025;year++){
|
println DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd")+" 更新量:"+ dbcursor.count()
|
||||||
def q = new BasicDBObject("city", city)
|
def ret = []
|
||||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern(year+"-01-01", "yyyy-MM-dd")).append('$lte', DateUtils.parseFromTightByPattern(year+1+"-01-01", "yyyy-MM-dd")))
|
for(def o : dbcursor){
|
||||||
def dbcursor = dealPriceConn.find(q).batchSize(batchsize)
|
ret << o
|
||||||
println city +" "+ dbcursor.count()
|
}
|
||||||
def ret = []
|
save(ret)
|
||||||
for(def o : dbcursor){
|
|
||||||
ret << o
|
|
||||||
}
|
|
||||||
save(ret)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// def index = 0
|
// def batchsize = 2000
|
||||||
// for(def d : dbcursor){
|
// def citys = getCityCount()
|
||||||
// ret << d
|
// citys.each{city,count->
|
||||||
// if(ret.size()>=2000){
|
// if(city instanceof String){
|
||||||
|
// for(def year = 1999;year<=2025;year++){
|
||||||
|
// def q = new BasicDBObject("city", city)
|
||||||
|
// q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern(year+"-01-01", "yyyy-MM-dd")).append('$lte', DateUtils.parseFromTightByPattern(year+1+"-01-01", "yyyy-MM-dd")))
|
||||||
|
// def dbcursor = dealPriceConn.find(q).batchSize(batchsize)
|
||||||
|
// println city +" "+ dbcursor.count()
|
||||||
|
// def ret = []
|
||||||
|
// for(def o : dbcursor){
|
||||||
|
// ret << o
|
||||||
|
// }
|
||||||
// save(ret)
|
// save(ret)
|
||||||
// ret.clear()
|
|
||||||
// index ++
|
|
||||||
// println index
|
|
||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
}
|
// }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -93,7 +105,7 @@ class SynchDealPriceToEs {
|
|||||||
// def map = rows.get(key)
|
// def map = rows.get(key)
|
||||||
def data_id = map._id.toString()
|
def data_id = map._id.toString()
|
||||||
map.remove("_id")
|
map.remove("_id")
|
||||||
request.add(new IndexRequest("deal_price", "_doc", data_id).source(map, XContentType.JSON));
|
request.add(new IndexRequest("trades", "_doc", data_id).source(map, XContentType.JSON));
|
||||||
|
|
||||||
// 批量提交数据
|
// 批量提交数据
|
||||||
if (request.numberOfActions() >= batchSize) {
|
if (request.numberOfActions() >= batchSize) {
|
||||||
|
|||||||
+32
-32
@@ -67,39 +67,39 @@ class DistrictFieldAddBatch extends BaseBatch{
|
|||||||
]
|
]
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
// runBatch(new DistrictFieldAddBatch(), args)
|
runBatch(new DistrictFieldAddBatch(), args)
|
||||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\上海别墅.xlsx",{d->
|
// ExcelUtils.multiThreadingModify("D:\\数据\\小区\\上海别墅.xlsx",{d->
|
||||||
def dist = CommonDatautils.findDistrict(d['B'], d['G'], d['C'], "district")
|
// def dist = CommonDatautils.findDistrict(d['B'], d['G'], d['C'], "district")
|
||||||
// if(dist){
|
//// if(dist){
|
||||||
// d['L'] = dist.loudong_count
|
//// d['L'] = dist.loudong_count
|
||||||
// d['M'] = dist.household_count
|
//// d['M'] = dist.household_count
|
||||||
// d['Q'] = dist.estate_type2
|
//// d['Q'] = dist.estate_type2
|
||||||
|
//// }
|
||||||
|
// def list = findEsDatasByCityAndName(d['B'], d['G'], d['C'], "搜房")
|
||||||
|
// if(!list && dist){
|
||||||
|
// def alias = dist.alias
|
||||||
|
// for(def name : alias){
|
||||||
|
// list = findEsDatasByCityAndName(d['B'], d['G'], name, "搜房")
|
||||||
|
// if(list){
|
||||||
|
// d['Z'] = "别名"
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }else{
|
||||||
|
// d['Z'] = "名称"
|
||||||
// }
|
// }
|
||||||
def list = findEsDatasByCityAndName(d['B'], d['G'], d['C'], "搜房")
|
// if(list){
|
||||||
if(!list && dist){
|
// def obj = list[0]
|
||||||
def alias = dist.alias
|
// if(obj.loudong_cnt){
|
||||||
for(def name : alias){
|
// d['W'] = obj.loudong_cnt
|
||||||
list = findEsDatasByCityAndName(d['B'], d['G'], name, "搜房")
|
// }
|
||||||
if(list){
|
//
|
||||||
d['Z'] = "别名"
|
// if(obj.households){
|
||||||
}
|
// d['X'] = obj.households
|
||||||
}
|
// }
|
||||||
}else{
|
// d['Y'] = obj.url
|
||||||
d['Z'] = "名称"
|
// }
|
||||||
}
|
//
|
||||||
if(list){
|
// })
|
||||||
def obj = list[0]
|
|
||||||
if(obj.loudong_cnt){
|
|
||||||
d['W'] = obj.loudong_cnt
|
|
||||||
}
|
|
||||||
|
|
||||||
if(obj.households){
|
|
||||||
d['X'] = obj.households
|
|
||||||
}
|
|
||||||
d['Y'] = obj.url
|
|
||||||
}
|
|
||||||
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|||||||
Reference in New Issue
Block a user