diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/SynchDealPriceToEs.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/SynchDealPriceToEs.groovy index f9f518cb8..dba4650b9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/SynchDealPriceToEs.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/SynchDealPriceToEs.groovy @@ -1,6 +1,9 @@ package com.fangjia.script.dealPrice import com.fangjia.core.util.MongoUtils +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher import com.fangjia.script.gj.xiamen.ElasticUtils89 import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils @@ -15,43 +18,52 @@ import org.elasticsearch.action.support.WriteRequest import org.elasticsearch.client.RequestOptions import org.elasticsearch.common.xcontent.XContentType -class SynchDealPriceToEs { +/** + * 成交同步206,按日同步 + */ +@BatchOwner(Watcher.GaoJuan) +class SynchDealPriceToEs extends BaseBatch{ static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017) static main(args) { - def task = new SynchDealPriceToEs() - task.run() + runBatch(new SynchDealPriceToEs(), args) + } + + + @Override + void process(Object args) { + run(); } def run(){ - def batchsize = 2000 - def citys = getCityCount() - citys.each{city,count-> - if(city instanceof String){ - for(def year = 1999;year<=2025;year++){ - def q = new BasicDBObject("city", city) - q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern(year+"-01-01", "yyyy-MM-dd")).append('$lte', DateUtils.parseFromTightByPattern(year+1+"-01-01", "yyyy-MM-dd"))) - def dbcursor = dealPriceConn.find(q).batchSize(batchsize) - println city +" "+ dbcursor.count() - def ret = [] - for(def o : dbcursor){ - ret << o - } - save(ret) - } - } + def startDate = DateUtils.formatDateByPattern(new Date()-1, "yyyy-MM-dd") + def q = new BasicDBObject() + q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate))) + def dbcursor = dealPriceConn.find(q) + println DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd")+" 更新量:"+ dbcursor.count() + def ret = [] + for(def o : dbcursor){ + ret << o + } + save(ret) -// def index = 0 -// for(def d : dbcursor){ -// ret << d -// if(ret.size()>=2000){ +// def batchsize = 2000 +// def citys = getCityCount() +// citys.each{city,count-> +// if(city instanceof String){ +// for(def year = 1999;year<=2025;year++){ +// def q = new BasicDBObject("city", city) +// q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern(year+"-01-01", "yyyy-MM-dd")).append('$lte', DateUtils.parseFromTightByPattern(year+1+"-01-01", "yyyy-MM-dd"))) +// def dbcursor = dealPriceConn.find(q).batchSize(batchsize) +// println city +" "+ dbcursor.count() +// def ret = [] +// for(def o : dbcursor){ +// ret << o +// } // save(ret) -// ret.clear() -// index ++ -// println index // } // } - } +// } } @@ -93,7 +105,7 @@ class SynchDealPriceToEs { // def map = rows.get(key) def data_id = map._id.toString() map.remove("_id") - request.add(new IndexRequest("deal_price", "_doc", data_id).source(map, XContentType.JSON)); + request.add(new IndexRequest("trades", "_doc", data_id).source(map, XContentType.JSON)); // 批量提交数据 if (request.numberOfActions() >= batchSize) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index d6e73b168..e2625366c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -67,39 +67,39 @@ class DistrictFieldAddBatch extends BaseBatch{ ] static main(args) { -// runBatch(new DistrictFieldAddBatch(), args) - ExcelUtils.multiThreadingModify("D:\\数据\\小区\\上海别墅.xlsx",{d-> - def dist = CommonDatautils.findDistrict(d['B'], d['G'], d['C'], "district") -// if(dist){ -// d['L'] = dist.loudong_count -// d['M'] = dist.household_count -// d['Q'] = dist.estate_type2 + runBatch(new DistrictFieldAddBatch(), args) +// ExcelUtils.multiThreadingModify("D:\\数据\\小区\\上海别墅.xlsx",{d-> +// def dist = CommonDatautils.findDistrict(d['B'], d['G'], d['C'], "district") +//// if(dist){ +//// d['L'] = dist.loudong_count +//// d['M'] = dist.household_count +//// d['Q'] = dist.estate_type2 +//// } +// def list = findEsDatasByCityAndName(d['B'], d['G'], d['C'], "搜房") +// if(!list && dist){ +// def alias = dist.alias +// for(def name : alias){ +// list = findEsDatasByCityAndName(d['B'], d['G'], name, "搜房") +// if(list){ +// d['Z'] = "别名" +// } +// } +// }else{ +// d['Z'] = "名称" // } - def list = findEsDatasByCityAndName(d['B'], d['G'], d['C'], "搜房") - if(!list && dist){ - def alias = dist.alias - for(def name : alias){ - list = findEsDatasByCityAndName(d['B'], d['G'], name, "搜房") - if(list){ - d['Z'] = "别名" - } - } - }else{ - d['Z'] = "名称" - } - if(list){ - def obj = list[0] - if(obj.loudong_cnt){ - d['W'] = obj.loudong_cnt - } - - if(obj.households){ - d['X'] = obj.households - } - d['Y'] = obj.url - } - - }) +// if(list){ +// def obj = list[0] +// if(obj.loudong_cnt){ +// d['W'] = obj.loudong_cnt +// } +// +// if(obj.households){ +// d['X'] = obj.households +// } +// d['Y'] = obj.url +// } +// +// }) } @Override