This commit is contained in:
尹吉欢
2018-09-18 20:51:25 +08:00
parent 5ef6b528c8
commit 099f819ea8
8 changed files with 99 additions and 18 deletions
@@ -17,10 +17,13 @@ public class SpringEsConfig {
@Value("${es.url}")
private String url;
@Value("${es.cluster.name}")
private String clusterName;
@SuppressWarnings("resource")
@Bean
public TransportClient transportClient() {
Settings settings = Settings.builder().put("cluster.name", "fangjia")
Settings settings = Settings.builder().put("cluster.name", clusterName)
.put("client.transport.ping_timeout", "60s").build();
try {
return new PreBuiltTransportClient(settings)
@@ -0,0 +1,64 @@
package com.fangjia.job.basicdata.job;
import java.util.concurrent.atomic.AtomicInteger;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import com.dangdang.ddframe.job.api.ShardingContext;
import com.dangdang.ddframe.job.api.simple.SimpleJob;
import com.fangjia.job.common.util.JsonUtils;
import com.mongodb.Bytes;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
/**
* 拍卖数据同步任务
* @author yinjihuan
*
*/
public class AuctionDataSyncEsJob implements SimpleJob {
private Logger logger = LoggerFactory.getLogger(AuctionDataSyncEsJob.class);
@Autowired
@Qualifier("businessMongoTemplate")
private MongoTemplate businessMongoTemplate;
@Autowired
private TransportClient transportClient;
@Override
public void execute(ShardingContext shardingContext) {
DBCollection collection = businessMongoTemplate.getCollection("auction_data");
DBCursor cursor = collection.find().batchSize(1000).addOption(Bytes.QUERYOPTION_NOTIMEOUT);
BulkRequestBuilder bulkRequest = transportClient.prepareBulk();
int index = 0;
AtomicInteger ac = new AtomicInteger();
while (cursor.hasNext()) {
DBObject data = cursor.next();
ac.incrementAndGet();
String id = data.get("_id").toString();
data.removeField("_id");
bulkRequest.add(transportClient.prepareIndex("auction_data","doc")
.setSource(JsonUtils.toJson(data), XContentType.JSON)
.setId(id));
index++;
if (index % 10000 == 0) {
bulkRequest.execute().actionGet();
logger.info("已导入:" + index);
// 清空数据
bulkRequest = transportClient.prepareBulk();
}
}
bulkRequest.execute().actionGet();
logger.info("导入结束,共导入:" + ac.addAndGet(0));
}
}
@@ -38,7 +38,7 @@ public class EsfDealPriceSyncEsJob implements SimpleJob {
@Override
public void execute(ShardingContext shardingContext) {
DBCollection collection = businessMongoTemplate.getCollection("esf_deal_price_fj");
DBCollection collection = businessMongoTemplate.getCollection("res_esfdealprice");
DBCursor cursor = collection.find().batchSize(1000).addOption(Bytes.QUERYOPTION_NOTIMEOUT);
BulkRequestBuilder bulkRequest = transportClient.prepareBulk();
int index = 0;
@@ -48,11 +48,11 @@ public class EsfDealPriceSyncEsJob implements SimpleJob {
ac.incrementAndGet();
String id = data.get("_id").toString();
data.removeField("_id");
bulkRequest.add(transportClient.prepareIndex("esf_deal_price","doc")
bulkRequest.add(transportClient.prepareIndex("esf_deal_price_ls","doc")
.setSource(JsonUtils.toJson(data), XContentType.JSON)
.setId(id));
index++;
if (index % 50000 == 0) {
if (index % 10000 == 0) {
bulkRequest.execute().actionGet();
logger.info("已导入:" + index);
// 清空数据
@@ -28,7 +28,7 @@ import com.mongodb.DBObject;
*/
public class NewDealPriceSyncEsJob implements SimpleJob {
private Logger logger = LoggerFactory.getLogger(EsfDealPriceSyncEsJob.class);
private Logger logger = LoggerFactory.getLogger(NewDealPriceSyncEsJob.class);
@Autowired
@Qualifier("businessMongoTemplate")
@@ -39,7 +39,7 @@ public class NewDealPriceSyncEsJob implements SimpleJob {
@Override
public void execute(ShardingContext shardingContext) {
DBCollection collection = businessMongoTemplate.getCollection("new_deal_price_fj");
DBCollection collection = businessMongoTemplate.getCollection("res_newdealprice");
DBCursor cursor = collection.find().batchSize(500).addOption(Bytes.QUERYOPTION_NOTIMEOUT);
BulkRequestBuilder bulkRequest = transportClient.prepareBulk();
int index = 0;
@@ -49,7 +49,7 @@ public class NewDealPriceSyncEsJob implements SimpleJob {
ac.incrementAndGet();
String id = data.get("_id").toString();
data.removeField("_id");
bulkRequest.add(transportClient.prepareIndex("new_deal_price","doc")
bulkRequest.add(transportClient.prepareIndex("new_deal_price_ls","doc")
.setSource(JsonUtils.toJson(data), XContentType.JSON)
.setId(id));
index++;
@@ -1,13 +1,14 @@
mongo.business.url=192.168.0.136
mongo.business.url=114.80.150.198
mongo.business.name=business
mongo.business.port=27017
spring.datasource.log.url=jdbc:mysql://192.168.0.136:3306/batch_log
spring.datasource.log.username=sa
spring.datasource.log.password=G00jia
spring.datasource.log.url=jdbc:mysql://localhost:3306/batch_log
spring.datasource.log.username=root
spring.datasource.log.password=123456
spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver
zk.server-lists=192.168.10.124:2181
zk.namespace=fangjia-job-basicdata
es.url=192.168.0.45
es.cluster.name=fangjia
@@ -11,3 +11,4 @@ zk.server-lists=192.168.0.207:2181,192.168.0.24:2181,192.168.0.25:2181
zk.namespace=fangjia-job-basicdata
es.url=192.168.0.45
es.cluster.name=fangjia
@@ -11,3 +11,4 @@ zk.server-lists=192.168.10.124:2181
zk.namespace=fangjia-job-basicdata
es.url=192.168.0.45
es.cluster.name=fangjia
@@ -27,12 +27,23 @@
base-sleep-time-milliseconds="1000" max-sleep-time-milliseconds="3000" max-retries="3" />
<!-- 企业统计任务 每天0点1分执行 -->
<job:simple id="esfDealPriceSyncEsJob" class="com.fangjia.job.basicdata.job.EsfDealPriceSyncEsJob" registry-center-ref="regCenter"
<!-- 一手成交数据导入ES任务 -->
<!-- <job:simple id="newDealPriceSyncEsJob" class="com.fangjia.job.basicdata.job.NewDealPriceSyncEsJob" registry-center-ref="regCenter"
sharding-total-count="1" cron="0/10 * * * * ?" sharding-item-parameters=""
failover="true" description="【基础数据处理任务】企业统计任务,每天0点1分开始执行,主要统计企业相关的数据,统计内容参考 EnterpriseStat"
overwrite="true" event-trace-rdb-data-source="elasticJobLog">
failover="true" description="【基础数据处理任务】一手成交数据导入ES任务"
overwrite="true" event-trace-rdb-data-source="elasticJobLog"> -->
<!-- 二手成交数据导入ES任务 -->
<!-- <job:simple id="esfDealPriceSyncEsJob" class="com.fangjia.job.basicdata.job.EsfDealPriceSyncEsJob" registry-center-ref="regCenter"
sharding-total-count="1" cron="0/10 * * * * ?" sharding-item-parameters=""
failover="true" description="【基础数据处理任务】二手成交数据导入ES任务"
overwrite="true" event-trace-rdb-data-source="elasticJobLog"> -->
<!-- 拍卖数据导入ES任务 -->
<!-- <job:simple id="auctionDataSyncEsJob" class="com.fangjia.job.basicdata.job.AuctionDataSyncEsJob" registry-center-ref="regCenter"
sharding-total-count="1" cron="0/10 * * * * ?" sharding-item-parameters=""
failover="true" description="【基础数据处理任务】拍卖数据导入ES任务"
overwrite="true" event-trace-rdb-data-source="elasticJobLog"> -->
</job:simple>