add
This commit is contained in:
+4
-1
@@ -17,10 +17,13 @@ public class SpringEsConfig {
|
||||
@Value("${es.url}")
|
||||
private String url;
|
||||
|
||||
@Value("${es.cluster.name}")
|
||||
private String clusterName;
|
||||
|
||||
@SuppressWarnings("resource")
|
||||
@Bean
|
||||
public TransportClient transportClient() {
|
||||
Settings settings = Settings.builder().put("cluster.name", "fangjia")
|
||||
Settings settings = Settings.builder().put("cluster.name", clusterName)
|
||||
.put("client.transport.ping_timeout", "60s").build();
|
||||
try {
|
||||
return new PreBuiltTransportClient(settings)
|
||||
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
package com.fangjia.job.basicdata.job;
|
||||
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import org.elasticsearch.action.bulk.BulkRequestBuilder;
|
||||
import org.elasticsearch.client.transport.TransportClient;
|
||||
import org.elasticsearch.common.xcontent.XContentType;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.data.mongodb.core.MongoTemplate;
|
||||
|
||||
import com.dangdang.ddframe.job.api.ShardingContext;
|
||||
import com.dangdang.ddframe.job.api.simple.SimpleJob;
|
||||
import com.fangjia.job.common.util.JsonUtils;
|
||||
import com.mongodb.Bytes;
|
||||
import com.mongodb.DBCollection;
|
||||
import com.mongodb.DBCursor;
|
||||
import com.mongodb.DBObject;
|
||||
/**
|
||||
* 拍卖数据同步任务
|
||||
* @author yinjihuan
|
||||
*
|
||||
*/
|
||||
public class AuctionDataSyncEsJob implements SimpleJob {
|
||||
|
||||
private Logger logger = LoggerFactory.getLogger(AuctionDataSyncEsJob.class);
|
||||
|
||||
@Autowired
|
||||
@Qualifier("businessMongoTemplate")
|
||||
private MongoTemplate businessMongoTemplate;
|
||||
|
||||
@Autowired
|
||||
private TransportClient transportClient;
|
||||
|
||||
@Override
|
||||
public void execute(ShardingContext shardingContext) {
|
||||
DBCollection collection = businessMongoTemplate.getCollection("auction_data");
|
||||
DBCursor cursor = collection.find().batchSize(1000).addOption(Bytes.QUERYOPTION_NOTIMEOUT);
|
||||
BulkRequestBuilder bulkRequest = transportClient.prepareBulk();
|
||||
int index = 0;
|
||||
AtomicInteger ac = new AtomicInteger();
|
||||
while (cursor.hasNext()) {
|
||||
DBObject data = cursor.next();
|
||||
ac.incrementAndGet();
|
||||
String id = data.get("_id").toString();
|
||||
data.removeField("_id");
|
||||
bulkRequest.add(transportClient.prepareIndex("auction_data","doc")
|
||||
.setSource(JsonUtils.toJson(data), XContentType.JSON)
|
||||
.setId(id));
|
||||
index++;
|
||||
if (index % 10000 == 0) {
|
||||
bulkRequest.execute().actionGet();
|
||||
logger.info("已导入:" + index);
|
||||
// 清空数据
|
||||
bulkRequest = transportClient.prepareBulk();
|
||||
}
|
||||
}
|
||||
bulkRequest.execute().actionGet();
|
||||
logger.info("导入结束,共导入:" + ac.addAndGet(0));
|
||||
}
|
||||
|
||||
}
|
||||
+3
-3
@@ -38,7 +38,7 @@ public class EsfDealPriceSyncEsJob implements SimpleJob {
|
||||
|
||||
@Override
|
||||
public void execute(ShardingContext shardingContext) {
|
||||
DBCollection collection = businessMongoTemplate.getCollection("esf_deal_price_fj");
|
||||
DBCollection collection = businessMongoTemplate.getCollection("res_esfdealprice");
|
||||
DBCursor cursor = collection.find().batchSize(1000).addOption(Bytes.QUERYOPTION_NOTIMEOUT);
|
||||
BulkRequestBuilder bulkRequest = transportClient.prepareBulk();
|
||||
int index = 0;
|
||||
@@ -48,11 +48,11 @@ public class EsfDealPriceSyncEsJob implements SimpleJob {
|
||||
ac.incrementAndGet();
|
||||
String id = data.get("_id").toString();
|
||||
data.removeField("_id");
|
||||
bulkRequest.add(transportClient.prepareIndex("esf_deal_price","doc")
|
||||
bulkRequest.add(transportClient.prepareIndex("esf_deal_price_ls","doc")
|
||||
.setSource(JsonUtils.toJson(data), XContentType.JSON)
|
||||
.setId(id));
|
||||
index++;
|
||||
if (index % 50000 == 0) {
|
||||
if (index % 10000 == 0) {
|
||||
bulkRequest.execute().actionGet();
|
||||
logger.info("已导入:" + index);
|
||||
// 清空数据
|
||||
|
||||
+3
-3
@@ -28,7 +28,7 @@ import com.mongodb.DBObject;
|
||||
*/
|
||||
public class NewDealPriceSyncEsJob implements SimpleJob {
|
||||
|
||||
private Logger logger = LoggerFactory.getLogger(EsfDealPriceSyncEsJob.class);
|
||||
private Logger logger = LoggerFactory.getLogger(NewDealPriceSyncEsJob.class);
|
||||
|
||||
@Autowired
|
||||
@Qualifier("businessMongoTemplate")
|
||||
@@ -39,7 +39,7 @@ public class NewDealPriceSyncEsJob implements SimpleJob {
|
||||
|
||||
@Override
|
||||
public void execute(ShardingContext shardingContext) {
|
||||
DBCollection collection = businessMongoTemplate.getCollection("new_deal_price_fj");
|
||||
DBCollection collection = businessMongoTemplate.getCollection("res_newdealprice");
|
||||
DBCursor cursor = collection.find().batchSize(500).addOption(Bytes.QUERYOPTION_NOTIMEOUT);
|
||||
BulkRequestBuilder bulkRequest = transportClient.prepareBulk();
|
||||
int index = 0;
|
||||
@@ -49,7 +49,7 @@ public class NewDealPriceSyncEsJob implements SimpleJob {
|
||||
ac.incrementAndGet();
|
||||
String id = data.get("_id").toString();
|
||||
data.removeField("_id");
|
||||
bulkRequest.add(transportClient.prepareIndex("new_deal_price","doc")
|
||||
bulkRequest.add(transportClient.prepareIndex("new_deal_price_ls","doc")
|
||||
.setSource(JsonUtils.toJson(data), XContentType.JSON)
|
||||
.setId(id));
|
||||
index++;
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
mongo.business.url=192.168.0.136
|
||||
mongo.business.url=114.80.150.198
|
||||
mongo.business.name=business
|
||||
mongo.business.port=27017
|
||||
|
||||
spring.datasource.log.url=jdbc:mysql://192.168.0.136:3306/batch_log
|
||||
spring.datasource.log.username=sa
|
||||
spring.datasource.log.password=G00jia
|
||||
spring.datasource.log.url=jdbc:mysql://localhost:3306/batch_log
|
||||
spring.datasource.log.username=root
|
||||
spring.datasource.log.password=123456
|
||||
spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver
|
||||
|
||||
zk.server-lists=192.168.10.124:2181
|
||||
zk.namespace=fangjia-job-basicdata
|
||||
|
||||
es.url=192.168.0.45
|
||||
es.cluster.name=fangjia
|
||||
@@ -11,3 +11,4 @@ zk.server-lists=192.168.0.207:2181,192.168.0.24:2181,192.168.0.25:2181
|
||||
zk.namespace=fangjia-job-basicdata
|
||||
|
||||
es.url=192.168.0.45
|
||||
es.cluster.name=fangjia
|
||||
@@ -11,3 +11,4 @@ zk.server-lists=192.168.10.124:2181
|
||||
zk.namespace=fangjia-job-basicdata
|
||||
|
||||
es.url=192.168.0.45
|
||||
es.cluster.name=fangjia
|
||||
@@ -27,12 +27,23 @@
|
||||
base-sleep-time-milliseconds="1000" max-sleep-time-milliseconds="3000" max-retries="3" />
|
||||
|
||||
|
||||
<!-- 企业统计任务 每天0点1分执行 -->
|
||||
<job:simple id="esfDealPriceSyncEsJob" class="com.fangjia.job.basicdata.job.EsfDealPriceSyncEsJob" registry-center-ref="regCenter"
|
||||
<!-- 一手成交数据导入ES任务 -->
|
||||
<!-- <job:simple id="newDealPriceSyncEsJob" class="com.fangjia.job.basicdata.job.NewDealPriceSyncEsJob" registry-center-ref="regCenter"
|
||||
sharding-total-count="1" cron="0/10 * * * * ?" sharding-item-parameters=""
|
||||
failover="true" description="【基础数据处理任务】企业统计任务,每天0点1分开始执行,主要统计企业相关的数据,统计内容参考 EnterpriseStat"
|
||||
overwrite="true" event-trace-rdb-data-source="elasticJobLog">
|
||||
failover="true" description="【基础数据处理任务】一手成交数据导入ES任务"
|
||||
overwrite="true" event-trace-rdb-data-source="elasticJobLog"> -->
|
||||
|
||||
<!-- 二手成交数据导入ES任务 -->
|
||||
<!-- <job:simple id="esfDealPriceSyncEsJob" class="com.fangjia.job.basicdata.job.EsfDealPriceSyncEsJob" registry-center-ref="regCenter"
|
||||
sharding-total-count="1" cron="0/10 * * * * ?" sharding-item-parameters=""
|
||||
failover="true" description="【基础数据处理任务】二手成交数据导入ES任务"
|
||||
overwrite="true" event-trace-rdb-data-source="elasticJobLog"> -->
|
||||
|
||||
<!-- 拍卖数据导入ES任务 -->
|
||||
<!-- <job:simple id="auctionDataSyncEsJob" class="com.fangjia.job.basicdata.job.AuctionDataSyncEsJob" registry-center-ref="regCenter"
|
||||
sharding-total-count="1" cron="0/10 * * * * ?" sharding-item-parameters=""
|
||||
failover="true" description="【基础数据处理任务】拍卖数据导入ES任务"
|
||||
overwrite="true" event-trace-rdb-data-source="elasticJobLog"> -->
|
||||
|
||||
</job:simple>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user