diff --git a/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/config/SpringEsConfig.java b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/config/SpringEsConfig.java index c8f634b..bd96515 100644 --- a/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/config/SpringEsConfig.java +++ b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/config/SpringEsConfig.java @@ -17,10 +17,13 @@ public class SpringEsConfig { @Value("${es.url}") private String url; + @Value("${es.cluster.name}") + private String clusterName; + @SuppressWarnings("resource") @Bean public TransportClient transportClient() { - Settings settings = Settings.builder().put("cluster.name", "fangjia") + Settings settings = Settings.builder().put("cluster.name", clusterName) .put("client.transport.ping_timeout", "60s").build(); try { return new PreBuiltTransportClient(settings) diff --git a/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/job/AuctionDataSyncEsJob.java b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/job/AuctionDataSyncEsJob.java new file mode 100644 index 0000000..f384ff2 --- /dev/null +++ b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/job/AuctionDataSyncEsJob.java @@ -0,0 +1,64 @@ +package com.fangjia.job.basicdata.job; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.xcontent.XContentType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.mongodb.core.MongoTemplate; + +import com.dangdang.ddframe.job.api.ShardingContext; +import com.dangdang.ddframe.job.api.simple.SimpleJob; +import com.fangjia.job.common.util.JsonUtils; +import com.mongodb.Bytes; +import com.mongodb.DBCollection; +import com.mongodb.DBCursor; +import com.mongodb.DBObject; +/** + * 拍卖数据同步任务 + * @author yinjihuan + * + */ +public class AuctionDataSyncEsJob implements SimpleJob { + + private Logger logger = LoggerFactory.getLogger(AuctionDataSyncEsJob.class); + + @Autowired + @Qualifier("businessMongoTemplate") + private MongoTemplate businessMongoTemplate; + + @Autowired + private TransportClient transportClient; + + @Override + public void execute(ShardingContext shardingContext) { + DBCollection collection = businessMongoTemplate.getCollection("auction_data"); + DBCursor cursor = collection.find().batchSize(1000).addOption(Bytes.QUERYOPTION_NOTIMEOUT); + BulkRequestBuilder bulkRequest = transportClient.prepareBulk(); + int index = 0; + AtomicInteger ac = new AtomicInteger(); + while (cursor.hasNext()) { + DBObject data = cursor.next(); + ac.incrementAndGet(); + String id = data.get("_id").toString(); + data.removeField("_id"); + bulkRequest.add(transportClient.prepareIndex("auction_data","doc") + .setSource(JsonUtils.toJson(data), XContentType.JSON) + .setId(id)); + index++; + if (index % 10000 == 0) { + bulkRequest.execute().actionGet(); + logger.info("已导入:" + index); + // 清空数据 + bulkRequest = transportClient.prepareBulk(); + } + } + bulkRequest.execute().actionGet(); + logger.info("导入结束,共导入:" + ac.addAndGet(0)); + } + +} diff --git a/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/job/EsfDealPriceSyncEsJob.java b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/job/EsfDealPriceSyncEsJob.java index 8f0d71c..675eaa0 100644 --- a/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/job/EsfDealPriceSyncEsJob.java +++ b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/job/EsfDealPriceSyncEsJob.java @@ -38,7 +38,7 @@ public class EsfDealPriceSyncEsJob implements SimpleJob { @Override public void execute(ShardingContext shardingContext) { - DBCollection collection = businessMongoTemplate.getCollection("esf_deal_price_fj"); + DBCollection collection = businessMongoTemplate.getCollection("res_esfdealprice"); DBCursor cursor = collection.find().batchSize(1000).addOption(Bytes.QUERYOPTION_NOTIMEOUT); BulkRequestBuilder bulkRequest = transportClient.prepareBulk(); int index = 0; @@ -48,11 +48,11 @@ public class EsfDealPriceSyncEsJob implements SimpleJob { ac.incrementAndGet(); String id = data.get("_id").toString(); data.removeField("_id"); - bulkRequest.add(transportClient.prepareIndex("esf_deal_price","doc") + bulkRequest.add(transportClient.prepareIndex("esf_deal_price_ls","doc") .setSource(JsonUtils.toJson(data), XContentType.JSON) .setId(id)); index++; - if (index % 50000 == 0) { + if (index % 10000 == 0) { bulkRequest.execute().actionGet(); logger.info("已导入:" + index); // 清空数据 diff --git a/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/job/NewDealPriceSyncEsJob.java b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/job/NewDealPriceSyncEsJob.java index b865187..a43fd68 100644 --- a/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/job/NewDealPriceSyncEsJob.java +++ b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/job/NewDealPriceSyncEsJob.java @@ -28,7 +28,7 @@ import com.mongodb.DBObject; */ public class NewDealPriceSyncEsJob implements SimpleJob { - private Logger logger = LoggerFactory.getLogger(EsfDealPriceSyncEsJob.class); + private Logger logger = LoggerFactory.getLogger(NewDealPriceSyncEsJob.class); @Autowired @Qualifier("businessMongoTemplate") @@ -39,7 +39,7 @@ public class NewDealPriceSyncEsJob implements SimpleJob { @Override public void execute(ShardingContext shardingContext) { - DBCollection collection = businessMongoTemplate.getCollection("new_deal_price_fj"); + DBCollection collection = businessMongoTemplate.getCollection("res_newdealprice"); DBCursor cursor = collection.find().batchSize(500).addOption(Bytes.QUERYOPTION_NOTIMEOUT); BulkRequestBuilder bulkRequest = transportClient.prepareBulk(); int index = 0; @@ -49,7 +49,7 @@ public class NewDealPriceSyncEsJob implements SimpleJob { ac.incrementAndGet(); String id = data.get("_id").toString(); data.removeField("_id"); - bulkRequest.add(transportClient.prepareIndex("new_deal_price","doc") + bulkRequest.add(transportClient.prepareIndex("new_deal_price_ls","doc") .setSource(JsonUtils.toJson(data), XContentType.JSON) .setId(id)); index++; diff --git a/fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties index d6bd1b8..dc3c82d 100644 --- a/fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties +++ b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties @@ -1,13 +1,14 @@ -mongo.business.url=192.168.0.136 +mongo.business.url=114.80.150.198 mongo.business.name=business mongo.business.port=27017 -spring.datasource.log.url=jdbc:mysql://192.168.0.136:3306/batch_log -spring.datasource.log.username=sa -spring.datasource.log.password=G00jia +spring.datasource.log.url=jdbc:mysql://localhost:3306/batch_log +spring.datasource.log.username=root +spring.datasource.log.password=123456 spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver zk.server-lists=192.168.10.124:2181 zk.namespace=fangjia-job-basicdata -es.url=192.168.0.45 \ No newline at end of file +es.url=192.168.0.45 +es.cluster.name=fangjia \ No newline at end of file diff --git a/fangjia-job/fangjia-job-basicdata/src/main/resources/application-prod.properties b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-prod.properties index 250cee1..37cf237 100644 --- a/fangjia-job/fangjia-job-basicdata/src/main/resources/application-prod.properties +++ b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-prod.properties @@ -10,4 +10,5 @@ spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver zk.server-lists=192.168.0.207:2181,192.168.0.24:2181,192.168.0.25:2181 zk.namespace=fangjia-job-basicdata -es.url=192.168.0.45 \ No newline at end of file +es.url=192.168.0.45 +es.cluster.name=fangjia \ No newline at end of file diff --git a/fangjia-job/fangjia-job-basicdata/src/main/resources/application-test.properties b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-test.properties index d6bd1b8..6ffe205 100644 --- a/fangjia-job/fangjia-job-basicdata/src/main/resources/application-test.properties +++ b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-test.properties @@ -10,4 +10,5 @@ spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver zk.server-lists=192.168.10.124:2181 zk.namespace=fangjia-job-basicdata -es.url=192.168.0.45 \ No newline at end of file +es.url=192.168.0.45 +es.cluster.name=fangjia \ No newline at end of file diff --git a/fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext.xml b/fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext.xml index 83e58f7..c4450b0 100644 --- a/fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext.xml +++ b/fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext.xml @@ -27,12 +27,23 @@ base-sleep-time-milliseconds="1000" max-sleep-time-milliseconds="3000" max-retries="3" /> - - + + + + + +