From f5f2ed957d8b6521677bf54e2162c471af0b56d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=B9=E5=90=89=E6=AC=A2?= Date: Wed, 29 Aug 2018 21:16:03 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E5=9F=BA=E7=A1=80=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=A4=84=E7=90=86=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fangjia-job/fangjia-job-basicdata/pom.xml | 87 +++++++++++++++++++ .../job/basicdata/BasicDataJobApp.java | 29 +++++++ .../job/basicdata/config/SpringEsConfig.java | 34 ++++++++ .../basicdata/config/SpringMongoConfig.java | 60 +++++++++++++ .../basicdata/job/EsfDealPriceSyncEsJob.java | 66 ++++++++++++++ .../basicdata/job/NewDealPriceSyncEsJob.java | 67 ++++++++++++++ .../main/resources/application-dev.properties | 13 +++ .../resources/application-prod.properties | 13 +++ .../resources/application-test.properties | 13 +++ .../src/main/resources/application.properties | 1 + .../resources/applicationContext-conf.xml | 28 ++++++ .../src/main/resources/applicationContext.xml | 39 +++++++++ .../src/main/resources/logback-spring.xml | 39 +++++++++ .../main/resources/application-dev.properties | 14 +-- .../src/main/resources/applicationContext.xml | 2 +- fangjia-job/pom.xml | 6 +- 16 files changed, 500 insertions(+), 11 deletions(-) create mode 100644 fangjia-job/fangjia-job-basicdata/pom.xml create mode 100644 fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/BasicDataJobApp.java create mode 100644 fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/config/SpringEsConfig.java create mode 100644 fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/config/SpringMongoConfig.java create mode 100644 fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/job/EsfDealPriceSyncEsJob.java create mode 100644 fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/job/NewDealPriceSyncEsJob.java create mode 100644 fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties create mode 100644 fangjia-job/fangjia-job-basicdata/src/main/resources/application-prod.properties create mode 100644 fangjia-job/fangjia-job-basicdata/src/main/resources/application-test.properties create mode 100644 fangjia-job/fangjia-job-basicdata/src/main/resources/application.properties create mode 100644 fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext-conf.xml create mode 100644 fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext.xml create mode 100644 fangjia-job/fangjia-job-basicdata/src/main/resources/logback-spring.xml diff --git a/fangjia-job/fangjia-job-basicdata/pom.xml b/fangjia-job/fangjia-job-basicdata/pom.xml new file mode 100644 index 0000000..269291e --- /dev/null +++ b/fangjia-job/fangjia-job-basicdata/pom.xml @@ -0,0 +1,87 @@ + + + 4.0.0 + + com.fangjia + fangjia-job + 1.0 + + fangjia-job-basicdata + fangjia-job-basicdata + http://maven.apache.org + + UTF-8 + + + + + com.fangjia + fangjia-job-common + + + + com.dangdang + elastic-job-lite-core + + + + com.dangdang + elastic-job-lite-spring + + + + org.springframework.boot + spring-boot-starter-web + + + + org.springframework.boot + spring-boot-starter-data-mongodb + + + + org.springframework.boot + spring-boot-starter-jdbc + + + + mysql + mysql-connector-java + + + + commons-dbcp + commons-dbcp + + + + org.projectlombok + lombok + + + + org.elasticsearch.client + transport + + + + org.elasticsearch + elasticsearch + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + true + org.fangjia.job.basicdata.BasicDataJobApp + + + + + diff --git a/fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/BasicDataJobApp.java b/fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/BasicDataJobApp.java new file mode 100644 index 0000000..399dacc --- /dev/null +++ b/fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/BasicDataJobApp.java @@ -0,0 +1,29 @@ +package org.fangjia.job.basicdata; + +import java.util.concurrent.CountDownLatch; + +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.context.annotation.ImportResource; + +/** + * 基础数据任务启动入口 + * + *

专门基础数据的处理,比如:从MongoDb同步到ES

+ * + * @author yinjihuan + * + * @date 2018-08-28 + * + */ +@SpringBootApplication +@ImportResource(locations = { "classpath:applicationContext*.xml" }) +public class BasicDataJobApp { + public static void main(String[] args) { + new SpringApplicationBuilder().sources(BasicDataJobApp.class).web(false).run(args); + try { + new CountDownLatch(1).await(); + } catch (InterruptedException e) { + } + } +} diff --git a/fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/config/SpringEsConfig.java b/fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/config/SpringEsConfig.java new file mode 100644 index 0000000..79933e4 --- /dev/null +++ b/fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/config/SpringEsConfig.java @@ -0,0 +1,34 @@ +package org.fangjia.job.basicdata.config; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.transport.client.PreBuiltTransportClient; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class SpringEsConfig { + + @Value("${es.url}") + private String url; + + @SuppressWarnings("resource") + @Bean + public TransportClient transportClient() { + Settings settings = Settings.builder().put("cluster.name", "fangjia") + .put("client.transport.ping_timeout", "60s").build(); + try { + return new PreBuiltTransportClient(settings) + .addTransportAddress(new TransportAddress(InetAddress.getByName(url), 9300)); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + return null; + } + +} diff --git a/fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/config/SpringMongoConfig.java b/fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/config/SpringMongoConfig.java new file mode 100644 index 0000000..5a3371c --- /dev/null +++ b/fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/config/SpringMongoConfig.java @@ -0,0 +1,60 @@ +package org.fangjia.job.basicdata.config; + +import java.util.Arrays; +import java.util.List; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDbFactory; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoDbFactory; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; + +import com.mongodb.MongoClient; +import com.mongodb.MongoClientOptions; +import com.mongodb.ServerAddress; + +@Configuration +public class SpringMongoConfig { + + @Value("${mongo.business.url}") + private String businessUrl; + + @Value("${mongo.business.name}") + private String businessName; + + @Value("${mongo.business.port}") + private int businessPort; + + @Primary + @Bean + public MongoDbFactory businessMongoDbFactory() throws Exception { + List seeds = Arrays.asList(new ServerAddress(businessUrl, businessPort)); + MongoClientOptions.Builder builder = new MongoClientOptions.Builder(); + int connPerHost = 10; + builder.connectionsPerHost(connPerHost); + int threadCount = 10; + builder.threadsAllowedToBlockForConnectionMultiplier(threadCount); + int maxWaitTime = 10000; + builder.maxWaitTime(maxWaitTime); + int timeOut = 10000; + builder.connectTimeout(timeOut); + MongoClientOptions options = builder.build(); + MongoClient mongoClient = new MongoClient(seeds, options); + return new SimpleMongoDbFactory(mongoClient, businessName); + } + + @Primary + @Bean + @SuppressWarnings("deprecation") + public MongoTemplate businessMongoTemplate() throws Exception { + //remove _class + MappingMongoConverter converter = new MappingMongoConverter(businessMongoDbFactory(), new MongoMappingContext()); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + MongoTemplate mongoTemplate = new MongoTemplate(businessMongoDbFactory(), converter); + return mongoTemplate; + } +} diff --git a/fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/job/EsfDealPriceSyncEsJob.java b/fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/job/EsfDealPriceSyncEsJob.java new file mode 100644 index 0000000..7be3ea9 --- /dev/null +++ b/fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/job/EsfDealPriceSyncEsJob.java @@ -0,0 +1,66 @@ +package org.fangjia.job.basicdata.job; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.xcontent.XContentType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.mongodb.core.MongoTemplate; +import com.dangdang.ddframe.job.api.ShardingContext; +import com.dangdang.ddframe.job.api.simple.SimpleJob; +import com.fangjia.job.common.util.JsonUtils; +import com.mongodb.Bytes; +import com.mongodb.DBCollection; +import com.mongodb.DBCursor; +import com.mongodb.DBObject; +/** + * 二手成交数据导入ES任务 + * + * @author yinjihuan + * + * @date 2018-08-29 + * + */ +public class EsfDealPriceSyncEsJob implements SimpleJob { + + private Logger logger = LoggerFactory.getLogger(EsfDealPriceSyncEsJob.class); + + @Autowired + @Qualifier("businessMongoTemplate") + private MongoTemplate businessMongoTemplate; + + @Autowired + private TransportClient transportClient; + + @Override + public void execute(ShardingContext shardingContext) { + DBCollection collection = businessMongoTemplate.getCollection("esf_deal_price_fj"); + DBCursor cursor = collection.find().batchSize(500).addOption(Bytes.QUERYOPTION_NOTIMEOUT); + BulkRequestBuilder bulkRequest = transportClient.prepareBulk(); + int index = 0; + AtomicInteger ac = new AtomicInteger(); + while (cursor.hasNext()) { + DBObject data = cursor.next(); + ac.incrementAndGet(); + String id = data.get("_id").toString(); + data.removeField("_id"); + bulkRequest.add(transportClient.prepareIndex("esf_deal_price","doc") + .setSource(JsonUtils.toJson(data), XContentType.JSON) + .setId(id)); + index++; + if (index % 10000 == 0) { + bulkRequest.execute().actionGet(); + logger.info("已导入:" + index); + // 清空数据 + bulkRequest = transportClient.prepareBulk(); + } + } + bulkRequest.execute().actionGet(); + logger.info("导入结束,共导入:" + ac.addAndGet(0)); + } + +} diff --git a/fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/job/NewDealPriceSyncEsJob.java b/fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/job/NewDealPriceSyncEsJob.java new file mode 100644 index 0000000..2365339 --- /dev/null +++ b/fangjia-job/fangjia-job-basicdata/src/main/java/org/fangjia/job/basicdata/job/NewDealPriceSyncEsJob.java @@ -0,0 +1,67 @@ +package org.fangjia.job.basicdata.job; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.xcontent.XContentType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.mongodb.core.MongoTemplate; + +import com.dangdang.ddframe.job.api.ShardingContext; +import com.dangdang.ddframe.job.api.simple.SimpleJob; +import com.fangjia.job.common.util.JsonUtils; +import com.mongodb.Bytes; +import com.mongodb.DBCollection; +import com.mongodb.DBCursor; +import com.mongodb.DBObject; +/** + * 一手成交数据导入ES任务 + * + * @author yinjihuan + * + * @date 2018-08-29 + * + */ +public class NewDealPriceSyncEsJob implements SimpleJob { + + private Logger logger = LoggerFactory.getLogger(EsfDealPriceSyncEsJob.class); + + @Autowired + @Qualifier("businessMongoTemplate") + private MongoTemplate businessMongoTemplate; + + @Autowired + private TransportClient transportClient; + + @Override + public void execute(ShardingContext shardingContext) { + DBCollection collection = businessMongoTemplate.getCollection("new_deal_price_fj"); + DBCursor cursor = collection.find().batchSize(500).addOption(Bytes.QUERYOPTION_NOTIMEOUT); + BulkRequestBuilder bulkRequest = transportClient.prepareBulk(); + int index = 0; + AtomicInteger ac = new AtomicInteger(); + while (cursor.hasNext()) { + DBObject data = cursor.next(); + ac.incrementAndGet(); + String id = data.get("_id").toString(); + data.removeField("_id"); + bulkRequest.add(transportClient.prepareIndex("new_deal_price","doc") + .setSource(JsonUtils.toJson(data), XContentType.JSON) + .setId(id)); + index++; + if (index % 10000 == 0) { + bulkRequest.execute().actionGet(); + logger.info("已导入:" + index); + // 清空数据 + bulkRequest = transportClient.prepareBulk(); + } + } + bulkRequest.execute().actionGet(); + logger.info("导入结束,共导入:" + ac.addAndGet(0)); + } + +} diff --git a/fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties new file mode 100644 index 0000000..d6bd1b8 --- /dev/null +++ b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties @@ -0,0 +1,13 @@ +mongo.business.url=192.168.0.136 +mongo.business.name=business +mongo.business.port=27017 + +spring.datasource.log.url=jdbc:mysql://192.168.0.136:3306/batch_log +spring.datasource.log.username=sa +spring.datasource.log.password=G00jia +spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver + +zk.server-lists=192.168.10.124:2181 +zk.namespace=fangjia-job-basicdata + +es.url=192.168.0.45 \ No newline at end of file diff --git a/fangjia-job/fangjia-job-basicdata/src/main/resources/application-prod.properties b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-prod.properties new file mode 100644 index 0000000..250cee1 --- /dev/null +++ b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-prod.properties @@ -0,0 +1,13 @@ +mongo.business.url=mongo.db.fangjia.com +mongo.business.name=business +mongo.business.port=27017 + +spring.datasource.log.url=jdbc:mysql://mysql.db.fangjia.com:4306/batch_log +spring.datasource.log.username=sa +spring.datasource.log.password=G00jia +spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver + +zk.server-lists=192.168.0.207:2181,192.168.0.24:2181,192.168.0.25:2181 +zk.namespace=fangjia-job-basicdata + +es.url=192.168.0.45 \ No newline at end of file diff --git a/fangjia-job/fangjia-job-basicdata/src/main/resources/application-test.properties b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-test.properties new file mode 100644 index 0000000..d6bd1b8 --- /dev/null +++ b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-test.properties @@ -0,0 +1,13 @@ +mongo.business.url=192.168.0.136 +mongo.business.name=business +mongo.business.port=27017 + +spring.datasource.log.url=jdbc:mysql://192.168.0.136:3306/batch_log +spring.datasource.log.username=sa +spring.datasource.log.password=G00jia +spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver + +zk.server-lists=192.168.10.124:2181 +zk.namespace=fangjia-job-basicdata + +es.url=192.168.0.45 \ No newline at end of file diff --git a/fangjia-job/fangjia-job-basicdata/src/main/resources/application.properties b/fangjia-job/fangjia-job-basicdata/src/main/resources/application.properties new file mode 100644 index 0000000..257b306 --- /dev/null +++ b/fangjia-job/fangjia-job-basicdata/src/main/resources/application.properties @@ -0,0 +1 @@ +spring.profiles.active=dev \ No newline at end of file diff --git a/fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext-conf.xml b/fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext-conf.xml new file mode 100644 index 0000000..1aa6e44 --- /dev/null +++ b/fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext-conf.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext.xml b/fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext.xml new file mode 100644 index 0000000..3770b21 --- /dev/null +++ b/fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/fangjia-job/fangjia-job-basicdata/src/main/resources/logback-spring.xml b/fangjia-job/fangjia-job-basicdata/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..23e6720 --- /dev/null +++ b/fangjia-job/fangjia-job-basicdata/src/main/resources/logback-spring.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n + + + + + + + + ${LOG_HOME}/basicdata-job.log.%d{yyyy-MM-dd}.log + + 7 + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n + + + + 100MB + + + + + + + + + + \ No newline at end of file diff --git a/fangjia-job/fangjia-job-youfang/src/main/resources/application-dev.properties b/fangjia-job/fangjia-job-youfang/src/main/resources/application-dev.properties index c9f9502..ba9cb2a 100644 --- a/fangjia-job/fangjia-job-youfang/src/main/resources/application-dev.properties +++ b/fangjia-job/fangjia-job-youfang/src/main/resources/application-dev.properties @@ -1,28 +1,28 @@ -mongo.youfang.url=mongo.db.fangjia.com +mongo.youfang.url=192.168.0.136 mongo.youfang.name=fangjia_youfang mongo.youfang.port=27017 -mongo.seaweed.url=mongo.db.fangjia.com +mongo.seaweed.url=192.168.0.136 mongo.seaweed.name=fangjia mongo.seaweed.port=27017 -spring.datasource.url=jdbc:mysql://192.168.0.132:4306/fangjia_youfang +spring.datasource.url=jdbc:mysql://192.168.0.136:3306/fangjia_youfang spring.datasource.username=sa spring.datasource.password=G00jia spring.datasource.driver-class-name=com.mysql.jdbc.Driver -spring.datasource.log.url=jdbc:mysql://192.168.0.132:4306/batch_log +spring.datasource.log.url=jdbc:mysql://192.168.0.136:3306/batch_log spring.datasource.log.username=sa spring.datasource.log.password=G00jia spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver fsh.url=http://192.168.10.47:8086 -zk.server-lists=192.168.10.47:2181 +zk.server-lists=192.168.10.124:2181 zk.namespace=fangjia-job-youfang-test -open.url=http://192.168.10.47:9180 +open.url=http://test2.open.fangjia.com FANGJIA_API_TOKEN=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB -es.url=192.168.0.222 \ No newline at end of file +es.url=222.73.196.132 \ No newline at end of file diff --git a/fangjia-job/fangjia-job-youfang/src/main/resources/applicationContext.xml b/fangjia-job/fangjia-job-youfang/src/main/resources/applicationContext.xml index f38a80e..b06feef 100644 --- a/fangjia-job/fangjia-job-youfang/src/main/resources/applicationContext.xml +++ b/fangjia-job/fangjia-job-youfang/src/main/resources/applicationContext.xml @@ -95,7 +95,7 @@ diff --git a/fangjia-job/pom.xml b/fangjia-job/pom.xml index 77001fb..7089d77 100644 --- a/fangjia-job/pom.xml +++ b/fangjia-job/pom.xml @@ -1,6 +1,5 @@ - + 4.0.0 com.fangjia fangjia-job @@ -10,7 +9,8 @@ fangjia-job-fsh fangjia-job-common fangjia-job-youfang - + fangjia-job-basicdata + 1.4.2.RELEASE