提交基础数据处理任务

This commit is contained in:
尹吉欢
2018-08-29 21:16:03 +08:00
parent 35bf159c77
commit f5f2ed957d
16 changed files with 500 additions and 11 deletions
+87
View File
@@ -0,0 +1,87 @@
<?xml version="1.0"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.fangjia</groupId>
<artifactId>fangjia-job</artifactId>
<version>1.0</version>
</parent>
<artifactId>fangjia-job-basicdata</artifactId>
<name>fangjia-job-basicdata</name>
<url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>com.fangjia</groupId>
<artifactId>fangjia-job-common</artifactId>
</dependency>
<dependency>
<groupId>com.dangdang</groupId>
<artifactId>elastic-job-lite-core</artifactId>
</dependency>
<dependency>
<groupId>com.dangdang</groupId>
<artifactId>elastic-job-lite-spring</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-mongodb</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
<dependency>
<groupId>commons-dbcp</groupId>
<artifactId>commons-dbcp</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<executable>true</executable>
<mainClass>org.fangjia.job.basicdata.BasicDataJobApp</mainClass>
</configuration>
</plugin>
</plugins>
</build>
</project>
@@ -0,0 +1,29 @@
package org.fangjia.job.basicdata;
import java.util.concurrent.CountDownLatch;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.context.annotation.ImportResource;
/**
* 基础数据任务启动入口
*
* <p>专门基础数据的处理,比如:从MongoDb同步到ES<p>
*
* @author yinjihuan
*
* @date 2018-08-28
*
*/
@SpringBootApplication
@ImportResource(locations = { "classpath:applicationContext*.xml" })
public class BasicDataJobApp {
public static void main(String[] args) {
new SpringApplicationBuilder().sources(BasicDataJobApp.class).web(false).run(args);
try {
new CountDownLatch(1).await();
} catch (InterruptedException e) {
}
}
}
@@ -0,0 +1,34 @@
package org.fangjia.job.basicdata.config;
import java.net.InetAddress;
import java.net.UnknownHostException;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class SpringEsConfig {
@Value("${es.url}")
private String url;
@SuppressWarnings("resource")
@Bean
public TransportClient transportClient() {
Settings settings = Settings.builder().put("cluster.name", "fangjia")
.put("client.transport.ping_timeout", "60s").build();
try {
return new PreBuiltTransportClient(settings)
.addTransportAddress(new TransportAddress(InetAddress.getByName(url), 9300));
} catch (UnknownHostException e) {
e.printStackTrace();
}
return null;
}
}
@@ -0,0 +1,60 @@
package org.fangjia.job.basicdata.config;
import java.util.Arrays;
import java.util.List;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.data.mongodb.MongoDbFactory;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.SimpleMongoDbFactory;
import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper;
import org.springframework.data.mongodb.core.convert.MappingMongoConverter;
import org.springframework.data.mongodb.core.mapping.MongoMappingContext;
import com.mongodb.MongoClient;
import com.mongodb.MongoClientOptions;
import com.mongodb.ServerAddress;
@Configuration
public class SpringMongoConfig {
@Value("${mongo.business.url}")
private String businessUrl;
@Value("${mongo.business.name}")
private String businessName;
@Value("${mongo.business.port}")
private int businessPort;
@Primary
@Bean
public MongoDbFactory businessMongoDbFactory() throws Exception {
List<ServerAddress> seeds = Arrays.asList(new ServerAddress(businessUrl, businessPort));
MongoClientOptions.Builder builder = new MongoClientOptions.Builder();
int connPerHost = 10;
builder.connectionsPerHost(connPerHost);
int threadCount = 10;
builder.threadsAllowedToBlockForConnectionMultiplier(threadCount);
int maxWaitTime = 10000;
builder.maxWaitTime(maxWaitTime);
int timeOut = 10000;
builder.connectTimeout(timeOut);
MongoClientOptions options = builder.build();
MongoClient mongoClient = new MongoClient(seeds, options);
return new SimpleMongoDbFactory(mongoClient, businessName);
}
@Primary
@Bean
@SuppressWarnings("deprecation")
public MongoTemplate businessMongoTemplate() throws Exception {
//remove _class
MappingMongoConverter converter = new MappingMongoConverter(businessMongoDbFactory(), new MongoMappingContext());
converter.setTypeMapper(new DefaultMongoTypeMapper(null));
MongoTemplate mongoTemplate = new MongoTemplate(businessMongoDbFactory(), converter);
return mongoTemplate;
}
}
@@ -0,0 +1,66 @@
package org.fangjia.job.basicdata.job;
import java.util.concurrent.atomic.AtomicInteger;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import com.dangdang.ddframe.job.api.ShardingContext;
import com.dangdang.ddframe.job.api.simple.SimpleJob;
import com.fangjia.job.common.util.JsonUtils;
import com.mongodb.Bytes;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
/**
* 二手成交数据导入ES任务
*
* @author yinjihuan
*
* @date 2018-08-29
*
*/
public class EsfDealPriceSyncEsJob implements SimpleJob {
private Logger logger = LoggerFactory.getLogger(EsfDealPriceSyncEsJob.class);
@Autowired
@Qualifier("businessMongoTemplate")
private MongoTemplate businessMongoTemplate;
@Autowired
private TransportClient transportClient;
@Override
public void execute(ShardingContext shardingContext) {
DBCollection collection = businessMongoTemplate.getCollection("esf_deal_price_fj");
DBCursor cursor = collection.find().batchSize(500).addOption(Bytes.QUERYOPTION_NOTIMEOUT);
BulkRequestBuilder bulkRequest = transportClient.prepareBulk();
int index = 0;
AtomicInteger ac = new AtomicInteger();
while (cursor.hasNext()) {
DBObject data = cursor.next();
ac.incrementAndGet();
String id = data.get("_id").toString();
data.removeField("_id");
bulkRequest.add(transportClient.prepareIndex("esf_deal_price","doc")
.setSource(JsonUtils.toJson(data), XContentType.JSON)
.setId(id));
index++;
if (index % 10000 == 0) {
bulkRequest.execute().actionGet();
logger.info("已导入:" + index);
// 清空数据
bulkRequest = transportClient.prepareBulk();
}
}
bulkRequest.execute().actionGet();
logger.info("导入结束,共导入:" + ac.addAndGet(0));
}
}
@@ -0,0 +1,67 @@
package org.fangjia.job.basicdata.job;
import java.util.concurrent.atomic.AtomicInteger;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import com.dangdang.ddframe.job.api.ShardingContext;
import com.dangdang.ddframe.job.api.simple.SimpleJob;
import com.fangjia.job.common.util.JsonUtils;
import com.mongodb.Bytes;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
/**
* 一手成交数据导入ES任务
*
* @author yinjihuan
*
* @date 2018-08-29
*
*/
public class NewDealPriceSyncEsJob implements SimpleJob {
private Logger logger = LoggerFactory.getLogger(EsfDealPriceSyncEsJob.class);
@Autowired
@Qualifier("businessMongoTemplate")
private MongoTemplate businessMongoTemplate;
@Autowired
private TransportClient transportClient;
@Override
public void execute(ShardingContext shardingContext) {
DBCollection collection = businessMongoTemplate.getCollection("new_deal_price_fj");
DBCursor cursor = collection.find().batchSize(500).addOption(Bytes.QUERYOPTION_NOTIMEOUT);
BulkRequestBuilder bulkRequest = transportClient.prepareBulk();
int index = 0;
AtomicInteger ac = new AtomicInteger();
while (cursor.hasNext()) {
DBObject data = cursor.next();
ac.incrementAndGet();
String id = data.get("_id").toString();
data.removeField("_id");
bulkRequest.add(transportClient.prepareIndex("new_deal_price","doc")
.setSource(JsonUtils.toJson(data), XContentType.JSON)
.setId(id));
index++;
if (index % 10000 == 0) {
bulkRequest.execute().actionGet();
logger.info("已导入:" + index);
// 清空数据
bulkRequest = transportClient.prepareBulk();
}
}
bulkRequest.execute().actionGet();
logger.info("导入结束,共导入:" + ac.addAndGet(0));
}
}
@@ -0,0 +1,13 @@
mongo.business.url=192.168.0.136
mongo.business.name=business
mongo.business.port=27017
spring.datasource.log.url=jdbc:mysql://192.168.0.136:3306/batch_log
spring.datasource.log.username=sa
spring.datasource.log.password=G00jia
spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver
zk.server-lists=192.168.10.124:2181
zk.namespace=fangjia-job-basicdata
es.url=192.168.0.45
@@ -0,0 +1,13 @@
mongo.business.url=mongo.db.fangjia.com
mongo.business.name=business
mongo.business.port=27017
spring.datasource.log.url=jdbc:mysql://mysql.db.fangjia.com:4306/batch_log
spring.datasource.log.username=sa
spring.datasource.log.password=G00jia
spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver
zk.server-lists=192.168.0.207:2181,192.168.0.24:2181,192.168.0.25:2181
zk.namespace=fangjia-job-basicdata
es.url=192.168.0.45
@@ -0,0 +1,13 @@
mongo.business.url=192.168.0.136
mongo.business.name=business
mongo.business.port=27017
spring.datasource.log.url=jdbc:mysql://192.168.0.136:3306/batch_log
spring.datasource.log.username=sa
spring.datasource.log.password=G00jia
spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver
zk.server-lists=192.168.10.124:2181
zk.namespace=fangjia-job-basicdata
es.url=192.168.0.45
@@ -0,0 +1 @@
spring.profiles.active=dev
@@ -0,0 +1,28 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:context="http://www.springframework.org/schema/context"
xmlns:reg="http://www.dangdang.com/schema/ddframe/reg" xmlns:job="http://www.dangdang.com/schema/ddframe/job"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context.xsd
http://www.dangdang.com/schema/ddframe/reg
http://www.dangdang.com/schema/ddframe/reg/reg.xsd
http://www.dangdang.com/schema/ddframe/job
http://www.dangdang.com/schema/ddframe/job/job.xsd
">
<context:component-scan base-package="com.fangjia.job" />
<beans profile="dev">
<context:property-placeholder location="classpath:application-dev.properties" />
</beans>
<beans profile="test">
<context:property-placeholder location="classpath:application-test.properties" />
</beans>
<beans profile="prod">
<context:property-placeholder location="classpath:application-prod.properties" />
</beans>
</beans>
@@ -0,0 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:reg="http://www.dangdang.com/schema/ddframe/reg"
xmlns:job="http://www.dangdang.com/schema/ddframe/job"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context.xsd
http://www.dangdang.com/schema/ddframe/reg
http://www.dangdang.com/schema/ddframe/reg/reg.xsd
http://www.dangdang.com/schema/ddframe/job
http://www.dangdang.com/schema/ddframe/job/job.xsd
">
<bean id="elasticJobLog" class="org.apache.commons.dbcp.BasicDataSource" destroy-method="close">
<property name="driverClassName" value="${spring.datasource.log.driver-class-name}"/>
<property name="url" value="${spring.datasource.log.url}"/>
<property name="username" value="${spring.datasource.log.username}"/>
<property name="password" value="${spring.datasource.log.password}"/>
<property name="validationQuery" value="SELECT 1" />
<property name="testOnBorrow" value="true"/>
</bean>
<reg:zookeeper id="regCenter" server-lists="${zk.server-lists}" namespace="${zk.namespace}"
base-sleep-time-milliseconds="1000" max-sleep-time-milliseconds="3000" max-retries="3" />
<!-- 企业统计任务 每天0点1分执行 -->
<job:simple id="esfDealPriceSyncEsJob" class="org.fangjia.job.basicdata.job.EsfDealPriceSyncEsJob" registry-center-ref="regCenter"
sharding-total-count="1" cron="0/10 * * * * ?" sharding-item-parameters=""
failover="true" description="【基础数据处理任务】企业统计任务,每天0点1分开始执行,主要统计企业相关的数据,统计内容参考 EnterpriseStat"
overwrite="true" event-trace-rdb-data-source="elasticJobLog">
</job:simple>
</beans>
@@ -0,0 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false">
<!--定义日志文件的存储地址 勿在 LogBack 的配置中使用相对路径-->
<property name="LOG_HOME" value="/home/goojia/logs" />
<!-- 控制台输出 -->
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<!--格式化输出:%d表示日期,%thread表示线程名,%-5level:级别从左显示5个字符宽度%msg:日志消息,%n是换行符-->
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
</encoder>
</appender>
<!-- 按照每天生成日志文件 -->
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!--日志文件输出的文件名-->
<FileNamePattern>${LOG_HOME}/basicdata-job.log.%d{yyyy-MM-dd}.log</FileNamePattern>
<!--日志文件保留天数-->
<MaxHistory>7</MaxHistory>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<!--格式化输出:%d表示日期,%thread表示线程名,%-5level:级别从左显示5个字符宽度%msg:日志消息,%n是换行符-->
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
</encoder>
<!--日志文件最大的大小-->
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
<MaxFileSize>100MB</MaxFileSize>
</triggeringPolicy>
</appender>
<!-- 日志输出级别 -->
<root level="INFO">
<appender-ref ref="STDOUT" />
<appender-ref ref="FILE" />
</root>
</configuration>
@@ -1,28 +1,28 @@
mongo.youfang.url=mongo.db.fangjia.com
mongo.youfang.url=192.168.0.136
mongo.youfang.name=fangjia_youfang
mongo.youfang.port=27017
mongo.seaweed.url=mongo.db.fangjia.com
mongo.seaweed.url=192.168.0.136
mongo.seaweed.name=fangjia
mongo.seaweed.port=27017
spring.datasource.url=jdbc:mysql://192.168.0.132:4306/fangjia_youfang
spring.datasource.url=jdbc:mysql://192.168.0.136:3306/fangjia_youfang
spring.datasource.username=sa
spring.datasource.password=G00jia
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.log.url=jdbc:mysql://192.168.0.132:4306/batch_log
spring.datasource.log.url=jdbc:mysql://192.168.0.136:3306/batch_log
spring.datasource.log.username=sa
spring.datasource.log.password=G00jia
spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver
fsh.url=http://192.168.10.47:8086
zk.server-lists=192.168.10.47:2181
zk.server-lists=192.168.10.124:2181
zk.namespace=fangjia-job-youfang-test
open.url=http://192.168.10.47:9180
open.url=http://test2.open.fangjia.com
FANGJIA_API_TOKEN=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB
es.url=192.168.0.222
es.url=222.73.196.132
@@ -95,7 +95,7 @@
<!-- 有房小区详情信息汇总 每周六0点10分执行-->
<job:simple id="districtInfoJob" class="com.fangjia.job.youfang.job.DistrictInfoJob" registry-center-ref="regCenter"
sharding-total-count="1" cron="0 10 0 ? * 7" sharding-item-parameters=""
sharding-total-count="1" cron="0/10 * * * * ?" sharding-item-parameters=""
failover="true" description="【有房】 有房小区详情信息汇总"
overwrite="true" event-trace-rdb-data-source="elasticJobLog" job-exception-handler="com.fangjia.job.youfang.handler.CustomJobExceptionHandler">
+2 -2
View File
@@ -1,6 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.fangjia</groupId>
<artifactId>fangjia-job</artifactId>
@@ -10,6 +9,7 @@
<module>fangjia-job-fsh</module>
<module>fangjia-job-common</module>
<module>fangjia-job-youfang</module>
<module>fangjia-job-basicdata</module>
</modules>
<properties>