Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 523ceddab7 | |||
| 328efa6cc8 | |||
| fae953db27 | |||
| 663390b271 | |||
| 48ffa21de2 | |||
| b8795df41d | |||
| 65bad8efc5 |
@@ -82,7 +82,12 @@
|
|||||||
<artifactId>fastjson</artifactId>
|
<artifactId>fastjson</artifactId>
|
||||||
<version>1.2.51</version>
|
<version>1.2.51</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<!-- add the x-pack jar as a dependency -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.elasticsearch.client</groupId>
|
||||||
|
<artifactId>x-pack-transport</artifactId>
|
||||||
|
<version>6.4.2</version>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
<build>
|
<build>
|
||||||
@@ -97,4 +102,18 @@
|
|||||||
</plugin>
|
</plugin>
|
||||||
</plugins>
|
</plugins>
|
||||||
</build>
|
</build>
|
||||||
|
|
||||||
|
<repositories>
|
||||||
|
<!-- add the elasticsearch repo -->
|
||||||
|
<repository>
|
||||||
|
<id>elasticsearch-releases</id>
|
||||||
|
<url>https://artifacts.elastic.co/maven</url>
|
||||||
|
<releases>
|
||||||
|
<enabled>true</enabled>
|
||||||
|
</releases>
|
||||||
|
<snapshots>
|
||||||
|
<enabled>false</enabled>
|
||||||
|
</snapshots>
|
||||||
|
</repository>
|
||||||
|
</repositories>
|
||||||
</project>
|
</project>
|
||||||
|
|||||||
+84
@@ -0,0 +1,84 @@
|
|||||||
|
package com.fangjia.job.basicdata.config;
|
||||||
|
|
||||||
|
import java.net.InetAddress;
|
||||||
|
|
||||||
|
import org.elasticsearch.client.transport.TransportClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.transport.TransportAddress;
|
||||||
|
import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.context.annotation.Bean;
|
||||||
|
import org.springframework.context.annotation.Configuration;
|
||||||
|
import org.springframework.context.annotation.Primary;
|
||||||
|
|
||||||
|
@Configuration
|
||||||
|
public class ElasticSearchConfig {
|
||||||
|
|
||||||
|
private Logger logger = LoggerFactory.getLogger(ElasticSearchConfig.class);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ES集群名称
|
||||||
|
*/
|
||||||
|
@Value("${spring.data.elasticsearch.cluster-name:fangjia}")
|
||||||
|
private String clusterName;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ES用户名
|
||||||
|
*/
|
||||||
|
@Value("${spring.data.elasticsearch.username:elastic}")
|
||||||
|
private String username;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ES密码
|
||||||
|
*/
|
||||||
|
@Value("${spring.data.elasticsearch.password:goojia123456}")
|
||||||
|
private String password;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ES证书地址
|
||||||
|
*/
|
||||||
|
@Value("${spring.data.elasticsearch.keystore-path:C:\\opt\\settings\\elastic-certificates.p12}")
|
||||||
|
private String keystorePath;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ES集群节点
|
||||||
|
*/
|
||||||
|
@Value("${spring.data.elasticsearch.cluster-nodes:192.168.0.45:9300}")
|
||||||
|
private String clusterNodes;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ES心跳间隔
|
||||||
|
*/
|
||||||
|
@Value("${spring.data.elasticsearch.ping_timeout:5s}")
|
||||||
|
private String pingTimeout;
|
||||||
|
|
||||||
|
@SuppressWarnings("resource")
|
||||||
|
@Bean
|
||||||
|
@Primary
|
||||||
|
public TransportClient transportClient() {
|
||||||
|
try {
|
||||||
|
Settings.Builder builder = Settings.builder()
|
||||||
|
.put("client.transport.ping_timeout", pingTimeout)
|
||||||
|
.put("cluster.name", clusterName)
|
||||||
|
.put("xpack.security.user", username + ":" + password)
|
||||||
|
.put("xpack.security.transport.ssl.enabled", "true")
|
||||||
|
.put("xpack.security.transport.ssl.truststore.path", keystorePath)
|
||||||
|
.put("xpack.security.transport.ssl.keystore.path", keystorePath)
|
||||||
|
.put("xpack.security.transport.ssl.verification_mode", "certificate");
|
||||||
|
Settings settings = builder.build();
|
||||||
|
String[] nodes = clusterNodes.split(",");
|
||||||
|
TransportAddress[] addressArray = new TransportAddress[nodes.length];
|
||||||
|
for (int i = 0; i < nodes.length; i++) {
|
||||||
|
String[] nodeArray = nodes[i].split(":");
|
||||||
|
addressArray[i] = new TransportAddress(InetAddress.getByName(nodeArray[0]), Integer.parseInt(nodeArray[1]));
|
||||||
|
}
|
||||||
|
return new PreBuiltXPackTransportClient(settings).addTransportAddresses(addressArray);
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error("初始化ESClient异常", e);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
+1
-1
@@ -11,7 +11,7 @@ import org.springframework.beans.factory.annotation.Value;
|
|||||||
import org.springframework.context.annotation.Bean;
|
import org.springframework.context.annotation.Bean;
|
||||||
import org.springframework.context.annotation.Configuration;
|
import org.springframework.context.annotation.Configuration;
|
||||||
|
|
||||||
@Configuration
|
//@Configuration
|
||||||
public class SpringEsConfig {
|
public class SpringEsConfig {
|
||||||
|
|
||||||
@Value("${es.url}")
|
@Value("${es.url}")
|
||||||
|
|||||||
+2
-1
@@ -34,6 +34,7 @@ public class EsfDealPriceSyncEsJob implements SimpleJob {
|
|||||||
private MongoTemplate businessMongoTemplate;
|
private MongoTemplate businessMongoTemplate;
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
|
@Qualifier("transportClient")
|
||||||
private TransportClient transportClient;
|
private TransportClient transportClient;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -48,7 +49,7 @@ public class EsfDealPriceSyncEsJob implements SimpleJob {
|
|||||||
ac.incrementAndGet();
|
ac.incrementAndGet();
|
||||||
String id = data.get("_id").toString();
|
String id = data.get("_id").toString();
|
||||||
data.removeField("_id");
|
data.removeField("_id");
|
||||||
bulkRequest.add(transportClient.prepareIndex("esf_deal_price_ls","doc")
|
bulkRequest.add(transportClient.prepareIndex("esf_deal_price_lansi","doc")
|
||||||
.setSource(JsonUtils.toJson(data), XContentType.JSON)
|
.setSource(JsonUtils.toJson(data), XContentType.JSON)
|
||||||
.setId(id));
|
.setId(id));
|
||||||
index++;
|
index++;
|
||||||
|
|||||||
+1
-1
@@ -49,7 +49,7 @@ public class NewDealPriceSyncEsJob implements SimpleJob {
|
|||||||
ac.incrementAndGet();
|
ac.incrementAndGet();
|
||||||
String id = data.get("_id").toString();
|
String id = data.get("_id").toString();
|
||||||
data.removeField("_id");
|
data.removeField("_id");
|
||||||
bulkRequest.add(transportClient.prepareIndex("new_deal_price_ls","doc")
|
bulkRequest.add(transportClient.prepareIndex("new_deal_price_lansi","doc")
|
||||||
.setSource(JsonUtils.toJson(data), XContentType.JSON)
|
.setSource(JsonUtils.toJson(data), XContentType.JSON)
|
||||||
.setId(id));
|
.setId(id));
|
||||||
index++;
|
index++;
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
mongo.business.url=114.80.150.198
|
mongo.business.url=192.168.0.136
|
||||||
mongo.business.name=business
|
mongo.business.name=business
|
||||||
mongo.business.port=27017
|
mongo.business.port=27017
|
||||||
|
|
||||||
@@ -14,7 +14,7 @@ spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver
|
|||||||
zk.server-lists=192.168.10.124:2181
|
zk.server-lists=192.168.10.124:2181
|
||||||
zk.namespace=fangjia-job-basicdata
|
zk.namespace=fangjia-job-basicdata
|
||||||
|
|
||||||
es.url=192.168.0.45
|
es.url=192.168.0.228
|
||||||
es.cluster.name=fangjia
|
es.cluster.name=fangjia
|
||||||
|
|
||||||
es.206.url=192.168.0.206
|
es.206.url=192.168.0.206
|
||||||
|
|||||||
@@ -34,11 +34,11 @@
|
|||||||
overwrite="true" event-trace-rdb-data-source="elasticJobLog"> -->
|
overwrite="true" event-trace-rdb-data-source="elasticJobLog"> -->
|
||||||
|
|
||||||
<!-- 二手成交数据导入ES任务 -->
|
<!-- 二手成交数据导入ES任务 -->
|
||||||
<!-- <job:simple id="esfDealPriceSyncEsJob" class="com.fangjia.job.basicdata.job.EsfDealPriceSyncEsJob" registry-center-ref="regCenter"
|
<job:simple id="esfDealPriceSyncEsJob" class="com.fangjia.job.basicdata.job.EsfDealPriceSyncEsJob" registry-center-ref="regCenter"
|
||||||
sharding-total-count="1" cron="0/10 * * * * ?" sharding-item-parameters=""
|
sharding-total-count="1" cron="0/10 * * * * ?" sharding-item-parameters=""
|
||||||
failover="true" description="【基础数据处理任务】二手成交数据导入ES任务"
|
failover="true" description="【基础数据处理任务】二手成交数据导入ES任务"
|
||||||
overwrite="true" event-trace-rdb-data-source="elasticJobLog"> -->
|
overwrite="true" event-trace-rdb-data-source="elasticJobLog">
|
||||||
|
</job:simple>
|
||||||
<!-- 拍卖数据导入ES任务 -->
|
<!-- 拍卖数据导入ES任务 -->
|
||||||
<!-- <job:simple id="auctionDataSyncEsJob" class="com.fangjia.job.basicdata.job.AuctionDataSyncEsJob" registry-center-ref="regCenter"
|
<!-- <job:simple id="auctionDataSyncEsJob" class="com.fangjia.job.basicdata.job.AuctionDataSyncEsJob" registry-center-ref="regCenter"
|
||||||
sharding-total-count="1" cron="0/10 * * * * ?" sharding-item-parameters=""
|
sharding-total-count="1" cron="0/10 * * * * ?" sharding-item-parameters=""
|
||||||
|
|||||||
@@ -5,13 +5,16 @@ package com.fangjia.job.fsh;
|
|||||||
* @author yinjihuan
|
* @author yinjihuan
|
||||||
*/
|
*/
|
||||||
import java.util.concurrent.CountDownLatch;
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
|
||||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||||
|
import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration;
|
||||||
|
import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
|
||||||
import org.springframework.boot.builder.SpringApplicationBuilder;
|
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||||
import org.springframework.context.annotation.ImportResource;
|
import org.springframework.context.annotation.ImportResource;
|
||||||
|
|
||||||
import com.fangjia.job.fsh.config.InitBeanConfig;
|
import com.fangjia.job.fsh.config.InitBeanConfig;
|
||||||
|
|
||||||
@SpringBootApplication
|
@SpringBootApplication(exclude = {MongoAutoConfiguration.class,MongoDataAutoConfiguration.class})
|
||||||
@ImportResource(locations = { "classpath:applicationContext*.xml" })
|
@ImportResource(locations = { "classpath:applicationContext*.xml" })
|
||||||
public class FshJobApp {
|
public class FshJobApp {
|
||||||
|
|
||||||
|
|||||||
+10
-2
@@ -1,7 +1,9 @@
|
|||||||
package com.fangjia.job.fsh.config;
|
package com.fangjia.job.fsh.config;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import org.springframework.context.annotation.Bean;
|
import org.springframework.context.annotation.Bean;
|
||||||
import org.springframework.context.annotation.Configuration;
|
import org.springframework.context.annotation.Configuration;
|
||||||
import org.springframework.data.mongodb.MongoDbFactory;
|
import org.springframework.data.mongodb.MongoDbFactory;
|
||||||
@@ -10,15 +12,19 @@ import org.springframework.data.mongodb.core.SimpleMongoDbFactory;
|
|||||||
import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper;
|
import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper;
|
||||||
import org.springframework.data.mongodb.core.convert.MappingMongoConverter;
|
import org.springframework.data.mongodb.core.convert.MappingMongoConverter;
|
||||||
import org.springframework.data.mongodb.core.mapping.MongoMappingContext;
|
import org.springframework.data.mongodb.core.mapping.MongoMappingContext;
|
||||||
|
|
||||||
import com.mongodb.MongoClient;
|
import com.mongodb.MongoClient;
|
||||||
import com.mongodb.MongoClientOptions;
|
import com.mongodb.MongoClientOptions;
|
||||||
|
import com.mongodb.MongoCredential;
|
||||||
import com.mongodb.ServerAddress;
|
import com.mongodb.ServerAddress;
|
||||||
|
|
||||||
@Configuration
|
@Configuration
|
||||||
public class SpringMongoConfig {
|
public class SpringMongoConfig {
|
||||||
|
|
||||||
public @Bean MongoDbFactory mongoDbFactory() throws Exception {
|
public @Bean MongoDbFactory mongoDbFactory() throws Exception {
|
||||||
List<ServerAddress> seeds = Arrays.asList(new ServerAddress("mongo.db.fangjia.com", 27017));
|
List<ServerAddress> seeds = Arrays.asList(
|
||||||
|
new ServerAddress("mongo.db.fangjia.com", 28017),
|
||||||
|
new ServerAddress("secondary.mongo.db.fangjia.com", 28017));
|
||||||
MongoClientOptions.Builder builder = new MongoClientOptions.Builder();
|
MongoClientOptions.Builder builder = new MongoClientOptions.Builder();
|
||||||
int connPerHost = 10;
|
int connPerHost = 10;
|
||||||
builder.connectionsPerHost(connPerHost);
|
builder.connectionsPerHost(connPerHost);
|
||||||
@@ -29,7 +35,9 @@ public class SpringMongoConfig {
|
|||||||
int timeOut = 60000;
|
int timeOut = 60000;
|
||||||
builder.connectTimeout(timeOut);
|
builder.connectTimeout(timeOut);
|
||||||
MongoClientOptions options = builder.build();
|
MongoClientOptions options = builder.build();
|
||||||
MongoClient mongoClient = new MongoClient(seeds, options);
|
List<MongoCredential> cres = new ArrayList<>();
|
||||||
|
cres.add(MongoCredential.createCredential("developer", "admin", "G00IDCDEV12385".toCharArray()));
|
||||||
|
MongoClient mongoClient = new MongoClient(seeds, cres, options);
|
||||||
return new SimpleMongoDbFactory(mongoClient, "fangjia_fsh");
|
return new SimpleMongoDbFactory(mongoClient, "fangjia_fsh");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,19 +1,13 @@
|
|||||||
spring.data.mongodb.uri=mongodb://mongo.db.fangjia.com/fangjia_fsh
|
|
||||||
|
|
||||||
spring.datasource.url=jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_fsh
|
spring.datasource.url=jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_fsh
|
||||||
spring.datasource.username=sa
|
spring.datasource.username=sa
|
||||||
spring.datasource.password=G00jia
|
spring.datasource.password=G00jia
|
||||||
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
|
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
|
||||||
|
|
||||||
#spring.datasource.log.url=jdbc:mysql://192.168.0.41:4306/batch_log
|
spring.datasource.log.url=jdbc:mysql://mysql.db.fangjia.com:4306/batch_log
|
||||||
#spring.datasource.log.username=root
|
|
||||||
#spring.datasource.log.password=Ga041243
|
|
||||||
#spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver
|
|
||||||
spring.datasource.log.url=jdbc:mysql://192.168.0.132:4306/batch_log
|
|
||||||
spring.datasource.log.username=sa
|
spring.datasource.log.username=sa
|
||||||
spring.datasource.log.password=G00jia
|
spring.datasource.log.password=G00jia
|
||||||
spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver
|
spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver
|
||||||
fsh.url=http://fsh.fangjia.com
|
fsh.url=http://fsh.fangjia.com
|
||||||
|
|
||||||
zk.server-lists=192.168.0.45:2181,192.168.0.46:2181,192.168.0.109:2181
|
zk.server-lists=192.168.0.207:2181,192.168.0.24:2181,192.168.0.25:2181
|
||||||
zk.namespace=fangjia-job-fsh
|
zk.namespace=fangjia-job-fsh
|
||||||
+8
-2
@@ -1,5 +1,6 @@
|
|||||||
package com.fangjia.job.youfang.config;
|
package com.fangjia.job.youfang.config;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
@@ -17,6 +18,7 @@ import org.springframework.data.mongodb.core.mapping.MongoMappingContext;
|
|||||||
|
|
||||||
import com.mongodb.MongoClient;
|
import com.mongodb.MongoClient;
|
||||||
import com.mongodb.MongoClientOptions;
|
import com.mongodb.MongoClientOptions;
|
||||||
|
import com.mongodb.MongoCredential;
|
||||||
import com.mongodb.ServerAddress;
|
import com.mongodb.ServerAddress;
|
||||||
|
|
||||||
@Configuration
|
@Configuration
|
||||||
@@ -43,7 +45,9 @@ public class SpringMongoConfig {
|
|||||||
@Primary
|
@Primary
|
||||||
@Bean
|
@Bean
|
||||||
public MongoDbFactory mongoDbFactory() throws Exception {
|
public MongoDbFactory mongoDbFactory() throws Exception {
|
||||||
List<ServerAddress> seeds = Arrays.asList(new ServerAddress(youfangUrl, youfangPort));
|
List<ServerAddress> seeds = Arrays.asList(
|
||||||
|
new ServerAddress("mongo.db.fangjia.com", 28017),
|
||||||
|
new ServerAddress("secondary.mongo.db.fangjia.com", 28017));
|
||||||
MongoClientOptions.Builder builder = new MongoClientOptions.Builder();
|
MongoClientOptions.Builder builder = new MongoClientOptions.Builder();
|
||||||
int connPerHost = 10;
|
int connPerHost = 10;
|
||||||
builder.connectionsPerHost(connPerHost);
|
builder.connectionsPerHost(connPerHost);
|
||||||
@@ -54,7 +58,9 @@ public class SpringMongoConfig {
|
|||||||
int timeOut = 10000;
|
int timeOut = 10000;
|
||||||
builder.connectTimeout(timeOut);
|
builder.connectTimeout(timeOut);
|
||||||
MongoClientOptions options = builder.build();
|
MongoClientOptions options = builder.build();
|
||||||
MongoClient mongoClient = new MongoClient(seeds, options);
|
List<MongoCredential> cres = new ArrayList<>();
|
||||||
|
cres.add(MongoCredential.createCredential("developer", "admin", "G00IDCDEV12385".toCharArray()));
|
||||||
|
MongoClient mongoClient = new MongoClient(seeds, cres, options);
|
||||||
return new SimpleMongoDbFactory(mongoClient, youfangName);
|
return new SimpleMongoDbFactory(mongoClient, youfangName);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -112,7 +112,7 @@ public class DistrictInfoJob implements SimpleJob{
|
|||||||
districtDetail.setHouseStructure((String) data.get("house_structure"));
|
districtDetail.setHouseStructure((String) data.get("house_structure"));
|
||||||
}
|
}
|
||||||
mongoTemplate.save(districtDetail);
|
mongoTemplate.save(districtDetail);
|
||||||
initDataToEs(districtDetail);
|
//initDataToEs(districtDetail);
|
||||||
}catch(Exception e) {
|
}catch(Exception e) {
|
||||||
logger.error("有房小区详情数据汇总异常", e);
|
logger.error("有房小区详情数据汇总异常", e);
|
||||||
DingDingMessageUtil.sendTextMessage("有房小区详情数据汇总异常:" + e.getMessage());
|
DingDingMessageUtil.sendTextMessage("有房小区详情数据汇总异常:" + e.getMessage());
|
||||||
|
|||||||
+2
-2
@@ -91,12 +91,12 @@
|
|||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.elasticsearch.client</groupId>
|
<groupId>org.elasticsearch.client</groupId>
|
||||||
<artifactId>transport</artifactId>
|
<artifactId>transport</artifactId>
|
||||||
<version>6.0.0</version>
|
<version>6.4.2</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.elasticsearch</groupId>
|
<groupId>org.elasticsearch</groupId>
|
||||||
<artifactId>elasticsearch</artifactId>
|
<artifactId>elasticsearch</artifactId>
|
||||||
<version>6.0.0</version>
|
<version>6.4.2</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|||||||
Reference in New Issue
Block a user