Merge branch 'FSHJOB1.2' into 'master'
小区详情数据汇总存入ES See merge request !30
This commit is contained in:
+1
-1
@@ -22,7 +22,7 @@ public class RestApiCallUtils {
|
||||
* @return
|
||||
*/
|
||||
public static <T> T getRestClient(Class<T> apiType, String url) {
|
||||
return Feign.builder().encoder(new JacksonEncoder()).decoder(new JacksonDecoder()).options(new Options(10 * 1000, 100 * 1000))
|
||||
return Feign.builder().encoder(new JacksonEncoder()).decoder(new JacksonDecoder()).options(new Options(10 * 1000, 120 * 1000))
|
||||
.logLevel(Logger.Level.FULL)
|
||||
.target(apiType, url);
|
||||
}
|
||||
|
||||
@@ -77,6 +77,15 @@
|
||||
<artifactId>lombok</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.client</groupId>
|
||||
<artifactId>transport</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch</groupId>
|
||||
<artifactId>elasticsearch</artifactId>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
package com.fangjia.job.youfang.config;
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
|
||||
import org.elasticsearch.client.transport.TransportClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.transport.TransportAddress;
|
||||
import org.elasticsearch.transport.client.PreBuiltTransportClient;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
public class SpringEsConfig {
|
||||
|
||||
@Value("${es.url}")
|
||||
private String url;
|
||||
|
||||
@SuppressWarnings("resource")
|
||||
@Bean
|
||||
public TransportClient transportClient() {
|
||||
Settings settings = Settings.builder().put("cluster.name", "fangjia")
|
||||
.put("client.transport.ping_timeout", "60s").put("client.transport.sniff", true).build();
|
||||
try {
|
||||
return new PreBuiltTransportClient(settings)
|
||||
.addTransportAddress(new TransportAddress(InetAddress.getByName(url), 9300));
|
||||
} catch (UnknownHostException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
+11
@@ -260,6 +260,17 @@ public class DistrictDetail {
|
||||
*/
|
||||
private String rank;
|
||||
|
||||
@Field("lo")
|
||||
private String location;
|
||||
|
||||
public String getLocation() {
|
||||
return location;
|
||||
}
|
||||
|
||||
public void setLocation(String location) {
|
||||
this.location = location;
|
||||
}
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
+16
-2
@@ -1,12 +1,14 @@
|
||||
package com.fangjia.job.youfang.job;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Semaphore;
|
||||
|
||||
import org.elasticsearch.client.transport.TransportClient;
|
||||
import org.elasticsearch.common.xcontent.XContentType;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
@@ -20,6 +22,7 @@ import org.springframework.data.mongodb.core.query.Query;
|
||||
import com.dangdang.ddframe.job.api.ShardingContext;
|
||||
import com.dangdang.ddframe.job.api.simple.SimpleJob;
|
||||
import com.fangjia.job.common.util.DingDingMessageUtil;
|
||||
import com.fangjia.job.common.util.JsonUtils;
|
||||
import com.fangjia.job.common.util.RestApiCallUtils;
|
||||
import com.fangjia.job.youfang.document.DistrictDetail;
|
||||
import com.fangjia.job.youfang.dto.DistrictDetailDto;
|
||||
@@ -47,6 +50,9 @@ public class DistrictInfoJob implements SimpleJob{
|
||||
@Qualifier("seaweedMongoTemplate")
|
||||
private MongoTemplate seaweedMongoTemplate;
|
||||
|
||||
@Autowired
|
||||
private TransportClient transportClient;
|
||||
|
||||
private ExecutorService fixPool = Executors.newFixedThreadPool(10);
|
||||
|
||||
@Override
|
||||
@@ -56,7 +62,7 @@ public class DistrictInfoJob implements SimpleJob{
|
||||
DBCollection collection = seaweedMongoTemplate.getCollection("seaweed");
|
||||
DBCursor cursor = collection.find(new BasicDBObject("cat", "district").append("status", 0),
|
||||
new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("estate_type2", 1)).addOption(Bytes.QUERYOPTION_NOTIMEOUT);
|
||||
final Semaphore semaphore = new Semaphore(10);
|
||||
final Semaphore semaphore = new Semaphore(8);
|
||||
while (cursor.hasNext()) {
|
||||
final DBObject data = cursor.next();
|
||||
try {
|
||||
@@ -94,6 +100,8 @@ public class DistrictInfoJob implements SimpleJob{
|
||||
if(data.containsKey("estate_type2")){
|
||||
estateType = (String) data.get("estate_type2");
|
||||
}
|
||||
districtDetail.setPoiTags(districtDetail.getPoiTags() == null ? new ArrayList<String>() : districtDetail.getPoiTags());
|
||||
districtDetail.setLocation(districtDetail.getLat2() + "," + districtDetail.getLng2());
|
||||
districtDetail.setDistrictKey(city + "_" + region + "_" + name);
|
||||
districtDetail.setCity(city);
|
||||
districtDetail.setRegion(region);
|
||||
@@ -101,6 +109,7 @@ public class DistrictInfoJob implements SimpleJob{
|
||||
districtDetail.setEstateType(estateType);
|
||||
districtDetail.setModifyTime(new Date());
|
||||
mongoTemplate.save(districtDetail);
|
||||
initDataToEs(districtDetail);
|
||||
}catch(Exception e) {
|
||||
logger.error("有房小区详情数据汇总异常", e);
|
||||
DingDingMessageUtil.sendTextMessage("有房小区详情数据汇总异常:" + e.getMessage());
|
||||
@@ -113,5 +122,10 @@ public class DistrictInfoJob implements SimpleJob{
|
||||
logger.info("结束小区数据汇总任务");
|
||||
}
|
||||
|
||||
private void initDataToEs(DistrictDetail districtDetail) {
|
||||
transportClient.prepareIndex("youfang", "district_detail")
|
||||
.setSource(JsonUtils.toJson(districtDetail), XContentType.JSON)
|
||||
.setId(districtDetail.getId()).execute().actionGet();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -23,4 +23,6 @@ zk.namespace=fangjia-job-youfang-test
|
||||
|
||||
|
||||
open.url=http://192.168.10.47:9180
|
||||
FANGJIA_API_TOKEN=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB
|
||||
FANGJIA_API_TOKEN=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB
|
||||
|
||||
es.url=192.168.0.222
|
||||
@@ -26,4 +26,6 @@ zk.namespace=fangjia-job-youfang
|
||||
|
||||
|
||||
open.url=http://open.fangjia.com
|
||||
FANGJIA_API_TOKEN=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB
|
||||
FANGJIA_API_TOKEN=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB
|
||||
|
||||
es.url=222.73.196.132
|
||||
@@ -23,4 +23,6 @@ zk.namespace=fangjia-job-youfang-test
|
||||
|
||||
|
||||
open.url=http://192.168.10.47:9180
|
||||
FANGJIA_API_TOKEN=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB
|
||||
FANGJIA_API_TOKEN=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB
|
||||
|
||||
es.url=192.168.0.222
|
||||
@@ -88,6 +88,17 @@
|
||||
<version>1.16.20</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.client</groupId>
|
||||
<artifactId>transport</artifactId>
|
||||
<version>6.0.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch</groupId>
|
||||
<artifactId>elasticsearch</artifactId>
|
||||
<version>6.0.0</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user