diff --git a/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/RestApiCallUtils.java b/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/RestApiCallUtils.java index ba92750..8572f56 100644 --- a/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/RestApiCallUtils.java +++ b/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/RestApiCallUtils.java @@ -22,7 +22,7 @@ public class RestApiCallUtils { * @return */ public static T getRestClient(Class apiType, String url) { - return Feign.builder().encoder(new JacksonEncoder()).decoder(new JacksonDecoder()).options(new Options(10 * 1000, 100 * 1000)) + return Feign.builder().encoder(new JacksonEncoder()).decoder(new JacksonDecoder()).options(new Options(10 * 1000, 120 * 1000)) .logLevel(Logger.Level.FULL) .target(apiType, url); } diff --git a/fangjia-job/fangjia-job-youfang/pom.xml b/fangjia-job/fangjia-job-youfang/pom.xml index d25c2c9..fd24980 100644 --- a/fangjia-job/fangjia-job-youfang/pom.xml +++ b/fangjia-job/fangjia-job-youfang/pom.xml @@ -77,6 +77,15 @@ lombok + + org.elasticsearch.client + transport + + + org.elasticsearch + elasticsearch + + diff --git a/fangjia-job/fangjia-job-youfang/src/main/java/com/fangjia/job/youfang/config/SpringEsConfig.java b/fangjia-job/fangjia-job-youfang/src/main/java/com/fangjia/job/youfang/config/SpringEsConfig.java new file mode 100644 index 0000000..99f72d5 --- /dev/null +++ b/fangjia-job/fangjia-job-youfang/src/main/java/com/fangjia/job/youfang/config/SpringEsConfig.java @@ -0,0 +1,34 @@ +package com.fangjia.job.youfang.config; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.transport.client.PreBuiltTransportClient; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class SpringEsConfig { + + @Value("${es.url}") + private String url; + + @SuppressWarnings("resource") + @Bean + public TransportClient transportClient() { + Settings settings = Settings.builder().put("cluster.name", "fangjia") + .put("client.transport.ping_timeout", "60s").put("client.transport.sniff", true).build(); + try { + return new PreBuiltTransportClient(settings) + .addTransportAddress(new TransportAddress(InetAddress.getByName(url), 9300)); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + return null; + } + +} diff --git a/fangjia-job/fangjia-job-youfang/src/main/java/com/fangjia/job/youfang/document/DistrictDetail.java b/fangjia-job/fangjia-job-youfang/src/main/java/com/fangjia/job/youfang/document/DistrictDetail.java index 54c78af..8c39c9a 100644 --- a/fangjia-job/fangjia-job-youfang/src/main/java/com/fangjia/job/youfang/document/DistrictDetail.java +++ b/fangjia-job/fangjia-job-youfang/src/main/java/com/fangjia/job/youfang/document/DistrictDetail.java @@ -260,6 +260,17 @@ public class DistrictDetail { */ private String rank; + @Field("lo") + private String location; + + public String getLocation() { + return location; + } + + public void setLocation(String location) { + this.location = location; + } + public String getId() { return id; } diff --git a/fangjia-job/fangjia-job-youfang/src/main/java/com/fangjia/job/youfang/job/DistrictInfoJob.java b/fangjia-job/fangjia-job-youfang/src/main/java/com/fangjia/job/youfang/job/DistrictInfoJob.java index 0cff7e0..fd635dd 100644 --- a/fangjia-job/fangjia-job-youfang/src/main/java/com/fangjia/job/youfang/job/DistrictInfoJob.java +++ b/fangjia-job/fangjia-job-youfang/src/main/java/com/fangjia/job/youfang/job/DistrictInfoJob.java @@ -1,12 +1,14 @@ package com.fangjia.job.youfang.job; +import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Semaphore; - +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.xcontent.XContentType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; @@ -20,6 +22,7 @@ import org.springframework.data.mongodb.core.query.Query; import com.dangdang.ddframe.job.api.ShardingContext; import com.dangdang.ddframe.job.api.simple.SimpleJob; import com.fangjia.job.common.util.DingDingMessageUtil; +import com.fangjia.job.common.util.JsonUtils; import com.fangjia.job.common.util.RestApiCallUtils; import com.fangjia.job.youfang.document.DistrictDetail; import com.fangjia.job.youfang.dto.DistrictDetailDto; @@ -47,6 +50,9 @@ public class DistrictInfoJob implements SimpleJob{ @Qualifier("seaweedMongoTemplate") private MongoTemplate seaweedMongoTemplate; + @Autowired + private TransportClient transportClient; + private ExecutorService fixPool = Executors.newFixedThreadPool(10); @Override @@ -56,7 +62,7 @@ public class DistrictInfoJob implements SimpleJob{ DBCollection collection = seaweedMongoTemplate.getCollection("seaweed"); DBCursor cursor = collection.find(new BasicDBObject("cat", "district").append("status", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("estate_type2", 1)).addOption(Bytes.QUERYOPTION_NOTIMEOUT); - final Semaphore semaphore = new Semaphore(10); + final Semaphore semaphore = new Semaphore(8); while (cursor.hasNext()) { final DBObject data = cursor.next(); try { @@ -94,6 +100,8 @@ public class DistrictInfoJob implements SimpleJob{ if(data.containsKey("estate_type2")){ estateType = (String) data.get("estate_type2"); } + districtDetail.setPoiTags(districtDetail.getPoiTags() == null ? new ArrayList() : districtDetail.getPoiTags()); + districtDetail.setLocation(districtDetail.getLat2() + "," + districtDetail.getLng2()); districtDetail.setDistrictKey(city + "_" + region + "_" + name); districtDetail.setCity(city); districtDetail.setRegion(region); @@ -101,6 +109,7 @@ public class DistrictInfoJob implements SimpleJob{ districtDetail.setEstateType(estateType); districtDetail.setModifyTime(new Date()); mongoTemplate.save(districtDetail); + initDataToEs(districtDetail); }catch(Exception e) { logger.error("有房小区详情数据汇总异常", e); DingDingMessageUtil.sendTextMessage("有房小区详情数据汇总异常:" + e.getMessage()); @@ -113,5 +122,10 @@ public class DistrictInfoJob implements SimpleJob{ logger.info("结束小区数据汇总任务"); } + private void initDataToEs(DistrictDetail districtDetail) { + transportClient.prepareIndex("youfang", "district_detail") + .setSource(JsonUtils.toJson(districtDetail), XContentType.JSON) + .setId(districtDetail.getId()).execute().actionGet(); + } } diff --git a/fangjia-job/fangjia-job-youfang/src/main/resources/application-dev.properties b/fangjia-job/fangjia-job-youfang/src/main/resources/application-dev.properties index f86c77c..c9f9502 100644 --- a/fangjia-job/fangjia-job-youfang/src/main/resources/application-dev.properties +++ b/fangjia-job/fangjia-job-youfang/src/main/resources/application-dev.properties @@ -23,4 +23,6 @@ zk.namespace=fangjia-job-youfang-test open.url=http://192.168.10.47:9180 -FANGJIA_API_TOKEN=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB \ No newline at end of file +FANGJIA_API_TOKEN=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB + +es.url=192.168.0.222 \ No newline at end of file diff --git a/fangjia-job/fangjia-job-youfang/src/main/resources/application-prod.properties b/fangjia-job/fangjia-job-youfang/src/main/resources/application-prod.properties index ec5843b..ec79330 100644 --- a/fangjia-job/fangjia-job-youfang/src/main/resources/application-prod.properties +++ b/fangjia-job/fangjia-job-youfang/src/main/resources/application-prod.properties @@ -26,4 +26,6 @@ zk.namespace=fangjia-job-youfang open.url=http://open.fangjia.com -FANGJIA_API_TOKEN=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB \ No newline at end of file +FANGJIA_API_TOKEN=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB + +es.url=222.73.196.132 \ No newline at end of file diff --git a/fangjia-job/fangjia-job-youfang/src/main/resources/application-test.properties b/fangjia-job/fangjia-job-youfang/src/main/resources/application-test.properties index f86c77c..c9f9502 100644 --- a/fangjia-job/fangjia-job-youfang/src/main/resources/application-test.properties +++ b/fangjia-job/fangjia-job-youfang/src/main/resources/application-test.properties @@ -23,4 +23,6 @@ zk.namespace=fangjia-job-youfang-test open.url=http://192.168.10.47:9180 -FANGJIA_API_TOKEN=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB \ No newline at end of file +FANGJIA_API_TOKEN=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB + +es.url=192.168.0.222 \ No newline at end of file diff --git a/fangjia-job/pom.xml b/fangjia-job/pom.xml index d41e3d0..77001fb 100644 --- a/fangjia-job/pom.xml +++ b/fangjia-job/pom.xml @@ -88,6 +88,17 @@ 1.16.20 + + org.elasticsearch.client + transport + 6.0.0 + + + org.elasticsearch + elasticsearch + 6.0.0 + +