diff --git a/fangjia-job/fangjia-job-basicdata/pom.xml b/fangjia-job/fangjia-job-basicdata/pom.xml
index 269291e..9b3c029 100644
--- a/fangjia-job/fangjia-job-basicdata/pom.xml
+++ b/fangjia-job/fangjia-job-basicdata/pom.xml
@@ -69,7 +69,20 @@
org.elasticsearch
elasticsearch
-
+
+
+
+ org.apache.commons
+ commons-lang3
+ 3.8.1
+
+
+
+ com.alibaba
+ fastjson
+ 1.2.51
+
+
diff --git a/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/config/SpringEs206Config.java b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/config/SpringEs206Config.java
new file mode 100644
index 0000000..d55b94c
--- /dev/null
+++ b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/config/SpringEs206Config.java
@@ -0,0 +1,50 @@
+package com.fangjia.job.basicdata.config;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+import org.apache.commons.lang3.StringUtils;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.transport.client.PreBuiltTransportClient;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class SpringEs206Config {
+
+ @Value("${es.206.url}")
+ private String url;
+
+ @Value("${es.206.cluster.name}")
+ private String clusterName;
+
+ @Bean
+ public String getUrl() {
+ return this.url;
+ }
+
+ @Bean
+ public String getClusterName() {
+ return this.clusterName;
+ }
+
+ @SuppressWarnings("resource")
+ @Bean
+ public TransportClient transportClient206() {
+ if(StringUtils.isEmpty(clusterName)) {
+ return null;
+ }
+ Settings settings = Settings.builder().put("cluster.name", clusterName)
+ .put("client.transport.ping_timeout", "60s").build();
+ try {
+ return new PreBuiltTransportClient(settings)
+ .addTransportAddress(new TransportAddress(InetAddress.getByName(url), 9300));
+ } catch (UnknownHostException e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+}
diff --git a/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/config/SpringMongoSeabirdConfig.java b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/config/SpringMongoSeabirdConfig.java
new file mode 100644
index 0000000..8b92630
--- /dev/null
+++ b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/config/SpringMongoSeabirdConfig.java
@@ -0,0 +1,58 @@
+package com.fangjia.job.basicdata.config;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+import org.springframework.data.mongodb.MongoDbFactory;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.SimpleMongoDbFactory;
+import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper;
+import org.springframework.data.mongodb.core.convert.MappingMongoConverter;
+import org.springframework.data.mongodb.core.mapping.MongoMappingContext;
+
+import com.mongodb.MongoClient;
+import com.mongodb.MongoClientOptions;
+import com.mongodb.ServerAddress;
+
+@Configuration
+public class SpringMongoSeabirdConfig {
+ @Value("${mongo.fangjia_tmp.url}")
+ private String fangjiatmpUrl;
+
+ @Value("${mongo.fangjia_tmp.port}")
+ private int fangjiatmpPort;
+
+ @Value("${mongo.fangjia_tmp.name}")
+ private String fangjiatmpName;
+
+ @Bean
+ MongoDbFactory fangjiatmpMongoDbFactory() throws Exception {
+ List seeds = Arrays.asList(new ServerAddress(fangjiatmpUrl, fangjiatmpPort));
+ MongoClientOptions.Builder builder = new MongoClientOptions.Builder();
+ int connPerHost = 10;
+ builder.connectionsPerHost(connPerHost);
+ int threadCount = 10;
+ builder.threadsAllowedToBlockForConnectionMultiplier(threadCount);
+ int maxWaitTime = 10000;
+ builder.maxWaitTime(maxWaitTime);
+ int timeOut = 10000;
+ builder.connectTimeout(timeOut);
+ MongoClientOptions options = builder.build();
+ MongoClient mongoClient = new MongoClient(seeds, options);
+ return new SimpleMongoDbFactory(mongoClient, fangjiatmpName);
+ }
+
+ @Bean
+ @SuppressWarnings("deprecation")
+ public MongoTemplate fangjiatempMongoTemplate() throws Exception {
+ //remove _class
+ MappingMongoConverter converter = new MappingMongoConverter(fangjiatmpMongoDbFactory(), new MongoMappingContext());
+ converter.setTypeMapper(new DefaultMongoTypeMapper(null));
+ MongoTemplate mongoTemplate = new MongoTemplate(fangjiatmpMongoDbFactory(), converter);
+ return mongoTemplate;
+ }
+}
diff --git a/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/job/SeabirdDataSyncEsJob.java b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/job/SeabirdDataSyncEsJob.java
new file mode 100644
index 0000000..b7a5eff
--- /dev/null
+++ b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/job/SeabirdDataSyncEsJob.java
@@ -0,0 +1,837 @@
+package com.fangjia.job.basicdata.job;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Queue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.DateFormatUtils;
+import org.apache.commons.lang3.time.DateUtils;
+import org.bson.types.ObjectId;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.transport.client.PreBuiltTransportClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Field;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.stereotype.Component;
+
+import com.dangdang.ddframe.job.api.ShardingContext;
+import com.dangdang.ddframe.job.api.simple.SimpleJob;
+import com.fangjia.job.basicdata.config.SpringEs206Config;
+import com.fangjia.job.common.util.DataUtil;
+import com.fangjia.job.common.util.DateUtilCustom;
+import com.fangjia.job.common.util.JsonUtils;
+import com.fangjia.job.common.util.SpringContextUtil;
+import com.mongodb.BasicDBObject;
+import com.mongodb.Bytes;
+import com.mongodb.DBCollection;
+import com.mongodb.DBCursor;
+import com.mongodb.DBObject;
+import com.mongodb.util.JSON;
+
+/**
+ * seabird数据导入ES任务
+ *
+ * @author weilei
+ *
+ * @date 2018-08-29
+ *
+ */
+@Component
+public class SeabirdDataSyncEsJob implements SimpleJob {
+
+ private Logger logger = LoggerFactory.getLogger(SeabirdDataSyncEsJob.class);
+
+ @Autowired
+ @Qualifier("fangjiatempMongoTemplate")
+ private MongoTemplate fangjiatempMongoTemplate;
+
+ @Autowired
+ @Qualifier("transportClient206")
+ private TransportClient transportClient_206;
+
+// @Autowired
+// private SpringEs206Config springEs206Config;
+
+// Map transportClientMap = new HashMap<>();
+ Map bulkRequestBuilderMap = new HashMap<>();
+ Map addedEsCountMap = new HashMap<>();
+
+// BulkRequestBuilder bulkRequest = null;
+ Date currentDate = null;
+ int BatchCount = 1000;
+ int WholeProcBatchTimes = 10 * BatchCount;
+
+ Queue removeData = new LinkedBlockingQueue<>(100 * BatchCount);
+ boolean allThreadEnd = false;
+
+ @Override
+ public void execute(ShardingContext shardingContext) {
+// this.bulkRequest = this.defaultTransportClient.prepareBulk();
+ currentDate = new Date();
+
+ DBCollection collection_seabird = fangjiatempMongoTemplate.getCollection("seabird");
+ println("availableProcessors == " + Runtime.getRuntime().availableProcessors());
+ ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
+ executorService.execute(new Runnable() {
+ @Override
+ public void run() {
+ while(true) {
+ if(removeData.size() == 0) {
+ try {
+ println("removeData.size() == 0, sleep 1 sed");
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ continue;
+ }
+ try {
+ if(removeData.size() > 0) {
+ if(removeData.size() % BatchCount == 0) {
+ println("removeData.size() == " + removeData.size());
+ }
+ String _id = removeData.poll();
+ if(StringUtils.isNotEmpty(_id)) {
+ DBObject q = new BasicDBObject("_id", new ObjectId(_id));
+ collection_seabird.remove(q);
+ }
+ }
+ } catch (Exception e) {
+
+ }
+ if(allThreadEnd) {
+ break;
+ }
+ }
+ }
+ });
+ executorService.shutdown();
+
+ int count = 0;
+ for (String cat : CATEGORIES) {
+ try {
+ count += processFromSpidersByCat(collection_seabird, cat);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ int judgeCount = 0;
+ while(judgeCount <= 10) {
+ if(removeData.size() <= 0) {
+ judgeCount++;
+ }else {
+ judgeCount = 0;
+ try {
+ Thread.sleep(10 * 1000);
+ } catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+ }
+ allThreadEnd = true;
+ try {
+ Thread.sleep(10 * 1000);
+ } catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
+ if(executorService.isTerminated()){
+ System.out.println("所有的子线程都结束了");
+ }
+
+// TransportClient transportClient = getCustomTransportClient(springEs206Config.getUrl(), "");
+
+ logger.info("导入结束,共导入:" + count);
+ }
+
+// public TransportClient getCustomTransportClient(String indexName) {
+// if(StringUtils.isEmpty(indexName)) {
+// return null;
+// }
+// if(StringUtils.isEmpty(springEs206Config.getClusterName())) {
+// return null;
+// }
+// if(transportClientMap.containsKey(indexName)) {
+// return transportClientMap.get(indexName);
+// }
+// Settings settings = Settings.builder().put("cluster.name", springEs206Config.getClusterName())
+// .put("client.transport.ping_timeout", "60s").build();
+// try {
+// String url = springEs206Config.getUrl();
+// TransportClient result = new PreBuiltTransportClient(settings).addTransportAddress(new TransportAddress(InetAddress.getByName(url), 9300));
+// transportClientMap.put(indexName, result);
+// return result;
+// } catch (UnknownHostException e) {
+// e.printStackTrace();
+// }
+// return null;
+// }
+
+ public void setBulkRequestBuilder(String indexName, BulkRequestBuilder builder) {
+ if(StringUtils.isEmpty(indexName) || builder == null) {
+ return;
+ }
+ bulkRequestBuilderMap.put(indexName, builder);
+ }
+ public BulkRequestBuilder getBulkRequestBuilder(String indexName) {
+ if(StringUtils.isEmpty(indexName)) {
+ return null;
+ }
+ return bulkRequestBuilderMap.get(indexName);
+// try {
+// TransportClient transportClient = getCustomTransportClient(indexName);
+// if(transportClient != null) {
+// Settings settings = Settings.builder().put("cluster.name", springEs206Config.getClusterName())
+// .put("client.transport.ping_timeout", "60s").build();
+// bulkRequest = transportClient.prepareBulk();
+// bulkRequest.add(transportClient.prepareIndex(indexName,"doc"));
+// bulkRequestBuilderMap.put(indexName, bulkRequest);
+// return bulkRequest;
+// }
+// } catch (Exception e) {
+// e.printStackTrace();
+// }
+// return null;
+ }
+
+// void clearCustomBulkRequestBuilder(String indexName) {
+// if(StringUtils.isEmpty(indexName)) {
+// return;
+// }
+// TransportClient transportClient = getCustomTransportClient(indexName);
+// if(transportClient == null) {
+// return;
+// }
+// if(bulkRequestBuilderMap.containsKey(indexName)) {
+// BulkRequestBuilder bulkRequest = bulkRequestBuilderMap.get(indexName);
+// if(bulkRequest != null) {
+// bulkRequest = transportClient.prepareBulk();
+// bulkRequest.add(transportClient.prepareIndex(indexName,"doc"));
+// bulkRequestBuilderMap.put(indexName, bulkRequest);
+// }
+// }
+// }
+
+ void allBulkRequestSubmit() {
+ for(Entry entry : bulkRequestBuilderMap.entrySet()) {
+ try {
+ BulkRequestBuilder bulkRequest = entry.getValue();
+ bulkRequest.execute().actionGet();
+ }catch(Exception ex) {
+ ex.printStackTrace();
+ }
+ }
+ }
+
+ DBCursor getCursor(DBCollection collection_seabird, String cat) {
+ try {
+ Query query = Query.query(Criteria.where("cat").is(cat).and("c_date").lt(this.currentDate));
+ return collection_seabird.find(query.getQueryObject()).batchSize(1000).addOption(Bytes.QUERYOPTION_NOTIMEOUT);
+ }catch(Exception ex) {
+ ex.printStackTrace();
+ return null;
+ }
+ }
+
+ int processFromSpidersByCat(DBCollection collection_seabird, String cat) {
+ if(collection_seabird == null || StringUtils.isEmpty(cat)) {
+ return 0;
+ }
+
+ int count = 0;
+ int wholeDR = 0;
+ int fc = 0;
+// List batchDelList = new ArrayList<>();
+// DBObject query_seabird = new BasicDBObject();
+// query_seabird.put("cat", cat);
+// DBObject projection = new BasicDBObject();
+// projection.put("city", 1);
+
+ DBCursor cursor_seabird = null;
+ while((cursor_seabird = getCursor(collection_seabird, cat)) == null) {
+ try {
+ Thread.sleep(60 * 1000);
+ } catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+ System.out.println(cursor_seabird.count());
+ while (cursor_seabird.hasNext()) {
+ count++;
+ DBObject dbObj_seabird = cursor_seabird.next();
+ if (dbObj_seabird == null || !dbObj_seabird.containsField(MongoObjectIDKey)
+ || dbObj_seabird.get(MongoObjectIDKey) == null
+ || StringUtils.isEmpty(dbObj_seabird.get(MongoObjectIDKey).toString())) {
+ continue;
+ }
+ String _id = dbObj_seabird.get(MongoObjectIDKey).toString();
+ while(removeData.offer(_id) == false) {
+ try {
+ System.out.println("The queue is full, sleep.............60 seds");
+ Thread.sleep(60 * 1000);
+ }catch(Exception ex) {
+ ex.printStackTrace();
+ }
+ }
+
+ if (isValidData(dbObj_seabird)) {
+ DBObject standardData = formatToStandardData(dbObj_seabird);
+ if (dataClean(standardData)) {
+
+ try {
+ Integer pDate = parseInt(standardData.get("p_date"));
+ if(pDate <= 0) {
+ continue;
+ }
+ Date publishDate = parseDate(String.valueOf(pDate), "yyyyMMdd");
+ if(publishDate == null) {
+ continue;
+ }
+ String indexName = "seawater-" + DateFormatUtils.format(publishDate, "yyyy");
+
+ int dr = 0;
+ if(addedEsCountMap.containsKey(indexName)) {
+ dr = addedEsCountMap.get(indexName);
+ }
+ dr++;
+ wholeDR++;
+ addedEsCountMap.put(indexName, dr);
+// println("indexName - " + indexName + ", addedEsCount == " + dr);
+
+
+ BulkRequestBuilder bulkRequest = getBulkRequestBuilder(indexName);
+ //transportClient_206.prepareBulk();
+ if(bulkRequest == null) {
+ bulkRequest = transportClient_206.prepareBulk();
+ setBulkRequestBuilder(indexName, bulkRequest);
+ }
+
+ String id = standardData.get("_id").toString();
+ standardData.removeField("_id");
+ bulkRequest.add(transportClient_206.prepareIndex(indexName,"_doc")
+ .setSource(JsonUtils.toJson(standardData), XContentType.JSON)
+ .setId(id));
+
+ if(dr >= BatchCount) {
+ bulkRequest.execute().actionGet();
+ bulkRequest = transportClient_206.prepareBulk();
+ setBulkRequestBuilder(indexName, bulkRequest);
+ addedEsCountMap.put(indexName, 0);
+ }
+ }catch(Exception ex) {
+ ex.printStackTrace();
+ }
+
+// String indexName = this.importData2Elastic(standardData.toMap());
+// if(StringUtils.isNoneEmpty(indexName)) {
+// dr++;
+// int addedCount = 0;
+// if(addedEsCountMap.containsKey(indexName)) {
+// addedCount = addedEsCountMap.get(indexName);
+// println("indexName - " + indexName + ", addedEsCount == " + addedCount);
+// if(addedCount >= BatchCount) {
+// BulkRequestBuilder bulkRequest = getCustomBulkRequestBuilder(indexName);
+// if(bulkRequest != null) {
+// try {
+// bulkRequest.execute().actionGet();
+// WholeProcBatchTimes--;
+// addedEsCountMap.put(indexName, 0);
+// logger.info("已导入:" + dr);
+// // 清空数据
+// clearCustomBulkRequestBuilder(indexName);
+// }catch(Exception ex) {
+// ex.printStackTrace();
+// }
+//
+// }
+// }
+// }
+// }
+ }
+ } else {
+ println("放弃:" + dbObj_seabird.toString());
+ fc++;
+ }
+
+ if (count % BatchCount == 0) {
+ for(String key : addedEsCountMap.keySet()) {
+ println("index name == " + key);
+ }
+ println("count == " + String.valueOf(count) + ", wholeDR == " + wholeDR + ", fc == " + fc);
+ }
+ }
+
+ if(wholeDR > 0) {
+// this.bulkRequest.execute().actionGet();
+ allBulkRequestSubmit();
+ }
+
+ println(count + "/add:${ac},upd:${uc},igr:${ic},err:${fc}\t" + DateUtilCustom.formatToTightYYYYMMDDhhmmss(new Date()));
+
+ return count;
+ }
+
+ void removeSeabird(DBCollection collection_seabird, List batchDelList) {
+ if (collection_seabird != null && batchDelList!= null && batchDelList.size() >= BatchCount) {
+ while(batchDelList.size() > 0) {
+ try {
+// String _id = batchDelList.get(0);
+ DBObject q = new BasicDBObject("_id", new ObjectId(batchDelList.get(0)));
+// DBObject old = collection_seabird.findOne(q);
+ collection_seabird.remove(q);
+ }catch(Exception ex){
+ ex.printStackTrace();
+ }
+ batchDelList.remove(0);
+ }
+ }
+ }
+
+ private boolean isValidData(DBObject dbo) {
+ String error_isValidData = "isValidData error -> ";
+
+ String url = dbo.get("url") != null ? dbo.get("url").toString() : null;
+// StringUtils.startsWithIgnoreCase(url, "http://")
+ if (StringUtils.isEmpty(url)) {
+ println(error_isValidData + "url is null");
+ return false;
+ }
+ if (!(StringUtils.startsWithIgnoreCase(url, "http://") || StringUtils.startsWithIgnoreCase(url, "https://"))
+ && StringUtils.contains(url, "fangjia.com") == false) {
+ println(error_isValidData + "url:" + url);
+ return false;
+ }
+
+ String city = dbo.get("city") != null ? dbo.get("city").toString() : null;
+ if (StringUtils.isEmpty(city) || city.length() > 10) {
+ println(error_isValidData + "city:" + city);
+ return false;
+ }
+
+ String region = dbo.get("region") != null ? dbo.get("region").toString() : null;
+ if (StringUtils.isEmpty(region) || region.length() > 10) {
+ println(error_isValidData + "region:" + region);
+ return false;
+ }
+
+ String district_name = dbo.get("district_name") != null ? dbo.get("district_name").toString() : null;
+ if (StringUtils.isEmpty(district_name) || district_name.length() > 30) {
+ println(error_isValidData + "district_name:" + district_name);
+ return false;
+ }
+
+ if (dbo.get("area") == null || isDouble(dbo.get("area")) == false) {
+ println(error_isValidData + "area:" + dbo.get("area"));
+ return false;
+ }
+ double d_area = parseDouble(dbo.get("area"));
+
+ if (checkDate(dbo.get("publish_date")) == false) {
+ println(error_isValidData + "publish_date:" + dbo.get("publish_date"));
+ return false;
+ }
+
+ if (before2007Date == null) {
+ before2007Date = parseFromTightYYYYMMDD("2007-01-01");
+ }
+ if (dbo.get("publish_date") != null) {
+ Date publish_date = parseFromTightYYYYMMDD(dbo.get("publish_date").toString());
+ if (publish_date.before(before2007Date)) {
+ println(error_isValidData + "publish_date:" + dbo.get("publish_date").toString());
+ return false;
+ }
+ }
+
+ String cat = dbo.get("cat") != null ? dbo.get("cat").toString() : null;
+ int total_price = parseInt(dbo.get("total_price"));
+ if (cat == "sell") {
+ if (d_area < 10 || d_area > 2000) {
+ println(error_isValidData + "cat:" + cat + ",area:" + d_area);
+ return false;
+ }
+ if (total_price < 100000 || total_price > 900000000) {
+ println(error_isValidData + "cat:" + cat + ",total_price:" + total_price);
+ return false;
+ }
+
+ double avgPrice = total_price / d_area;
+ if (avgPrice > 300000 || avgPrice < 1000) {
+ println(error_isValidData + "cat:" + cat + ",avgPrice:" + avgPrice);
+ return false;
+ }
+
+ } else if (cat == "rent") {
+ if (d_area < 5 || d_area > 2000) {
+ println(error_isValidData + "cat:" + cat + ", area:" + d_area);
+ return false;
+ }
+
+ if (total_price < 10 || total_price > 10000000) {
+ println(error_isValidData + "cat:" + cat + ",total_price:" + total_price);
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private DBObject formatToStandardData(DBObject dbo) {
+ if (StringUtils.isNotBlank((String) dbo.get("name")) && ((String) dbo.get("name")).length() > 50) {
+ dbo.put("name", dbo.get("name").toString().substring(0, 50));
+ }
+
+ if (StringUtils.isNotBlank((String) dbo.get("address")) && ((String) dbo.get("address")).length() > 50) {
+ dbo.removeField("address");
+ }
+
+ if (StringUtils.isNotBlank((String) dbo.get("mobile")) && ((String) dbo.get("mobile")).length() > 30) {
+ dbo.removeField("mobile");
+ }
+
+ if (StringUtils.isNotBlank((String) dbo.get("estate_charge"))
+ && ((String) dbo.get("estate_charge")).length() > 40) {
+ dbo.removeField("estate_charge");
+ }
+
+ String url = processUrl(dbo.get("url") != null ? dbo.get("url").toString() : "");
+
+ String id = dbo.get(MongoObjectIDKey).toString();
+// if (url.indexOf("fangjia.com") > -1) {
+// id = dbo.get(MongoObjectIDKey).toString();
+// }else {
+// id = DigestUtils.md5Hex(url + "#" + url.hashCode());
+// }
+
+ DBObject standardObj = new BasicDBObject();
+ standardObj.put(MongoObjectIDKey, id);
+ standardObj.put("url", url);
+ standardObj.put("cat", dbo.get("cat"));
+ standardObj.put("city", dbo.get("city").toString().trim());
+ standardObj.put("region", dbo.get("region").toString().trim());
+ standardObj.put("district_name", DataUtil.toDBCcase(dbo.get("district_name").toString()).trim());
+ standardObj.put("p_date", DateUtilCustom.formatToTightYYYYMMDD(dbo.get("publish_date")));
+ standardObj.put("c_date", DateUtilCustom.formatToTightYYYYMMDD(dbo.get("c_date")));
+
+ // 数字类型的字段
+ for (String key : NUMBER_FIELDS) {
+ Object val = dbo.get(key);
+ if (val != null && val instanceof Number) {
+ standardObj.put(key, val);
+ }
+ }
+
+ // 文本类型的字段
+ for (String key : TEXT_FIELDS) {
+ Object val = dbo.get(key);
+ // 判断文本字段如果太长,就不保存该字段
+ if (val != null && val instanceof String && val.toString().length() <= 1024) {
+ standardObj.put(key, val.toString());
+ }
+ }
+
+ // 其他字段
+ for (String key : OTHER_FIELDS) {
+ Object val = dbo.get(key);
+ if (val != null) {
+ standardObj.put(key, val);
+ }
+ }
+// standardObj.put("md5_hash", getMd5Hash(standardObj))
+ return standardObj;
+ }
+
+ String processUrl(String url) {
+ String root = DataUtil.urlToSiteName(url);
+ if ("58.com".equals(root)) {
+ // http://huizhou.58.com/ershoufang/22416405318175x.shtml?psid=196886603191045603944627704&entinfo=22416405318175_0
+ if (url.indexOf("?") > -1) {
+ url = StringUtils.substringBefore(url, "?");
+ }
+ } else if ("anjuke.com".equals(root)) {
+ // http://wuhan.anjuke.com/prop/view/A417931448?from=comm_one-saleMetro&spread=commsearch_p
+ if (url.indexOf("?") > -1) {
+ url = StringUtils.substringBefore(url, "?");
+ }
+ } else if ("leju.com".equals(root)) {
+ // http://bj.esf.leju.com/detail/217448358#zn=pc-house-36
+ if (url.indexOf("#") > -1) {
+ url = StringUtils.substringBefore(url, "#");
+ }
+ }
+ return url;
+ }
+
+ static String err_dataClean = "dataClean() error -> ";
+
+ boolean dataClean(DBObject dbo) {
+ if (dbo == null) {
+ println(err_dataClean + " -> dbo == null");
+ return false;
+ }
+
+ if (dbo.get("_id") == null) {
+ println(err_dataClean + " -> _id == null");
+ return false;
+ }
+
+ if (isInteger(dbo.get("p_date")) == false) {
+ println(err_dataClean + " -> p_date");
+ return false;
+ }
+
+ List keys = new ArrayList(dbo.keySet());
+ for (String key : keys) {
+ if (!fields.contains(key)) {
+ dbo.removeField(key);
+ }
+ }
+
+ Integer pDate = parseInt(dbo.get("p_date"));
+ Date publishDate = parseDate(String.valueOf(pDate), "yyyyMMdd");
+ if (publishDate == null) {
+ println(err_dataClean + " -> publishDate == null");
+ return false;
+ }
+ dbo.put("publish_date", publishDate);
+
+ if (dbo.get("m_date") == null) {
+ dbo.put("m_date", dbo.get("c_date"));
+ }
+ if (dbo.get("m_date") == null) {
+ dbo.put("m_date", publishDate);
+ }
+
+ if(dbo.get("url") != null) {
+ String url = dbo.get("url").toString();
+ String root = DataUtil.urlToSiteName(url);
+ if (StringUtils.isNotBlank(root)) {
+ dbo.put("root", root);
+ }
+ }
+
+ return true;
+ }
+
+// String importData2Elastic(Map map){
+// if(map == null || map.size() == 0 || map.get("_id") == null || map.get("p_date") == null) {
+// return null;
+// }
+//
+// try{
+// Integer pDate = parseInt(map.get("p_date"));
+// if(pDate <= 0) {
+// return null;
+// }
+// Date publishDate = parseDate(String.valueOf(pDate), "yyyyMMdd");
+// if(publishDate == null) {
+// return null;
+// }
+// String indexName = "seawater-" + DateFormatUtils.format(publishDate, "yyyy");
+//
+// String id = map.get("_id").toString();
+//
+// map.remove("_id");
+// map.remove("c_date");
+// map.remove("md5_hash");
+//
+// TransportClient transportClient = getCustomTransportClient(indexName);
+// BulkRequestBuilder bulkRequest = getCustomBulkRequestBuilder(indexName);
+// if(transportClient != null && bulkRequest != null) {
+//// this.bulkRequest.add(transportClient.prepareIndex(indexName,"doc")
+// bulkRequest.add(transportClient.prepareIndex(indexName,"doc")
+// .setSource(JsonUtils.toJson(map), XContentType.JSON)
+// .setId(id));
+//
+// int addedCount = 0;
+// if(addedEsCountMap.containsKey(indexName)) {
+// addedCount = addedEsCountMap.get(indexName);
+// }
+// addedCount++;
+// addedEsCountMap.put(indexName, addedCount);
+// }
+//
+// return indexName;
+// }catch(Exception ex){
+// println("importData2Elastic error -> map:" + com.alibaba.fastjson.JSON.toJSONString(map) + ", exception:" + ex.getMessage());
+// ex.printStackTrace();
+// }
+// return null;
+// }
+
+ static void println(String msg) {
+ System.out.println(msg);
+ }
+
+ static Date getCSTDate(Object obj) {
+ if (obj == null) {
+ return null;
+ }
+ try {
+ SimpleDateFormat sdf = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy", Locale.US);
+ return sdf.parse(obj.toString());
+ } catch (Exception e) {
+ }
+ return null;
+ }
+
+ static boolean checkDate(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ SimpleDateFormat sd = new SimpleDateFormat(DATE_PATTERN);
+ try {
+ sd.setLenient(false);
+ sd.parse(obj.toString());
+ } catch (Exception e) {
+ if(getCSTDate(obj.toString()) != null) {
+ return true;
+ }
+
+ return false;
+ }
+ return true;
+ }
+
+ static boolean checkDateTime(String str) {
+ SimpleDateFormat sd = new SimpleDateFormat(DATETIME_PATTERN);
+ try {
+ sd.setLenient(false);
+ sd.parse(str);
+ } catch (Exception e) {
+ if(getCSTDate(str) != null) {
+ return true;
+ }
+ }
+ return true;
+ }
+
+ public static Date parseFromTightYYYYMMDD(String s) {
+ if (s == null || s.length() == 0) {
+ return null;
+ }
+ SimpleDateFormat format = new SimpleDateFormat(DATE_PATTERN);
+ try {
+ return format.parse(s);
+ } catch (ParseException e) {
+ return getCSTDate(s);
+ }
+ }
+
+ public static boolean isDouble(Object input) {
+ if(input == null) {
+ return false;
+ }
+ try {
+ Double.parseDouble(input.toString());
+ return true;
+ }catch(Exception ex) {
+ return false;
+ }
+ }
+
+ public static double parseDouble(Object obj) {
+ if (obj == null) {
+ return 0;
+ }
+ try {
+ return Double.parseDouble(obj.toString());
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return 0;
+ }
+
+ public static boolean isInteger(Object input) {
+ if(input == null) {
+ return false;
+ }
+ try {
+ Integer.parseInt(input.toString());
+ return true;
+ }catch(Exception ex) {
+ return false;
+ }
+ }
+
+ public static int parseInt(Object obj) {
+ if (obj == null) {
+ return 0;
+ }
+ try {
+ return Integer.parseInt(obj.toString());
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return 0;
+ }
+
+ public static Date parseDate(String dateStr, String pattern) {
+ Date date = null;
+ if (StringUtils.isNotBlank(dateStr)) {
+ SimpleDateFormat sdf = new SimpleDateFormat(pattern);
+ try {
+ date = sdf.parse(dateStr);
+ } catch (ParseException e) {
+ }
+ }
+ return date;
+ }
+
+ static Date before2007Date = null;
+
+ public static final String DATE_PATTERN = "yyyy-MM-dd";
+ public static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss";
+
+ static String MongoObjectIDKey = "_id";
+
+ static List fields = Arrays.asList("_id", "cat", "city", "region", "district_name", "total_price", "area",
+ "p_date", "url", "estate_type", "height", "floor", "room", "hall", "toilet", "fitment", "direction",
+ "rent_type", "user_id", "build_type", "build_date", "estate_charge", "status", "address", "block", "name",
+ "contact_people", "mobile", "company", "m_date", "floor_info", "community_support", "house_number",
+ "down_payment", "only_one", "buy_year", "unit_number", "unit_type", "owner_attitude", "service_desc",
+ "check_in_time", "rent_type", "pay_type", "house_allocation", "house_require", "house_advantage", "metro",
+ "line", "subject");
+
+ static String[] CATEGORIES = new String[] { "office_rent", "sell", "rent", "office_rent", "office_sell",
+ "shop_rent", "shop_sell" };
+ static String[] NUMBER_FIELDS = new String[] { "total_price", "area", "height", "floor", "room", "hall", "toilet",
+ "age" };
+ static String[] TEXT_FIELDS = new String[] { "name", "address", "block", "fitment", "estate_type", "mobile",
+ "build_type", "build_date", "company", "direction", "contact_people", "rent_type", "floor_info",
+ "house_number", "down_payment", "only_one", "buy_year", "unit_number", "unit_type", "owner_attitude",
+ "service_desc", "community_support", "description", "check_in_time", "house_allocation", "house_require",
+ "house_advantage", "pay_type", "metro", "line", "subject", "agency" };
+ static String[] OTHER_FIELDS = new String[] { "user_id", "publish_date" };
+
+}
diff --git a/fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties
index dc3c82d..0abfbe4 100644
--- a/fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties
+++ b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties
@@ -2,6 +2,10 @@ mongo.business.url=114.80.150.198
mongo.business.name=business
mongo.business.port=27017
+mongo.fangjia_tmp.url=192.168.0.209
+mongo.fangjia_tmp.name=fangjia_tmp
+mongo.fangjia_tmp.port=55007
+
spring.datasource.log.url=jdbc:mysql://localhost:3306/batch_log
spring.datasource.log.username=root
spring.datasource.log.password=123456
@@ -11,4 +15,7 @@ zk.server-lists=192.168.10.124:2181
zk.namespace=fangjia-job-basicdata
es.url=192.168.0.45
-es.cluster.name=fangjia
\ No newline at end of file
+es.cluster.name=fangjia
+
+es.206.url=192.168.0.206
+es.206.cluster.name=fangjia
\ No newline at end of file
diff --git a/fangjia-job/fangjia-job-basicdata/src/main/resources/application-prod.properties b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-prod.properties
index 37cf237..9a70863 100644
--- a/fangjia-job/fangjia-job-basicdata/src/main/resources/application-prod.properties
+++ b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-prod.properties
@@ -2,6 +2,10 @@ mongo.business.url=mongo.db.fangjia.com
mongo.business.name=business
mongo.business.port=27017
+mongo.fangjia_tmp.url=192.168.0.209
+mongo.fangjia_tmp.name=fangjia_tmp
+mongo.fangjia_tmp.port=55007
+
spring.datasource.log.url=jdbc:mysql://mysql.db.fangjia.com:4306/batch_log
spring.datasource.log.username=sa
spring.datasource.log.password=G00jia
@@ -11,4 +15,7 @@ zk.server-lists=192.168.0.207:2181,192.168.0.24:2181,192.168.0.25:2181
zk.namespace=fangjia-job-basicdata
es.url=192.168.0.45
-es.cluster.name=fangjia
\ No newline at end of file
+es.cluster.name=fangjia
+
+es.206.url=192.168.0.206
+es.206.cluster.name=fangjia
\ No newline at end of file
diff --git a/fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext.xml b/fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext.xml
index c4450b0..02060fe 100644
--- a/fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext.xml
+++ b/fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext.xml
@@ -43,8 +43,15 @@
+ overwrite="true" event-trace-rdb-data-source="elasticJobLog">
+
+ -->
+
+
\ No newline at end of file
diff --git a/fangjia-job/fangjia-job-common/pom.xml b/fangjia-job/fangjia-job-common/pom.xml
index dc55693..8b42706 100644
--- a/fangjia-job/fangjia-job-common/pom.xml
+++ b/fangjia-job/fangjia-job-common/pom.xml
@@ -33,5 +33,12 @@
io.github.openfeign
feign-core
+
+
+
+ org.apache.commons
+ commons-lang3
+ 3.8.1
+
diff --git a/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/DataUtil.java b/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/DataUtil.java
index b67ffee..c0501e2 100644
--- a/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/DataUtil.java
+++ b/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/DataUtil.java
@@ -2,6 +2,8 @@ package com.fangjia.job.common.util;
import java.math.BigDecimal;
+import org.apache.commons.lang3.StringUtils;
+
public class DataUtil {
public static double getDoubleRound(int scale, double dSource) {
@@ -22,5 +24,55 @@ public class DataUtil {
return divisor(n,m % n);
}
}
+
+ /**
+ * 全角转半角
+ *
+ * @param input
+ * @return
+ */
+ public static String toDBCcase(String input) {
+ if (StringUtils.isBlank(input))
+ return "";
+ char c[] = input.toCharArray();
+ for (int i = 0; i < c.length; i++) {
+ if (c[i] == '\u3000') {
+ c[i] = ' ';
+ } else if (c[i] > '\uFF00' && c[i] < '\uFF5F') {
+ c[i] = (char) (c[i] - 65248);
+ }
+ }
+ return new String(c);
+ }
+
+ public static String[] SITE_SUFFIXS = { ".com.cn", ".com", ".cn", ".net", ".org", ".cc", ".tm" };
+ public static String urlToSiteName(String url) {
+ if (StringUtils.isBlank(url)) {
+ return null;
+ }
+
+ for (String suffix : SITE_SUFFIXS) {
+ if (url.indexOf(suffix) > -1) {
+ String s = StringUtils.substringBefore(url, suffix);
+ if (s.indexOf(".") > -1) {
+ s = StringUtils.substringAfterLast(s, ".");
+ if (StringUtils.isNotBlank(s)) {
+ return s + suffix;
+ }
+ } else if (s.indexOf("://") > -1) {
+ s = StringUtils.substringAfterLast(s, "://");
+ if (StringUtils.isNotBlank(s)) {
+ return s + suffix;
+ }
+ } else if (s.indexOf("://") == -1 && s.indexOf(".") == -1) {
+ return s + suffix;
+ }
+
+ return null;
+ }
+ }
+
+ return null;
+ }
}
diff --git a/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/DateUtilCustom.java b/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/DateUtilCustom.java
index dd50763..a6fdee5 100644
--- a/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/DateUtilCustom.java
+++ b/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/DateUtilCustom.java
@@ -178,6 +178,38 @@ public class DateUtilCustom {
return false;
}
}
+
+ /**
+ * 转化日期成yyyymmdd的String格式
+ *
+ * @param date
+ * @return
+ */
+ public static String formatToTightYYYYMMDD(Object dateObj) {
+ if (dateObj == null)
+ return "";
+ String result = "";
+ try {
+ SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
+ result = format.format((Date)dateObj);
+ } catch (Exception e) {
+ return "";
+ }
+ return result;
+ }
+
+ public static String formatToTightYYYYMMDDhhmmss(Date date) {
+ if (date == null)
+ return "";
+ String result = "";
+ try {
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ result = format.format(date);
+ } catch (Exception e) {
+ return "";
+ }
+ return result;
+ }
public static void main(String[] args) {
diff --git a/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/SpringContextUtil.java b/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/SpringContextUtil.java
new file mode 100644
index 0000000..6230f5d
--- /dev/null
+++ b/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/SpringContextUtil.java
@@ -0,0 +1,38 @@
+package com.fangjia.job.common.util;
+
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import org.springframework.stereotype.Component;
+
+@Component
+public class SpringContextUtil implements ApplicationContextAware {
+
+ static SpringContextUtil Instance = new SpringContextUtil();
+
+ private ApplicationContext applicationContext;
+
+ public SpringContextUtil() {
+ }
+
+ public void setApplicationContext(ApplicationContext applicationContext) {
+ if (applicationContext != null) {
+ Instance.applicationContext = applicationContext;
+ }
+ }
+
+ public static T getBean(Class clzName) throws BeansException {
+ return Instance.applicationContext.getBean(clzName);
+ }
+
+
+
+ public static ApplicationContext getApplicationContext() {
+ return Instance.applicationContext;
+ }
+
+ public static Object getBean(String name, Class requiredType) {
+ return Instance.applicationContext.getBean(name, requiredType);
+ }
+
+}
\ No newline at end of file