From 1814fb5d7ec4874b3c90ca8fbdd40a46f751b8f3 Mon Sep 17 00:00:00 2001 From: weilei Date: Thu, 18 Oct 2018 15:00:30 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E7=88=AC=E8=99=AB=E6=95=B0=E6=8D=AE(mongo-?= =?UTF-8?q?209:55007)=E5=AF=BC=E5=85=A5=E5=88=B0Es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fangjia-job/fangjia-job-common/pom.xml | 7 +++ .../com/fangjia/job/common/util/DataUtil.java | 52 +++++++++++++++++++ .../job/common/util/DateUtilCustom.java | 32 ++++++++++++ 3 files changed, 91 insertions(+) diff --git a/fangjia-job/fangjia-job-common/pom.xml b/fangjia-job/fangjia-job-common/pom.xml index dc55693..8b42706 100644 --- a/fangjia-job/fangjia-job-common/pom.xml +++ b/fangjia-job/fangjia-job-common/pom.xml @@ -33,5 +33,12 @@ io.github.openfeign feign-core + + + + org.apache.commons + commons-lang3 + 3.8.1 + diff --git a/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/DataUtil.java b/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/DataUtil.java index b67ffee..c0501e2 100644 --- a/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/DataUtil.java +++ b/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/DataUtil.java @@ -2,6 +2,8 @@ package com.fangjia.job.common.util; import java.math.BigDecimal; +import org.apache.commons.lang3.StringUtils; + public class DataUtil { public static double getDoubleRound(int scale, double dSource) { @@ -22,5 +24,55 @@ public class DataUtil { return divisor(n,m % n); } } + + /** + * 全角转半角 + * + * @param input + * @return + */ + public static String toDBCcase(String input) { + if (StringUtils.isBlank(input)) + return ""; + char c[] = input.toCharArray(); + for (int i = 0; i < c.length; i++) { + if (c[i] == '\u3000') { + c[i] = ' '; + } else if (c[i] > '\uFF00' && c[i] < '\uFF5F') { + c[i] = (char) (c[i] - 65248); + } + } + return new String(c); + } + + public static String[] SITE_SUFFIXS = { ".com.cn", ".com", ".cn", ".net", ".org", ".cc", ".tm" }; + public static String urlToSiteName(String url) { + if (StringUtils.isBlank(url)) { + return null; + } + + for (String suffix : SITE_SUFFIXS) { + if (url.indexOf(suffix) > -1) { + String s = StringUtils.substringBefore(url, suffix); + if (s.indexOf(".") > -1) { + s = StringUtils.substringAfterLast(s, "."); + if (StringUtils.isNotBlank(s)) { + return s + suffix; + } + } else if (s.indexOf("://") > -1) { + s = StringUtils.substringAfterLast(s, "://"); + if (StringUtils.isNotBlank(s)) { + return s + suffix; + } + } else if (s.indexOf("://") == -1 && s.indexOf(".") == -1) { + return s + suffix; + } + + return null; + } + } + + return null; + } } diff --git a/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/DateUtilCustom.java b/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/DateUtilCustom.java index dd50763..a6fdee5 100644 --- a/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/DateUtilCustom.java +++ b/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/DateUtilCustom.java @@ -178,6 +178,38 @@ public class DateUtilCustom { return false; } } + + /** + * 转化日期成yyyymmdd的String格式 + * + * @param date + * @return + */ + public static String formatToTightYYYYMMDD(Object dateObj) { + if (dateObj == null) + return ""; + String result = ""; + try { + SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd"); + result = format.format((Date)dateObj); + } catch (Exception e) { + return ""; + } + return result; + } + + public static String formatToTightYYYYMMDDhhmmss(Date date) { + if (date == null) + return ""; + String result = ""; + try { + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + result = format.format(date); + } catch (Exception e) { + return ""; + } + return result; + } public static void main(String[] args) { From d1c1180732485a041a653329496a66c75d743aef Mon Sep 17 00:00:00 2001 From: weilei Date: Thu, 18 Oct 2018 15:00:44 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E7=88=AC=E8=99=AB=E6=95=B0=E6=8D=AE(mongo-?= =?UTF-8?q?209:55007)=E5=AF=BC=E5=85=A5=E5=88=B0Es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../job/common/util/SpringContextUtil.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/SpringContextUtil.java diff --git a/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/SpringContextUtil.java b/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/SpringContextUtil.java new file mode 100644 index 0000000..6230f5d --- /dev/null +++ b/fangjia-job/fangjia-job-common/src/main/java/com/fangjia/job/common/util/SpringContextUtil.java @@ -0,0 +1,38 @@ +package com.fangjia.job.common.util; + +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.stereotype.Component; + +@Component +public class SpringContextUtil implements ApplicationContextAware { + + static SpringContextUtil Instance = new SpringContextUtil(); + + private ApplicationContext applicationContext; + + public SpringContextUtil() { + } + + public void setApplicationContext(ApplicationContext applicationContext) { + if (applicationContext != null) { + Instance.applicationContext = applicationContext; + } + } + + public static T getBean(Class clzName) throws BeansException { + return Instance.applicationContext.getBean(clzName); + } + + + + public static ApplicationContext getApplicationContext() { + return Instance.applicationContext; + } + + public static Object getBean(String name, Class requiredType) { + return Instance.applicationContext.getBean(name, requiredType); + } + +} \ No newline at end of file From 66677b9c1ff2256795333dafcb5b51152152ef61 Mon Sep 17 00:00:00 2001 From: weilei Date: Thu, 18 Oct 2018 15:05:07 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E7=88=AC=E8=99=AB=E6=95=B0=E6=8D=AE(mongo-?= =?UTF-8?q?209:55007)=E5=AF=BC=E5=85=A5=E5=88=B0Es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fangjia-job/fangjia-job-basicdata/pom.xml | 15 +- .../basicdata/config/SpringEs206Config.java | 50 ++ .../config/SpringMongoSeabirdConfig.java | 58 ++ .../basicdata/job/SeabirdDataSyncEsJob.java | 837 ++++++++++++++++++ .../main/resources/application-dev.properties | 19 +- 5 files changed, 972 insertions(+), 7 deletions(-) create mode 100644 fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/config/SpringEs206Config.java create mode 100644 fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/config/SpringMongoSeabirdConfig.java create mode 100644 fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/job/SeabirdDataSyncEsJob.java diff --git a/fangjia-job/fangjia-job-basicdata/pom.xml b/fangjia-job/fangjia-job-basicdata/pom.xml index 269291e..9b3c029 100644 --- a/fangjia-job/fangjia-job-basicdata/pom.xml +++ b/fangjia-job/fangjia-job-basicdata/pom.xml @@ -69,7 +69,20 @@ org.elasticsearch elasticsearch - + + + + org.apache.commons + commons-lang3 + 3.8.1 + + + + com.alibaba + fastjson + 1.2.51 + + diff --git a/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/config/SpringEs206Config.java b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/config/SpringEs206Config.java new file mode 100644 index 0000000..d55b94c --- /dev/null +++ b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/config/SpringEs206Config.java @@ -0,0 +1,50 @@ +package com.fangjia.job.basicdata.config; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +import org.apache.commons.lang3.StringUtils; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.transport.client.PreBuiltTransportClient; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class SpringEs206Config { + + @Value("${es.206.url}") + private String url; + + @Value("${es.206.cluster.name}") + private String clusterName; + + @Bean + public String getUrl() { + return this.url; + } + + @Bean + public String getClusterName() { + return this.clusterName; + } + + @SuppressWarnings("resource") + @Bean + public TransportClient transportClient206() { + if(StringUtils.isEmpty(clusterName)) { + return null; + } + Settings settings = Settings.builder().put("cluster.name", clusterName) + .put("client.transport.ping_timeout", "60s").build(); + try { + return new PreBuiltTransportClient(settings) + .addTransportAddress(new TransportAddress(InetAddress.getByName(url), 9300)); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + return null; + } +} diff --git a/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/config/SpringMongoSeabirdConfig.java b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/config/SpringMongoSeabirdConfig.java new file mode 100644 index 0000000..8b92630 --- /dev/null +++ b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/config/SpringMongoSeabirdConfig.java @@ -0,0 +1,58 @@ +package com.fangjia.job.basicdata.config; + +import java.util.Arrays; +import java.util.List; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDbFactory; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoDbFactory; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; + +import com.mongodb.MongoClient; +import com.mongodb.MongoClientOptions; +import com.mongodb.ServerAddress; + +@Configuration +public class SpringMongoSeabirdConfig { + @Value("${mongo.fangjia_tmp.url}") + private String fangjiatmpUrl; + + @Value("${mongo.fangjia_tmp.port}") + private int fangjiatmpPort; + + @Value("${mongo.fangjia_tmp.name}") + private String fangjiatmpName; + + @Bean + MongoDbFactory fangjiatmpMongoDbFactory() throws Exception { + List seeds = Arrays.asList(new ServerAddress(fangjiatmpUrl, fangjiatmpPort)); + MongoClientOptions.Builder builder = new MongoClientOptions.Builder(); + int connPerHost = 10; + builder.connectionsPerHost(connPerHost); + int threadCount = 10; + builder.threadsAllowedToBlockForConnectionMultiplier(threadCount); + int maxWaitTime = 10000; + builder.maxWaitTime(maxWaitTime); + int timeOut = 10000; + builder.connectTimeout(timeOut); + MongoClientOptions options = builder.build(); + MongoClient mongoClient = new MongoClient(seeds, options); + return new SimpleMongoDbFactory(mongoClient, fangjiatmpName); + } + + @Bean + @SuppressWarnings("deprecation") + public MongoTemplate fangjiatempMongoTemplate() throws Exception { + //remove _class + MappingMongoConverter converter = new MappingMongoConverter(fangjiatmpMongoDbFactory(), new MongoMappingContext()); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + MongoTemplate mongoTemplate = new MongoTemplate(fangjiatmpMongoDbFactory(), converter); + return mongoTemplate; + } +} diff --git a/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/job/SeabirdDataSyncEsJob.java b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/job/SeabirdDataSyncEsJob.java new file mode 100644 index 0000000..b7a5eff --- /dev/null +++ b/fangjia-job/fangjia-job-basicdata/src/main/java/com/fangjia/job/basicdata/job/SeabirdDataSyncEsJob.java @@ -0,0 +1,837 @@ +package com.fangjia.job.basicdata.job; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Queue; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateFormatUtils; +import org.apache.commons.lang3.time.DateUtils; +import org.bson.types.ObjectId; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.transport.client.PreBuiltTransportClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Field; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.stereotype.Component; + +import com.dangdang.ddframe.job.api.ShardingContext; +import com.dangdang.ddframe.job.api.simple.SimpleJob; +import com.fangjia.job.basicdata.config.SpringEs206Config; +import com.fangjia.job.common.util.DataUtil; +import com.fangjia.job.common.util.DateUtilCustom; +import com.fangjia.job.common.util.JsonUtils; +import com.fangjia.job.common.util.SpringContextUtil; +import com.mongodb.BasicDBObject; +import com.mongodb.Bytes; +import com.mongodb.DBCollection; +import com.mongodb.DBCursor; +import com.mongodb.DBObject; +import com.mongodb.util.JSON; + +/** + * seabird数据导入ES任务 + * + * @author weilei + * + * @date 2018-08-29 + * + */ +@Component +public class SeabirdDataSyncEsJob implements SimpleJob { + + private Logger logger = LoggerFactory.getLogger(SeabirdDataSyncEsJob.class); + + @Autowired + @Qualifier("fangjiatempMongoTemplate") + private MongoTemplate fangjiatempMongoTemplate; + + @Autowired + @Qualifier("transportClient206") + private TransportClient transportClient_206; + +// @Autowired +// private SpringEs206Config springEs206Config; + +// Map transportClientMap = new HashMap<>(); + Map bulkRequestBuilderMap = new HashMap<>(); + Map addedEsCountMap = new HashMap<>(); + +// BulkRequestBuilder bulkRequest = null; + Date currentDate = null; + int BatchCount = 1000; + int WholeProcBatchTimes = 10 * BatchCount; + + Queue removeData = new LinkedBlockingQueue<>(100 * BatchCount); + boolean allThreadEnd = false; + + @Override + public void execute(ShardingContext shardingContext) { +// this.bulkRequest = this.defaultTransportClient.prepareBulk(); + currentDate = new Date(); + + DBCollection collection_seabird = fangjiatempMongoTemplate.getCollection("seabird"); + println("availableProcessors == " + Runtime.getRuntime().availableProcessors()); + ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2); + executorService.execute(new Runnable() { + @Override + public void run() { + while(true) { + if(removeData.size() == 0) { + try { + println("removeData.size() == 0, sleep 1 sed"); + Thread.sleep(1000); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + continue; + } + try { + if(removeData.size() > 0) { + if(removeData.size() % BatchCount == 0) { + println("removeData.size() == " + removeData.size()); + } + String _id = removeData.poll(); + if(StringUtils.isNotEmpty(_id)) { + DBObject q = new BasicDBObject("_id", new ObjectId(_id)); + collection_seabird.remove(q); + } + } + } catch (Exception e) { + + } + if(allThreadEnd) { + break; + } + } + } + }); + executorService.shutdown(); + + int count = 0; + for (String cat : CATEGORIES) { + try { + count += processFromSpidersByCat(collection_seabird, cat); + } catch (Exception e) { + e.printStackTrace(); + } + } + + int judgeCount = 0; + while(judgeCount <= 10) { + if(removeData.size() <= 0) { + judgeCount++; + }else { + judgeCount = 0; + try { + Thread.sleep(10 * 1000); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + allThreadEnd = true; + try { + Thread.sleep(10 * 1000); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + if(executorService.isTerminated()){ + System.out.println("所有的子线程都结束了"); + } + +// TransportClient transportClient = getCustomTransportClient(springEs206Config.getUrl(), ""); + + logger.info("导入结束,共导入:" + count); + } + +// public TransportClient getCustomTransportClient(String indexName) { +// if(StringUtils.isEmpty(indexName)) { +// return null; +// } +// if(StringUtils.isEmpty(springEs206Config.getClusterName())) { +// return null; +// } +// if(transportClientMap.containsKey(indexName)) { +// return transportClientMap.get(indexName); +// } +// Settings settings = Settings.builder().put("cluster.name", springEs206Config.getClusterName()) +// .put("client.transport.ping_timeout", "60s").build(); +// try { +// String url = springEs206Config.getUrl(); +// TransportClient result = new PreBuiltTransportClient(settings).addTransportAddress(new TransportAddress(InetAddress.getByName(url), 9300)); +// transportClientMap.put(indexName, result); +// return result; +// } catch (UnknownHostException e) { +// e.printStackTrace(); +// } +// return null; +// } + + public void setBulkRequestBuilder(String indexName, BulkRequestBuilder builder) { + if(StringUtils.isEmpty(indexName) || builder == null) { + return; + } + bulkRequestBuilderMap.put(indexName, builder); + } + public BulkRequestBuilder getBulkRequestBuilder(String indexName) { + if(StringUtils.isEmpty(indexName)) { + return null; + } + return bulkRequestBuilderMap.get(indexName); +// try { +// TransportClient transportClient = getCustomTransportClient(indexName); +// if(transportClient != null) { +// Settings settings = Settings.builder().put("cluster.name", springEs206Config.getClusterName()) +// .put("client.transport.ping_timeout", "60s").build(); +// bulkRequest = transportClient.prepareBulk(); +// bulkRequest.add(transportClient.prepareIndex(indexName,"doc")); +// bulkRequestBuilderMap.put(indexName, bulkRequest); +// return bulkRequest; +// } +// } catch (Exception e) { +// e.printStackTrace(); +// } +// return null; + } + +// void clearCustomBulkRequestBuilder(String indexName) { +// if(StringUtils.isEmpty(indexName)) { +// return; +// } +// TransportClient transportClient = getCustomTransportClient(indexName); +// if(transportClient == null) { +// return; +// } +// if(bulkRequestBuilderMap.containsKey(indexName)) { +// BulkRequestBuilder bulkRequest = bulkRequestBuilderMap.get(indexName); +// if(bulkRequest != null) { +// bulkRequest = transportClient.prepareBulk(); +// bulkRequest.add(transportClient.prepareIndex(indexName,"doc")); +// bulkRequestBuilderMap.put(indexName, bulkRequest); +// } +// } +// } + + void allBulkRequestSubmit() { + for(Entry entry : bulkRequestBuilderMap.entrySet()) { + try { + BulkRequestBuilder bulkRequest = entry.getValue(); + bulkRequest.execute().actionGet(); + }catch(Exception ex) { + ex.printStackTrace(); + } + } + } + + DBCursor getCursor(DBCollection collection_seabird, String cat) { + try { + Query query = Query.query(Criteria.where("cat").is(cat).and("c_date").lt(this.currentDate)); + return collection_seabird.find(query.getQueryObject()).batchSize(1000).addOption(Bytes.QUERYOPTION_NOTIMEOUT); + }catch(Exception ex) { + ex.printStackTrace(); + return null; + } + } + + int processFromSpidersByCat(DBCollection collection_seabird, String cat) { + if(collection_seabird == null || StringUtils.isEmpty(cat)) { + return 0; + } + + int count = 0; + int wholeDR = 0; + int fc = 0; +// List batchDelList = new ArrayList<>(); +// DBObject query_seabird = new BasicDBObject(); +// query_seabird.put("cat", cat); +// DBObject projection = new BasicDBObject(); +// projection.put("city", 1); + + DBCursor cursor_seabird = null; + while((cursor_seabird = getCursor(collection_seabird, cat)) == null) { + try { + Thread.sleep(60 * 1000); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + System.out.println(cursor_seabird.count()); + while (cursor_seabird.hasNext()) { + count++; + DBObject dbObj_seabird = cursor_seabird.next(); + if (dbObj_seabird == null || !dbObj_seabird.containsField(MongoObjectIDKey) + || dbObj_seabird.get(MongoObjectIDKey) == null + || StringUtils.isEmpty(dbObj_seabird.get(MongoObjectIDKey).toString())) { + continue; + } + String _id = dbObj_seabird.get(MongoObjectIDKey).toString(); + while(removeData.offer(_id) == false) { + try { + System.out.println("The queue is full, sleep.............60 seds"); + Thread.sleep(60 * 1000); + }catch(Exception ex) { + ex.printStackTrace(); + } + } + + if (isValidData(dbObj_seabird)) { + DBObject standardData = formatToStandardData(dbObj_seabird); + if (dataClean(standardData)) { + + try { + Integer pDate = parseInt(standardData.get("p_date")); + if(pDate <= 0) { + continue; + } + Date publishDate = parseDate(String.valueOf(pDate), "yyyyMMdd"); + if(publishDate == null) { + continue; + } + String indexName = "seawater-" + DateFormatUtils.format(publishDate, "yyyy"); + + int dr = 0; + if(addedEsCountMap.containsKey(indexName)) { + dr = addedEsCountMap.get(indexName); + } + dr++; + wholeDR++; + addedEsCountMap.put(indexName, dr); +// println("indexName - " + indexName + ", addedEsCount == " + dr); + + + BulkRequestBuilder bulkRequest = getBulkRequestBuilder(indexName); + //transportClient_206.prepareBulk(); + if(bulkRequest == null) { + bulkRequest = transportClient_206.prepareBulk(); + setBulkRequestBuilder(indexName, bulkRequest); + } + + String id = standardData.get("_id").toString(); + standardData.removeField("_id"); + bulkRequest.add(transportClient_206.prepareIndex(indexName,"_doc") + .setSource(JsonUtils.toJson(standardData), XContentType.JSON) + .setId(id)); + + if(dr >= BatchCount) { + bulkRequest.execute().actionGet(); + bulkRequest = transportClient_206.prepareBulk(); + setBulkRequestBuilder(indexName, bulkRequest); + addedEsCountMap.put(indexName, 0); + } + }catch(Exception ex) { + ex.printStackTrace(); + } + +// String indexName = this.importData2Elastic(standardData.toMap()); +// if(StringUtils.isNoneEmpty(indexName)) { +// dr++; +// int addedCount = 0; +// if(addedEsCountMap.containsKey(indexName)) { +// addedCount = addedEsCountMap.get(indexName); +// println("indexName - " + indexName + ", addedEsCount == " + addedCount); +// if(addedCount >= BatchCount) { +// BulkRequestBuilder bulkRequest = getCustomBulkRequestBuilder(indexName); +// if(bulkRequest != null) { +// try { +// bulkRequest.execute().actionGet(); +// WholeProcBatchTimes--; +// addedEsCountMap.put(indexName, 0); +// logger.info("已导入:" + dr); +// // 清空数据 +// clearCustomBulkRequestBuilder(indexName); +// }catch(Exception ex) { +// ex.printStackTrace(); +// } +// +// } +// } +// } +// } + } + } else { + println("放弃:" + dbObj_seabird.toString()); + fc++; + } + + if (count % BatchCount == 0) { + for(String key : addedEsCountMap.keySet()) { + println("index name == " + key); + } + println("count == " + String.valueOf(count) + ", wholeDR == " + wholeDR + ", fc == " + fc); + } + } + + if(wholeDR > 0) { +// this.bulkRequest.execute().actionGet(); + allBulkRequestSubmit(); + } + + println(count + "/add:${ac},upd:${uc},igr:${ic},err:${fc}\t" + DateUtilCustom.formatToTightYYYYMMDDhhmmss(new Date())); + + return count; + } + + void removeSeabird(DBCollection collection_seabird, List batchDelList) { + if (collection_seabird != null && batchDelList!= null && batchDelList.size() >= BatchCount) { + while(batchDelList.size() > 0) { + try { +// String _id = batchDelList.get(0); + DBObject q = new BasicDBObject("_id", new ObjectId(batchDelList.get(0))); +// DBObject old = collection_seabird.findOne(q); + collection_seabird.remove(q); + }catch(Exception ex){ + ex.printStackTrace(); + } + batchDelList.remove(0); + } + } + } + + private boolean isValidData(DBObject dbo) { + String error_isValidData = "isValidData error -> "; + + String url = dbo.get("url") != null ? dbo.get("url").toString() : null; +// StringUtils.startsWithIgnoreCase(url, "http://") + if (StringUtils.isEmpty(url)) { + println(error_isValidData + "url is null"); + return false; + } + if (!(StringUtils.startsWithIgnoreCase(url, "http://") || StringUtils.startsWithIgnoreCase(url, "https://")) + && StringUtils.contains(url, "fangjia.com") == false) { + println(error_isValidData + "url:" + url); + return false; + } + + String city = dbo.get("city") != null ? dbo.get("city").toString() : null; + if (StringUtils.isEmpty(city) || city.length() > 10) { + println(error_isValidData + "city:" + city); + return false; + } + + String region = dbo.get("region") != null ? dbo.get("region").toString() : null; + if (StringUtils.isEmpty(region) || region.length() > 10) { + println(error_isValidData + "region:" + region); + return false; + } + + String district_name = dbo.get("district_name") != null ? dbo.get("district_name").toString() : null; + if (StringUtils.isEmpty(district_name) || district_name.length() > 30) { + println(error_isValidData + "district_name:" + district_name); + return false; + } + + if (dbo.get("area") == null || isDouble(dbo.get("area")) == false) { + println(error_isValidData + "area:" + dbo.get("area")); + return false; + } + double d_area = parseDouble(dbo.get("area")); + + if (checkDate(dbo.get("publish_date")) == false) { + println(error_isValidData + "publish_date:" + dbo.get("publish_date")); + return false; + } + + if (before2007Date == null) { + before2007Date = parseFromTightYYYYMMDD("2007-01-01"); + } + if (dbo.get("publish_date") != null) { + Date publish_date = parseFromTightYYYYMMDD(dbo.get("publish_date").toString()); + if (publish_date.before(before2007Date)) { + println(error_isValidData + "publish_date:" + dbo.get("publish_date").toString()); + return false; + } + } + + String cat = dbo.get("cat") != null ? dbo.get("cat").toString() : null; + int total_price = parseInt(dbo.get("total_price")); + if (cat == "sell") { + if (d_area < 10 || d_area > 2000) { + println(error_isValidData + "cat:" + cat + ",area:" + d_area); + return false; + } + if (total_price < 100000 || total_price > 900000000) { + println(error_isValidData + "cat:" + cat + ",total_price:" + total_price); + return false; + } + + double avgPrice = total_price / d_area; + if (avgPrice > 300000 || avgPrice < 1000) { + println(error_isValidData + "cat:" + cat + ",avgPrice:" + avgPrice); + return false; + } + + } else if (cat == "rent") { + if (d_area < 5 || d_area > 2000) { + println(error_isValidData + "cat:" + cat + ", area:" + d_area); + return false; + } + + if (total_price < 10 || total_price > 10000000) { + println(error_isValidData + "cat:" + cat + ",total_price:" + total_price); + return false; + } + } + return true; + } + + private DBObject formatToStandardData(DBObject dbo) { + if (StringUtils.isNotBlank((String) dbo.get("name")) && ((String) dbo.get("name")).length() > 50) { + dbo.put("name", dbo.get("name").toString().substring(0, 50)); + } + + if (StringUtils.isNotBlank((String) dbo.get("address")) && ((String) dbo.get("address")).length() > 50) { + dbo.removeField("address"); + } + + if (StringUtils.isNotBlank((String) dbo.get("mobile")) && ((String) dbo.get("mobile")).length() > 30) { + dbo.removeField("mobile"); + } + + if (StringUtils.isNotBlank((String) dbo.get("estate_charge")) + && ((String) dbo.get("estate_charge")).length() > 40) { + dbo.removeField("estate_charge"); + } + + String url = processUrl(dbo.get("url") != null ? dbo.get("url").toString() : ""); + + String id = dbo.get(MongoObjectIDKey).toString(); +// if (url.indexOf("fangjia.com") > -1) { +// id = dbo.get(MongoObjectIDKey).toString(); +// }else { +// id = DigestUtils.md5Hex(url + "#" + url.hashCode()); +// } + + DBObject standardObj = new BasicDBObject(); + standardObj.put(MongoObjectIDKey, id); + standardObj.put("url", url); + standardObj.put("cat", dbo.get("cat")); + standardObj.put("city", dbo.get("city").toString().trim()); + standardObj.put("region", dbo.get("region").toString().trim()); + standardObj.put("district_name", DataUtil.toDBCcase(dbo.get("district_name").toString()).trim()); + standardObj.put("p_date", DateUtilCustom.formatToTightYYYYMMDD(dbo.get("publish_date"))); + standardObj.put("c_date", DateUtilCustom.formatToTightYYYYMMDD(dbo.get("c_date"))); + + // 数字类型的字段 + for (String key : NUMBER_FIELDS) { + Object val = dbo.get(key); + if (val != null && val instanceof Number) { + standardObj.put(key, val); + } + } + + // 文本类型的字段 + for (String key : TEXT_FIELDS) { + Object val = dbo.get(key); + // 判断文本字段如果太长,就不保存该字段 + if (val != null && val instanceof String && val.toString().length() <= 1024) { + standardObj.put(key, val.toString()); + } + } + + // 其他字段 + for (String key : OTHER_FIELDS) { + Object val = dbo.get(key); + if (val != null) { + standardObj.put(key, val); + } + } +// standardObj.put("md5_hash", getMd5Hash(standardObj)) + return standardObj; + } + + String processUrl(String url) { + String root = DataUtil.urlToSiteName(url); + if ("58.com".equals(root)) { + // http://huizhou.58.com/ershoufang/22416405318175x.shtml?psid=196886603191045603944627704&entinfo=22416405318175_0 + if (url.indexOf("?") > -1) { + url = StringUtils.substringBefore(url, "?"); + } + } else if ("anjuke.com".equals(root)) { + // http://wuhan.anjuke.com/prop/view/A417931448?from=comm_one-saleMetro&spread=commsearch_p + if (url.indexOf("?") > -1) { + url = StringUtils.substringBefore(url, "?"); + } + } else if ("leju.com".equals(root)) { + // http://bj.esf.leju.com/detail/217448358#zn=pc-house-36 + if (url.indexOf("#") > -1) { + url = StringUtils.substringBefore(url, "#"); + } + } + return url; + } + + static String err_dataClean = "dataClean() error -> "; + + boolean dataClean(DBObject dbo) { + if (dbo == null) { + println(err_dataClean + " -> dbo == null"); + return false; + } + + if (dbo.get("_id") == null) { + println(err_dataClean + " -> _id == null"); + return false; + } + + if (isInteger(dbo.get("p_date")) == false) { + println(err_dataClean + " -> p_date"); + return false; + } + + List keys = new ArrayList(dbo.keySet()); + for (String key : keys) { + if (!fields.contains(key)) { + dbo.removeField(key); + } + } + + Integer pDate = parseInt(dbo.get("p_date")); + Date publishDate = parseDate(String.valueOf(pDate), "yyyyMMdd"); + if (publishDate == null) { + println(err_dataClean + " -> publishDate == null"); + return false; + } + dbo.put("publish_date", publishDate); + + if (dbo.get("m_date") == null) { + dbo.put("m_date", dbo.get("c_date")); + } + if (dbo.get("m_date") == null) { + dbo.put("m_date", publishDate); + } + + if(dbo.get("url") != null) { + String url = dbo.get("url").toString(); + String root = DataUtil.urlToSiteName(url); + if (StringUtils.isNotBlank(root)) { + dbo.put("root", root); + } + } + + return true; + } + +// String importData2Elastic(Map map){ +// if(map == null || map.size() == 0 || map.get("_id") == null || map.get("p_date") == null) { +// return null; +// } +// +// try{ +// Integer pDate = parseInt(map.get("p_date")); +// if(pDate <= 0) { +// return null; +// } +// Date publishDate = parseDate(String.valueOf(pDate), "yyyyMMdd"); +// if(publishDate == null) { +// return null; +// } +// String indexName = "seawater-" + DateFormatUtils.format(publishDate, "yyyy"); +// +// String id = map.get("_id").toString(); +// +// map.remove("_id"); +// map.remove("c_date"); +// map.remove("md5_hash"); +// +// TransportClient transportClient = getCustomTransportClient(indexName); +// BulkRequestBuilder bulkRequest = getCustomBulkRequestBuilder(indexName); +// if(transportClient != null && bulkRequest != null) { +//// this.bulkRequest.add(transportClient.prepareIndex(indexName,"doc") +// bulkRequest.add(transportClient.prepareIndex(indexName,"doc") +// .setSource(JsonUtils.toJson(map), XContentType.JSON) +// .setId(id)); +// +// int addedCount = 0; +// if(addedEsCountMap.containsKey(indexName)) { +// addedCount = addedEsCountMap.get(indexName); +// } +// addedCount++; +// addedEsCountMap.put(indexName, addedCount); +// } +// +// return indexName; +// }catch(Exception ex){ +// println("importData2Elastic error -> map:" + com.alibaba.fastjson.JSON.toJSONString(map) + ", exception:" + ex.getMessage()); +// ex.printStackTrace(); +// } +// return null; +// } + + static void println(String msg) { + System.out.println(msg); + } + + static Date getCSTDate(Object obj) { + if (obj == null) { + return null; + } + try { + SimpleDateFormat sdf = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy", Locale.US); + return sdf.parse(obj.toString()); + } catch (Exception e) { + } + return null; + } + + static boolean checkDate(Object obj) { + if (obj == null) { + return false; + } + SimpleDateFormat sd = new SimpleDateFormat(DATE_PATTERN); + try { + sd.setLenient(false); + sd.parse(obj.toString()); + } catch (Exception e) { + if(getCSTDate(obj.toString()) != null) { + return true; + } + + return false; + } + return true; + } + + static boolean checkDateTime(String str) { + SimpleDateFormat sd = new SimpleDateFormat(DATETIME_PATTERN); + try { + sd.setLenient(false); + sd.parse(str); + } catch (Exception e) { + if(getCSTDate(str) != null) { + return true; + } + } + return true; + } + + public static Date parseFromTightYYYYMMDD(String s) { + if (s == null || s.length() == 0) { + return null; + } + SimpleDateFormat format = new SimpleDateFormat(DATE_PATTERN); + try { + return format.parse(s); + } catch (ParseException e) { + return getCSTDate(s); + } + } + + public static boolean isDouble(Object input) { + if(input == null) { + return false; + } + try { + Double.parseDouble(input.toString()); + return true; + }catch(Exception ex) { + return false; + } + } + + public static double parseDouble(Object obj) { + if (obj == null) { + return 0; + } + try { + return Double.parseDouble(obj.toString()); + } catch (Exception e) { + e.printStackTrace(); + } + return 0; + } + + public static boolean isInteger(Object input) { + if(input == null) { + return false; + } + try { + Integer.parseInt(input.toString()); + return true; + }catch(Exception ex) { + return false; + } + } + + public static int parseInt(Object obj) { + if (obj == null) { + return 0; + } + try { + return Integer.parseInt(obj.toString()); + } catch (Exception e) { + e.printStackTrace(); + } + return 0; + } + + public static Date parseDate(String dateStr, String pattern) { + Date date = null; + if (StringUtils.isNotBlank(dateStr)) { + SimpleDateFormat sdf = new SimpleDateFormat(pattern); + try { + date = sdf.parse(dateStr); + } catch (ParseException e) { + } + } + return date; + } + + static Date before2007Date = null; + + public static final String DATE_PATTERN = "yyyy-MM-dd"; + public static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss"; + + static String MongoObjectIDKey = "_id"; + + static List fields = Arrays.asList("_id", "cat", "city", "region", "district_name", "total_price", "area", + "p_date", "url", "estate_type", "height", "floor", "room", "hall", "toilet", "fitment", "direction", + "rent_type", "user_id", "build_type", "build_date", "estate_charge", "status", "address", "block", "name", + "contact_people", "mobile", "company", "m_date", "floor_info", "community_support", "house_number", + "down_payment", "only_one", "buy_year", "unit_number", "unit_type", "owner_attitude", "service_desc", + "check_in_time", "rent_type", "pay_type", "house_allocation", "house_require", "house_advantage", "metro", + "line", "subject"); + + static String[] CATEGORIES = new String[] { "office_rent", "sell", "rent", "office_rent", "office_sell", + "shop_rent", "shop_sell" }; + static String[] NUMBER_FIELDS = new String[] { "total_price", "area", "height", "floor", "room", "hall", "toilet", + "age" }; + static String[] TEXT_FIELDS = new String[] { "name", "address", "block", "fitment", "estate_type", "mobile", + "build_type", "build_date", "company", "direction", "contact_people", "rent_type", "floor_info", + "house_number", "down_payment", "only_one", "buy_year", "unit_number", "unit_type", "owner_attitude", + "service_desc", "community_support", "description", "check_in_time", "house_allocation", "house_require", + "house_advantage", "pay_type", "metro", "line", "subject", "agency" }; + static String[] OTHER_FIELDS = new String[] { "user_id", "publish_date" }; + +} diff --git a/fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties index dc3c82d..65913f1 100644 --- a/fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties +++ b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties @@ -1,14 +1,21 @@ -mongo.business.url=114.80.150.198 -mongo.business.name=business -mongo.business.port=27017 +#mongo.business.url=114.80.150.198 +#mongo.business.name=business +#mongo.business.port=27017 + +mongo.fangjia_tmp.url=192.168.0.209 +mongo.fangjia_tmp.name=fangjia_tmp +mongo.fangjia_tmp.port=55007 spring.datasource.log.url=jdbc:mysql://localhost:3306/batch_log spring.datasource.log.username=root spring.datasource.log.password=123456 spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver -zk.server-lists=192.168.10.124:2181 -zk.namespace=fangjia-job-basicdata +#zk.server-lists=192.168.10.124:2181 +#zk.namespace=fangjia-job-basicdata es.url=192.168.0.45 -es.cluster.name=fangjia \ No newline at end of file +es.cluster.name=fangjia + +es.206.url=192.168.0.206 +es.206.cluster.name=fangjia \ No newline at end of file From b0d34e9b473f6310bfb95104731a3c5dc64e8d58 Mon Sep 17 00:00:00 2001 From: weilei Date: Thu, 18 Oct 2018 15:08:46 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E7=88=AC=E8=99=AB=E6=95=B0=E6=8D=AE(mongo-?= =?UTF-8?q?209:55007)=E5=AF=BC=E5=85=A5=E5=88=B0Es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/resources/application-dev.properties | 10 +++++----- .../src/main/resources/application-prod.properties | 9 ++++++++- .../src/main/resources/applicationContext.xml | 9 ++++++++- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties index 65913f1..0abfbe4 100644 --- a/fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties +++ b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-dev.properties @@ -1,6 +1,6 @@ -#mongo.business.url=114.80.150.198 -#mongo.business.name=business -#mongo.business.port=27017 +mongo.business.url=114.80.150.198 +mongo.business.name=business +mongo.business.port=27017 mongo.fangjia_tmp.url=192.168.0.209 mongo.fangjia_tmp.name=fangjia_tmp @@ -11,8 +11,8 @@ spring.datasource.log.username=root spring.datasource.log.password=123456 spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver -#zk.server-lists=192.168.10.124:2181 -#zk.namespace=fangjia-job-basicdata +zk.server-lists=192.168.10.124:2181 +zk.namespace=fangjia-job-basicdata es.url=192.168.0.45 es.cluster.name=fangjia diff --git a/fangjia-job/fangjia-job-basicdata/src/main/resources/application-prod.properties b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-prod.properties index 37cf237..9a70863 100644 --- a/fangjia-job/fangjia-job-basicdata/src/main/resources/application-prod.properties +++ b/fangjia-job/fangjia-job-basicdata/src/main/resources/application-prod.properties @@ -2,6 +2,10 @@ mongo.business.url=mongo.db.fangjia.com mongo.business.name=business mongo.business.port=27017 +mongo.fangjia_tmp.url=192.168.0.209 +mongo.fangjia_tmp.name=fangjia_tmp +mongo.fangjia_tmp.port=55007 + spring.datasource.log.url=jdbc:mysql://mysql.db.fangjia.com:4306/batch_log spring.datasource.log.username=sa spring.datasource.log.password=G00jia @@ -11,4 +15,7 @@ zk.server-lists=192.168.0.207:2181,192.168.0.24:2181,192.168.0.25:2181 zk.namespace=fangjia-job-basicdata es.url=192.168.0.45 -es.cluster.name=fangjia \ No newline at end of file +es.cluster.name=fangjia + +es.206.url=192.168.0.206 +es.206.cluster.name=fangjia \ No newline at end of file diff --git a/fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext.xml b/fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext.xml index c4450b0..02060fe 100644 --- a/fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext.xml +++ b/fangjia-job/fangjia-job-basicdata/src/main/resources/applicationContext.xml @@ -43,8 +43,15 @@ + overwrite="true" event-trace-rdb-data-source="elasticJobLog"> + + --> + + \ No newline at end of file