爬虫数据(mongo-209:55007)导入到Es

This commit is contained in:
weilei
2018-10-18 15:00:30 +08:00
parent ff1787dbb4
commit 1814fb5d7e
3 changed files with 91 additions and 0 deletions
+7
View File
@@ -33,5 +33,12 @@
<groupId>io.github.openfeign</groupId>
<artifactId>feign-core</artifactId>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.8.1</version>
</dependency>
</dependencies>
</project>
@@ -2,6 +2,8 @@ package com.fangjia.job.common.util;
import java.math.BigDecimal;
import org.apache.commons.lang3.StringUtils;
public class DataUtil {
public static double getDoubleRound(int scale, double dSource) {
@@ -23,4 +25,54 @@ public class DataUtil {
}
}
/**
* 全角转半角
*
* @param input
* @return
*/
public static String toDBCcase(String input) {
if (StringUtils.isBlank(input))
return "";
char c[] = input.toCharArray();
for (int i = 0; i < c.length; i++) {
if (c[i] == '\u3000') {
c[i] = ' ';
} else if (c[i] > '\uFF00' && c[i] < '\uFF5F') {
c[i] = (char) (c[i] - 65248);
}
}
return new String(c);
}
public static String[] SITE_SUFFIXS = { ".com.cn", ".com", ".cn", ".net", ".org", ".cc", ".tm" };
public static String urlToSiteName(String url) {
if (StringUtils.isBlank(url)) {
return null;
}
for (String suffix : SITE_SUFFIXS) {
if (url.indexOf(suffix) > -1) {
String s = StringUtils.substringBefore(url, suffix);
if (s.indexOf(".") > -1) {
s = StringUtils.substringAfterLast(s, ".");
if (StringUtils.isNotBlank(s)) {
return s + suffix;
}
} else if (s.indexOf("://") > -1) {
s = StringUtils.substringAfterLast(s, "://");
if (StringUtils.isNotBlank(s)) {
return s + suffix;
}
} else if (s.indexOf("://") == -1 && s.indexOf(".") == -1) {
return s + suffix;
}
return null;
}
}
return null;
}
}
@@ -179,6 +179,38 @@ public class DateUtilCustom {
}
}
/**
* 转化日期成yyyymmdd的String格式
*
* @param date
* @return
*/
public static String formatToTightYYYYMMDD(Object dateObj) {
if (dateObj == null)
return "";
String result = "";
try {
SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
result = format.format((Date)dateObj);
} catch (Exception e) {
return "";
}
return result;
}
public static String formatToTightYYYYMMDDhhmmss(Date date) {
if (date == null)
return "";
String result = "";
try {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
result = format.format(date);
} catch (Exception e) {
return "";
}
return result;
}
public static void main(String[] args) {
System.err.println(JsonUtils.toJson(getLatelyDate(3)));