爬虫数据(mongo-209:55007)导入到Es

This commit is contained in:
weilei
2018-10-18 15:05:07 +08:00
parent d1c1180732
commit 66677b9c1f
5 changed files with 972 additions and 7 deletions
+13
View File
@@ -70,6 +70,19 @@
<artifactId>elasticsearch</artifactId>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.8.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.alibaba/fastjson -->
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.51</version>
</dependency>
</dependencies>
<build>
@@ -0,0 +1,50 @@
package com.fangjia.job.basicdata.config;
import java.net.InetAddress;
import java.net.UnknownHostException;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class SpringEs206Config {
@Value("${es.206.url}")
private String url;
@Value("${es.206.cluster.name}")
private String clusterName;
@Bean
public String getUrl() {
return this.url;
}
@Bean
public String getClusterName() {
return this.clusterName;
}
@SuppressWarnings("resource")
@Bean
public TransportClient transportClient206() {
if(StringUtils.isEmpty(clusterName)) {
return null;
}
Settings settings = Settings.builder().put("cluster.name", clusterName)
.put("client.transport.ping_timeout", "60s").build();
try {
return new PreBuiltTransportClient(settings)
.addTransportAddress(new TransportAddress(InetAddress.getByName(url), 9300));
} catch (UnknownHostException e) {
e.printStackTrace();
}
return null;
}
}
@@ -0,0 +1,58 @@
package com.fangjia.job.basicdata.config;
import java.util.Arrays;
import java.util.List;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.data.mongodb.MongoDbFactory;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.SimpleMongoDbFactory;
import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper;
import org.springframework.data.mongodb.core.convert.MappingMongoConverter;
import org.springframework.data.mongodb.core.mapping.MongoMappingContext;
import com.mongodb.MongoClient;
import com.mongodb.MongoClientOptions;
import com.mongodb.ServerAddress;
@Configuration
public class SpringMongoSeabirdConfig {
@Value("${mongo.fangjia_tmp.url}")
private String fangjiatmpUrl;
@Value("${mongo.fangjia_tmp.port}")
private int fangjiatmpPort;
@Value("${mongo.fangjia_tmp.name}")
private String fangjiatmpName;
@Bean
MongoDbFactory fangjiatmpMongoDbFactory() throws Exception {
List<ServerAddress> seeds = Arrays.asList(new ServerAddress(fangjiatmpUrl, fangjiatmpPort));
MongoClientOptions.Builder builder = new MongoClientOptions.Builder();
int connPerHost = 10;
builder.connectionsPerHost(connPerHost);
int threadCount = 10;
builder.threadsAllowedToBlockForConnectionMultiplier(threadCount);
int maxWaitTime = 10000;
builder.maxWaitTime(maxWaitTime);
int timeOut = 10000;
builder.connectTimeout(timeOut);
MongoClientOptions options = builder.build();
MongoClient mongoClient = new MongoClient(seeds, options);
return new SimpleMongoDbFactory(mongoClient, fangjiatmpName);
}
@Bean
@SuppressWarnings("deprecation")
public MongoTemplate fangjiatempMongoTemplate() throws Exception {
//remove _class
MappingMongoConverter converter = new MappingMongoConverter(fangjiatmpMongoDbFactory(), new MongoMappingContext());
converter.setTypeMapper(new DefaultMongoTypeMapper(null));
MongoTemplate mongoTemplate = new MongoTemplate(fangjiatmpMongoDbFactory(), converter);
return mongoTemplate;
}
}
@@ -0,0 +1,837 @@
package com.fangjia.job.basicdata.job;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Queue;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.bson.types.ObjectId;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Field;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Component;
import com.dangdang.ddframe.job.api.ShardingContext;
import com.dangdang.ddframe.job.api.simple.SimpleJob;
import com.fangjia.job.basicdata.config.SpringEs206Config;
import com.fangjia.job.common.util.DataUtil;
import com.fangjia.job.common.util.DateUtilCustom;
import com.fangjia.job.common.util.JsonUtils;
import com.fangjia.job.common.util.SpringContextUtil;
import com.mongodb.BasicDBObject;
import com.mongodb.Bytes;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.util.JSON;
/**
* seabird数据导入ES任务
*
* @author weilei
*
* @date 2018-08-29
*
*/
@Component
public class SeabirdDataSyncEsJob implements SimpleJob {
private Logger logger = LoggerFactory.getLogger(SeabirdDataSyncEsJob.class);
@Autowired
@Qualifier("fangjiatempMongoTemplate")
private MongoTemplate fangjiatempMongoTemplate;
@Autowired
@Qualifier("transportClient206")
private TransportClient transportClient_206;
// @Autowired
// private SpringEs206Config springEs206Config;
// Map<String, TransportClient> transportClientMap = new HashMap<>();
Map<String, BulkRequestBuilder> bulkRequestBuilderMap = new HashMap<>();
Map<String, Integer> addedEsCountMap = new HashMap<>();
// BulkRequestBuilder bulkRequest = null;
Date currentDate = null;
int BatchCount = 1000;
int WholeProcBatchTimes = 10 * BatchCount;
Queue<String> removeData = new LinkedBlockingQueue<>(100 * BatchCount);
boolean allThreadEnd = false;
@Override
public void execute(ShardingContext shardingContext) {
// this.bulkRequest = this.defaultTransportClient.prepareBulk();
currentDate = new Date();
DBCollection collection_seabird = fangjiatempMongoTemplate.getCollection("seabird");
println("availableProcessors == " + Runtime.getRuntime().availableProcessors());
ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
executorService.execute(new Runnable() {
@Override
public void run() {
while(true) {
if(removeData.size() == 0) {
try {
println("removeData.size() == 0, sleep 1 sed");
Thread.sleep(1000);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
continue;
}
try {
if(removeData.size() > 0) {
if(removeData.size() % BatchCount == 0) {
println("removeData.size() == " + removeData.size());
}
String _id = removeData.poll();
if(StringUtils.isNotEmpty(_id)) {
DBObject q = new BasicDBObject("_id", new ObjectId(_id));
collection_seabird.remove(q);
}
}
} catch (Exception e) {
}
if(allThreadEnd) {
break;
}
}
}
});
executorService.shutdown();
int count = 0;
for (String cat : CATEGORIES) {
try {
count += processFromSpidersByCat(collection_seabird, cat);
} catch (Exception e) {
e.printStackTrace();
}
}
int judgeCount = 0;
while(judgeCount <= 10) {
if(removeData.size() <= 0) {
judgeCount++;
}else {
judgeCount = 0;
try {
Thread.sleep(10 * 1000);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
allThreadEnd = true;
try {
Thread.sleep(10 * 1000);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
if(executorService.isTerminated()){
System.out.println("所有的子线程都结束了");
}
// TransportClient transportClient = getCustomTransportClient(springEs206Config.getUrl(), "");
logger.info("导入结束,共导入:" + count);
}
// public TransportClient getCustomTransportClient(String indexName) {
// if(StringUtils.isEmpty(indexName)) {
// return null;
// }
// if(StringUtils.isEmpty(springEs206Config.getClusterName())) {
// return null;
// }
// if(transportClientMap.containsKey(indexName)) {
// return transportClientMap.get(indexName);
// }
// Settings settings = Settings.builder().put("cluster.name", springEs206Config.getClusterName())
// .put("client.transport.ping_timeout", "60s").build();
// try {
// String url = springEs206Config.getUrl();
// TransportClient result = new PreBuiltTransportClient(settings).addTransportAddress(new TransportAddress(InetAddress.getByName(url), 9300));
// transportClientMap.put(indexName, result);
// return result;
// } catch (UnknownHostException e) {
// e.printStackTrace();
// }
// return null;
// }
public void setBulkRequestBuilder(String indexName, BulkRequestBuilder builder) {
if(StringUtils.isEmpty(indexName) || builder == null) {
return;
}
bulkRequestBuilderMap.put(indexName, builder);
}
public BulkRequestBuilder getBulkRequestBuilder(String indexName) {
if(StringUtils.isEmpty(indexName)) {
return null;
}
return bulkRequestBuilderMap.get(indexName);
// try {
// TransportClient transportClient = getCustomTransportClient(indexName);
// if(transportClient != null) {
// Settings settings = Settings.builder().put("cluster.name", springEs206Config.getClusterName())
// .put("client.transport.ping_timeout", "60s").build();
// bulkRequest = transportClient.prepareBulk();
// bulkRequest.add(transportClient.prepareIndex(indexName,"doc"));
// bulkRequestBuilderMap.put(indexName, bulkRequest);
// return bulkRequest;
// }
// } catch (Exception e) {
// e.printStackTrace();
// }
// return null;
}
// void clearCustomBulkRequestBuilder(String indexName) {
// if(StringUtils.isEmpty(indexName)) {
// return;
// }
// TransportClient transportClient = getCustomTransportClient(indexName);
// if(transportClient == null) {
// return;
// }
// if(bulkRequestBuilderMap.containsKey(indexName)) {
// BulkRequestBuilder bulkRequest = bulkRequestBuilderMap.get(indexName);
// if(bulkRequest != null) {
// bulkRequest = transportClient.prepareBulk();
// bulkRequest.add(transportClient.prepareIndex(indexName,"doc"));
// bulkRequestBuilderMap.put(indexName, bulkRequest);
// }
// }
// }
void allBulkRequestSubmit() {
for(Entry<String, BulkRequestBuilder> entry : bulkRequestBuilderMap.entrySet()) {
try {
BulkRequestBuilder bulkRequest = entry.getValue();
bulkRequest.execute().actionGet();
}catch(Exception ex) {
ex.printStackTrace();
}
}
}
DBCursor getCursor(DBCollection collection_seabird, String cat) {
try {
Query query = Query.query(Criteria.where("cat").is(cat).and("c_date").lt(this.currentDate));
return collection_seabird.find(query.getQueryObject()).batchSize(1000).addOption(Bytes.QUERYOPTION_NOTIMEOUT);
}catch(Exception ex) {
ex.printStackTrace();
return null;
}
}
int processFromSpidersByCat(DBCollection collection_seabird, String cat) {
if(collection_seabird == null || StringUtils.isEmpty(cat)) {
return 0;
}
int count = 0;
int wholeDR = 0;
int fc = 0;
// List<String> batchDelList = new ArrayList<>();
// DBObject query_seabird = new BasicDBObject();
// query_seabird.put("cat", cat);
// DBObject projection = new BasicDBObject();
// projection.put("city", 1);
DBCursor cursor_seabird = null;
while((cursor_seabird = getCursor(collection_seabird, cat)) == null) {
try {
Thread.sleep(60 * 1000);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
System.out.println(cursor_seabird.count());
while (cursor_seabird.hasNext()) {
count++;
DBObject dbObj_seabird = cursor_seabird.next();
if (dbObj_seabird == null || !dbObj_seabird.containsField(MongoObjectIDKey)
|| dbObj_seabird.get(MongoObjectIDKey) == null
|| StringUtils.isEmpty(dbObj_seabird.get(MongoObjectIDKey).toString())) {
continue;
}
String _id = dbObj_seabird.get(MongoObjectIDKey).toString();
while(removeData.offer(_id) == false) {
try {
System.out.println("The queue is full, sleep.............60 seds");
Thread.sleep(60 * 1000);
}catch(Exception ex) {
ex.printStackTrace();
}
}
if (isValidData(dbObj_seabird)) {
DBObject standardData = formatToStandardData(dbObj_seabird);
if (dataClean(standardData)) {
try {
Integer pDate = parseInt(standardData.get("p_date"));
if(pDate <= 0) {
continue;
}
Date publishDate = parseDate(String.valueOf(pDate), "yyyyMMdd");
if(publishDate == null) {
continue;
}
String indexName = "seawater-" + DateFormatUtils.format(publishDate, "yyyy");
int dr = 0;
if(addedEsCountMap.containsKey(indexName)) {
dr = addedEsCountMap.get(indexName);
}
dr++;
wholeDR++;
addedEsCountMap.put(indexName, dr);
// println("indexName - " + indexName + ", addedEsCount == " + dr);
BulkRequestBuilder bulkRequest = getBulkRequestBuilder(indexName);
//transportClient_206.prepareBulk();
if(bulkRequest == null) {
bulkRequest = transportClient_206.prepareBulk();
setBulkRequestBuilder(indexName, bulkRequest);
}
String id = standardData.get("_id").toString();
standardData.removeField("_id");
bulkRequest.add(transportClient_206.prepareIndex(indexName,"_doc")
.setSource(JsonUtils.toJson(standardData), XContentType.JSON)
.setId(id));
if(dr >= BatchCount) {
bulkRequest.execute().actionGet();
bulkRequest = transportClient_206.prepareBulk();
setBulkRequestBuilder(indexName, bulkRequest);
addedEsCountMap.put(indexName, 0);
}
}catch(Exception ex) {
ex.printStackTrace();
}
// String indexName = this.importData2Elastic(standardData.toMap());
// if(StringUtils.isNoneEmpty(indexName)) {
// dr++;
// int addedCount = 0;
// if(addedEsCountMap.containsKey(indexName)) {
// addedCount = addedEsCountMap.get(indexName);
// println("indexName - " + indexName + ", addedEsCount == " + addedCount);
// if(addedCount >= BatchCount) {
// BulkRequestBuilder bulkRequest = getCustomBulkRequestBuilder(indexName);
// if(bulkRequest != null) {
// try {
// bulkRequest.execute().actionGet();
// WholeProcBatchTimes--;
// addedEsCountMap.put(indexName, 0);
// logger.info("已导入:" + dr);
// // 清空数据
// clearCustomBulkRequestBuilder(indexName);
// }catch(Exception ex) {
// ex.printStackTrace();
// }
//
// }
// }
// }
// }
}
} else {
println("放弃:" + dbObj_seabird.toString());
fc++;
}
if (count % BatchCount == 0) {
for(String key : addedEsCountMap.keySet()) {
println("index name == " + key);
}
println("count == " + String.valueOf(count) + ", wholeDR == " + wholeDR + ", fc == " + fc);
}
}
if(wholeDR > 0) {
// this.bulkRequest.execute().actionGet();
allBulkRequestSubmit();
}
println(count + "/add:${ac},upd:${uc},igr:${ic},err:${fc}\t" + DateUtilCustom.formatToTightYYYYMMDDhhmmss(new Date()));
return count;
}
void removeSeabird(DBCollection collection_seabird, List<String> batchDelList) {
if (collection_seabird != null && batchDelList!= null && batchDelList.size() >= BatchCount) {
while(batchDelList.size() > 0) {
try {
// String _id = batchDelList.get(0);
DBObject q = new BasicDBObject("_id", new ObjectId(batchDelList.get(0)));
// DBObject old = collection_seabird.findOne(q);
collection_seabird.remove(q);
}catch(Exception ex){
ex.printStackTrace();
}
batchDelList.remove(0);
}
}
}
private boolean isValidData(DBObject dbo) {
String error_isValidData = "isValidData error -> ";
String url = dbo.get("url") != null ? dbo.get("url").toString() : null;
// StringUtils.startsWithIgnoreCase(url, "http://")
if (StringUtils.isEmpty(url)) {
println(error_isValidData + "url is null");
return false;
}
if (!(StringUtils.startsWithIgnoreCase(url, "http://") || StringUtils.startsWithIgnoreCase(url, "https://"))
&& StringUtils.contains(url, "fangjia.com") == false) {
println(error_isValidData + "url:" + url);
return false;
}
String city = dbo.get("city") != null ? dbo.get("city").toString() : null;
if (StringUtils.isEmpty(city) || city.length() > 10) {
println(error_isValidData + "city:" + city);
return false;
}
String region = dbo.get("region") != null ? dbo.get("region").toString() : null;
if (StringUtils.isEmpty(region) || region.length() > 10) {
println(error_isValidData + "region:" + region);
return false;
}
String district_name = dbo.get("district_name") != null ? dbo.get("district_name").toString() : null;
if (StringUtils.isEmpty(district_name) || district_name.length() > 30) {
println(error_isValidData + "district_name:" + district_name);
return false;
}
if (dbo.get("area") == null || isDouble(dbo.get("area")) == false) {
println(error_isValidData + "area:" + dbo.get("area"));
return false;
}
double d_area = parseDouble(dbo.get("area"));
if (checkDate(dbo.get("publish_date")) == false) {
println(error_isValidData + "publish_date:" + dbo.get("publish_date"));
return false;
}
if (before2007Date == null) {
before2007Date = parseFromTightYYYYMMDD("2007-01-01");
}
if (dbo.get("publish_date") != null) {
Date publish_date = parseFromTightYYYYMMDD(dbo.get("publish_date").toString());
if (publish_date.before(before2007Date)) {
println(error_isValidData + "publish_date:" + dbo.get("publish_date").toString());
return false;
}
}
String cat = dbo.get("cat") != null ? dbo.get("cat").toString() : null;
int total_price = parseInt(dbo.get("total_price"));
if (cat == "sell") {
if (d_area < 10 || d_area > 2000) {
println(error_isValidData + "cat:" + cat + ",area:" + d_area);
return false;
}
if (total_price < 100000 || total_price > 900000000) {
println(error_isValidData + "cat:" + cat + ",total_price:" + total_price);
return false;
}
double avgPrice = total_price / d_area;
if (avgPrice > 300000 || avgPrice < 1000) {
println(error_isValidData + "cat:" + cat + ",avgPrice:" + avgPrice);
return false;
}
} else if (cat == "rent") {
if (d_area < 5 || d_area > 2000) {
println(error_isValidData + "cat:" + cat + ", area:" + d_area);
return false;
}
if (total_price < 10 || total_price > 10000000) {
println(error_isValidData + "cat:" + cat + ",total_price:" + total_price);
return false;
}
}
return true;
}
private DBObject formatToStandardData(DBObject dbo) {
if (StringUtils.isNotBlank((String) dbo.get("name")) && ((String) dbo.get("name")).length() > 50) {
dbo.put("name", dbo.get("name").toString().substring(0, 50));
}
if (StringUtils.isNotBlank((String) dbo.get("address")) && ((String) dbo.get("address")).length() > 50) {
dbo.removeField("address");
}
if (StringUtils.isNotBlank((String) dbo.get("mobile")) && ((String) dbo.get("mobile")).length() > 30) {
dbo.removeField("mobile");
}
if (StringUtils.isNotBlank((String) dbo.get("estate_charge"))
&& ((String) dbo.get("estate_charge")).length() > 40) {
dbo.removeField("estate_charge");
}
String url = processUrl(dbo.get("url") != null ? dbo.get("url").toString() : "");
String id = dbo.get(MongoObjectIDKey).toString();
// if (url.indexOf("fangjia.com") > -1) {
// id = dbo.get(MongoObjectIDKey).toString();
// }else {
// id = DigestUtils.md5Hex(url + "#" + url.hashCode());
// }
DBObject standardObj = new BasicDBObject();
standardObj.put(MongoObjectIDKey, id);
standardObj.put("url", url);
standardObj.put("cat", dbo.get("cat"));
standardObj.put("city", dbo.get("city").toString().trim());
standardObj.put("region", dbo.get("region").toString().trim());
standardObj.put("district_name", DataUtil.toDBCcase(dbo.get("district_name").toString()).trim());
standardObj.put("p_date", DateUtilCustom.formatToTightYYYYMMDD(dbo.get("publish_date")));
standardObj.put("c_date", DateUtilCustom.formatToTightYYYYMMDD(dbo.get("c_date")));
// 数字类型的字段
for (String key : NUMBER_FIELDS) {
Object val = dbo.get(key);
if (val != null && val instanceof Number) {
standardObj.put(key, val);
}
}
// 文本类型的字段
for (String key : TEXT_FIELDS) {
Object val = dbo.get(key);
// 判断文本字段如果太长,就不保存该字段
if (val != null && val instanceof String && val.toString().length() <= 1024) {
standardObj.put(key, val.toString());
}
}
// 其他字段
for (String key : OTHER_FIELDS) {
Object val = dbo.get(key);
if (val != null) {
standardObj.put(key, val);
}
}
// standardObj.put("md5_hash", getMd5Hash(standardObj))
return standardObj;
}
String processUrl(String url) {
String root = DataUtil.urlToSiteName(url);
if ("58.com".equals(root)) {
// http://huizhou.58.com/ershoufang/22416405318175x.shtml?psid=196886603191045603944627704&entinfo=22416405318175_0
if (url.indexOf("?") > -1) {
url = StringUtils.substringBefore(url, "?");
}
} else if ("anjuke.com".equals(root)) {
// http://wuhan.anjuke.com/prop/view/A417931448?from=comm_one-saleMetro&spread=commsearch_p
if (url.indexOf("?") > -1) {
url = StringUtils.substringBefore(url, "?");
}
} else if ("leju.com".equals(root)) {
// http://bj.esf.leju.com/detail/217448358#zn=pc-house-36
if (url.indexOf("#") > -1) {
url = StringUtils.substringBefore(url, "#");
}
}
return url;
}
static String err_dataClean = "dataClean() error -> ";
boolean dataClean(DBObject dbo) {
if (dbo == null) {
println(err_dataClean + " -> dbo == null");
return false;
}
if (dbo.get("_id") == null) {
println(err_dataClean + " -> _id == null");
return false;
}
if (isInteger(dbo.get("p_date")) == false) {
println(err_dataClean + " -> p_date");
return false;
}
List<String> keys = new ArrayList<String>(dbo.keySet());
for (String key : keys) {
if (!fields.contains(key)) {
dbo.removeField(key);
}
}
Integer pDate = parseInt(dbo.get("p_date"));
Date publishDate = parseDate(String.valueOf(pDate), "yyyyMMdd");
if (publishDate == null) {
println(err_dataClean + " -> publishDate == null");
return false;
}
dbo.put("publish_date", publishDate);
if (dbo.get("m_date") == null) {
dbo.put("m_date", dbo.get("c_date"));
}
if (dbo.get("m_date") == null) {
dbo.put("m_date", publishDate);
}
if(dbo.get("url") != null) {
String url = dbo.get("url").toString();
String root = DataUtil.urlToSiteName(url);
if (StringUtils.isNotBlank(root)) {
dbo.put("root", root);
}
}
return true;
}
// String importData2Elastic(Map map){
// if(map == null || map.size() == 0 || map.get("_id") == null || map.get("p_date") == null) {
// return null;
// }
//
// try{
// Integer pDate = parseInt(map.get("p_date"));
// if(pDate <= 0) {
// return null;
// }
// Date publishDate = parseDate(String.valueOf(pDate), "yyyyMMdd");
// if(publishDate == null) {
// return null;
// }
// String indexName = "seawater-" + DateFormatUtils.format(publishDate, "yyyy");
//
// String id = map.get("_id").toString();
//
// map.remove("_id");
// map.remove("c_date");
// map.remove("md5_hash");
//
// TransportClient transportClient = getCustomTransportClient(indexName);
// BulkRequestBuilder bulkRequest = getCustomBulkRequestBuilder(indexName);
// if(transportClient != null && bulkRequest != null) {
//// this.bulkRequest.add(transportClient.prepareIndex(indexName,"doc")
// bulkRequest.add(transportClient.prepareIndex(indexName,"doc")
// .setSource(JsonUtils.toJson(map), XContentType.JSON)
// .setId(id));
//
// int addedCount = 0;
// if(addedEsCountMap.containsKey(indexName)) {
// addedCount = addedEsCountMap.get(indexName);
// }
// addedCount++;
// addedEsCountMap.put(indexName, addedCount);
// }
//
// return indexName;
// }catch(Exception ex){
// println("importData2Elastic error -> map:" + com.alibaba.fastjson.JSON.toJSONString(map) + ", exception:" + ex.getMessage());
// ex.printStackTrace();
// }
// return null;
// }
static void println(String msg) {
System.out.println(msg);
}
static Date getCSTDate(Object obj) {
if (obj == null) {
return null;
}
try {
SimpleDateFormat sdf = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy", Locale.US);
return sdf.parse(obj.toString());
} catch (Exception e) {
}
return null;
}
static boolean checkDate(Object obj) {
if (obj == null) {
return false;
}
SimpleDateFormat sd = new SimpleDateFormat(DATE_PATTERN);
try {
sd.setLenient(false);
sd.parse(obj.toString());
} catch (Exception e) {
if(getCSTDate(obj.toString()) != null) {
return true;
}
return false;
}
return true;
}
static boolean checkDateTime(String str) {
SimpleDateFormat sd = new SimpleDateFormat(DATETIME_PATTERN);
try {
sd.setLenient(false);
sd.parse(str);
} catch (Exception e) {
if(getCSTDate(str) != null) {
return true;
}
}
return true;
}
public static Date parseFromTightYYYYMMDD(String s) {
if (s == null || s.length() == 0) {
return null;
}
SimpleDateFormat format = new SimpleDateFormat(DATE_PATTERN);
try {
return format.parse(s);
} catch (ParseException e) {
return getCSTDate(s);
}
}
public static boolean isDouble(Object input) {
if(input == null) {
return false;
}
try {
Double.parseDouble(input.toString());
return true;
}catch(Exception ex) {
return false;
}
}
public static double parseDouble(Object obj) {
if (obj == null) {
return 0;
}
try {
return Double.parseDouble(obj.toString());
} catch (Exception e) {
e.printStackTrace();
}
return 0;
}
public static boolean isInteger(Object input) {
if(input == null) {
return false;
}
try {
Integer.parseInt(input.toString());
return true;
}catch(Exception ex) {
return false;
}
}
public static int parseInt(Object obj) {
if (obj == null) {
return 0;
}
try {
return Integer.parseInt(obj.toString());
} catch (Exception e) {
e.printStackTrace();
}
return 0;
}
public static Date parseDate(String dateStr, String pattern) {
Date date = null;
if (StringUtils.isNotBlank(dateStr)) {
SimpleDateFormat sdf = new SimpleDateFormat(pattern);
try {
date = sdf.parse(dateStr);
} catch (ParseException e) {
}
}
return date;
}
static Date before2007Date = null;
public static final String DATE_PATTERN = "yyyy-MM-dd";
public static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss";
static String MongoObjectIDKey = "_id";
static List<String> fields = Arrays.asList("_id", "cat", "city", "region", "district_name", "total_price", "area",
"p_date", "url", "estate_type", "height", "floor", "room", "hall", "toilet", "fitment", "direction",
"rent_type", "user_id", "build_type", "build_date", "estate_charge", "status", "address", "block", "name",
"contact_people", "mobile", "company", "m_date", "floor_info", "community_support", "house_number",
"down_payment", "only_one", "buy_year", "unit_number", "unit_type", "owner_attitude", "service_desc",
"check_in_time", "rent_type", "pay_type", "house_allocation", "house_require", "house_advantage", "metro",
"line", "subject");
static String[] CATEGORIES = new String[] { "office_rent", "sell", "rent", "office_rent", "office_sell",
"shop_rent", "shop_sell" };
static String[] NUMBER_FIELDS = new String[] { "total_price", "area", "height", "floor", "room", "hall", "toilet",
"age" };
static String[] TEXT_FIELDS = new String[] { "name", "address", "block", "fitment", "estate_type", "mobile",
"build_type", "build_date", "company", "direction", "contact_people", "rent_type", "floor_info",
"house_number", "down_payment", "only_one", "buy_year", "unit_number", "unit_type", "owner_attitude",
"service_desc", "community_support", "description", "check_in_time", "house_allocation", "house_require",
"house_advantage", "pay_type", "metro", "line", "subject", "agency" };
static String[] OTHER_FIELDS = new String[] { "user_id", "publish_date" };
}
@@ -1,14 +1,21 @@
mongo.business.url=114.80.150.198
mongo.business.name=business
mongo.business.port=27017
#mongo.business.url=114.80.150.198
#mongo.business.name=business
#mongo.business.port=27017
mongo.fangjia_tmp.url=192.168.0.209
mongo.fangjia_tmp.name=fangjia_tmp
mongo.fangjia_tmp.port=55007
spring.datasource.log.url=jdbc:mysql://localhost:3306/batch_log
spring.datasource.log.username=root
spring.datasource.log.password=123456
spring.datasource.log.driver-class-name=com.mysql.jdbc.Driver
zk.server-lists=192.168.10.124:2181
zk.namespace=fangjia-job-basicdata
#zk.server-lists=192.168.10.124:2181
#zk.namespace=fangjia-job-basicdata
es.url=192.168.0.45
es.cluster.name=fangjia
es.206.url=192.168.0.206
es.206.cluster.name=fangjia