diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java
index e11cd2246..adfb2c7c8 100644
--- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java
+++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java
@@ -1847,6 +1847,9 @@ public class DataUtil {
//
// System.out.println(FourUpSixInto("258.393", 2));
+ String href = encodePicUrl("62afd81d303c3f827833bb03", "o", "upload");
+ String link = "http://static.fangjia.com/stc/fs/"+href;
+ System.out.println(link);
}
}
diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java
index 41941514a..4b72f411c 100644
--- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java
+++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java
@@ -105,6 +105,7 @@ public class FjtEvaluation implements SqlEntity{
private String certificateAddress; //产证地址
private Date downPdfDate;//下载报告时间
+ private Integer loan ;//是否续贷 1:是 0:否
@GeneratedValue(strategy = GenerationType.AUTO)
@Id
@@ -773,6 +774,15 @@ public class FjtEvaluation implements SqlEntity{
this.downPdfDate = downPdfDate;
}
+ @Column(name = "loan")
+ public Integer getLoan() {
+ return loan;
+ }
+
+ public void setLoan(Integer loan) {
+ this.loan = loan;
+ }
+
@Override
public String toString() {
return "FjtEvaluation{" +
diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ReportFileAttach.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ReportFileAttach.java
index a22ef8332..acc9fcd36 100644
--- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ReportFileAttach.java
+++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ReportFileAttach.java
@@ -11,7 +11,7 @@ public class ReportFileAttach implements SqlEntity {
private Integer userId;
private Integer evaluationId;
private String queryId;
- private Integer fileClass;
+ private Integer fileClass;//1:询价单 2:产证 3:房屋照片 4:其它附件 5:产调报告或权籍
private String fileName;
private String fileId;
diff --git a/projects/fangjia-datastore/pom.xml b/projects/fangjia-datastore/pom.xml
index e86529d96..5c5fb279c 100644
--- a/projects/fangjia-datastore/pom.xml
+++ b/projects/fangjia-datastore/pom.xml
@@ -33,7 +33,7 @@
true
lib/
com.fangjia.datastore.DataStoreServer
-
+
@@ -94,8 +94,8 @@
org.mongodb
- mongo-java-driver
- 2.6.5
+ mongodb-driver-legacy
+ 3.9.1
log4j
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/ConfigurationException.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/ConfigurationException.java
index 1f740a992..9631409cf 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/ConfigurationException.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/ConfigurationException.java
@@ -1,22 +1,21 @@
package com.fangjia.datastore;
-public class ConfigurationException extends Exception {
-
- private static final long serialVersionUID = 6570405263317311822L;
-
- public ConfigurationException() {
- }
-
- public ConfigurationException(String message) {
- super(message);
- }
-
- public ConfigurationException(Throwable cause) {
- super(cause);
- }
-
- public ConfigurationException(String message, Throwable cause) {
- super(message, cause);
- }
-
+public class ConfigurationException extends Exception
+{
+ private static final long serialVersionUID = 6570405263317311822L;
+
+ public ConfigurationException() {
+ }
+
+ public ConfigurationException(final String message) {
+ super(message);
+ }
+
+ public ConfigurationException(final Throwable cause) {
+ super(cause);
+ }
+
+ public ConfigurationException(final String message, final Throwable cause) {
+ super(message, cause);
+ }
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Cursor.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Cursor.java
index ab27690ef..1303b2372 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Cursor.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Cursor.java
@@ -1,80 +1,83 @@
package com.fangjia.datastore;
+import com.fangjia.datastore.mongo.protocol.QueryMessage;
import java.util.concurrent.atomic.AtomicLong;
-import com.fangjia.datastore.mongo.protocol.QueryMessage;
-
-public class Cursor {
- private static final AtomicLong ID_GENERATOR = new AtomicLong(1);
- private long id;
- private int limit;
- private int skip;
- private long totalCount;
- private QueryMessage queryMessage;
-
- public Cursor(int skip, int limit, QueryMessage queryMessage, long totalCount) {
- id = ID_GENERATOR.getAndIncrement();
- this.skip = skip;
- this.limit = limit;
- this.totalCount = totalCount;
- this.queryMessage = queryMessage;
- }
-
- public long getId() {
- return id;
- }
-
- public int getLimit() {
- return limit;
- }
-
- public void setLimit(int limit) {
- this.limit = limit;
- }
-
- public int nextSkip() {
- skip += limit;
- return skip;
- }
-
- public void setSkip(int skip) {
- this.skip = skip;
- }
-
- public boolean hasMore() {
- return this.totalCount > this.skip;
- }
-
- public long getTotalCount() {
- return totalCount;
- }
-
- public void setTotalCount(long totalCount) {
- this.totalCount = totalCount;
- }
-
- public QueryMessage getQueryMessage() {
- return queryMessage;
- }
-
- public void setQueryMessage(QueryMessage queryMessage) {
- this.queryMessage = queryMessage;
- }
-
- public int getSkip() {
- return skip;
- }
-
- public String toString() {
- StringBuilder builder = new StringBuilder();
- builder.append("{");
- builder.append("id:").append(id);
- builder.append(", limit:").append(limit);
- builder.append(", skip:").append(skip);
- builder.append(", totalCount:").append(totalCount);
- builder.append(", queryMessage:").append(queryMessage);
- builder.append("}");
- return builder.toString();
- }
-
+public class Cursor
+{
+ private static final AtomicLong ID_GENERATOR;
+ private long id;
+ private int limit;
+ private int skip;
+ private long totalCount;
+ private QueryMessage queryMessage;
+
+ public Cursor(final int skip, final int limit, final QueryMessage queryMessage, final long totalCount) {
+ this.id = Cursor.ID_GENERATOR.getAndIncrement();
+ this.skip = skip;
+ this.limit = limit;
+ this.totalCount = totalCount;
+ this.queryMessage = queryMessage;
+ }
+
+ public long getId() {
+ return this.id;
+ }
+
+ public int getLimit() {
+ return this.limit;
+ }
+
+ public void setLimit(final int limit) {
+ this.limit = limit;
+ }
+
+ public int nextSkip() {
+ return this.skip += this.limit;
+ }
+
+ public void setSkip(final int skip) {
+ this.skip = skip;
+ }
+
+ public boolean hasMore() {
+ return this.totalCount > this.skip;
+ }
+
+ public long getTotalCount() {
+ return this.totalCount;
+ }
+
+ public void setTotalCount(final long totalCount) {
+ this.totalCount = totalCount;
+ }
+
+ public QueryMessage getQueryMessage() {
+ return this.queryMessage;
+ }
+
+ public void setQueryMessage(final QueryMessage queryMessage) {
+ this.queryMessage = queryMessage;
+ }
+
+ public int getSkip() {
+ return this.skip;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder builder = new StringBuilder();
+ builder.append("{");
+ builder.append("id:").append(this.id);
+ builder.append(", limit:").append(this.limit);
+ builder.append(", skip:").append(this.skip);
+ builder.append(", totalCount:").append(this.totalCount);
+ builder.append(", queryMessage:").append(this.queryMessage);
+ builder.append("}");
+ return builder.toString();
+ }
+
+ static {
+ ID_GENERATOR = new AtomicLong(1L);
+ }
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStore.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStore.java
index 52f2196ed..619d7f038 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStore.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStore.java
@@ -1,44 +1,35 @@
package com.fangjia.datastore;
-import java.util.Collection;
import java.util.List;
-
import org.bson.BSONObject;
-
-import com.fangjia.datastore.config.Reloadable;
-import com.fangjia.datastore.config.StoreConfig;
import com.fangjia.datastore.solr.Index;
+import java.util.Collection;
+import com.fangjia.datastore.config.StoreConfig;
+import com.fangjia.datastore.config.Reloadable;
-public interface DataStore extends Reloadable {
-
- StoreConfig getStoreConfig();
-
- Mongo getMongo();
-
- Object getAttribute(String key);
-
- Object setAttribute(String key, Object attr);
-
- Collection getIndexes() throws DataStoreException;
-
- /**
- * 按查询条件重建索引
- *
- * @param query
- * @throws DataStoreException
- */
- BSONObject reIndex(BSONObject query) throws DataStoreException;
-
- void ensureIndex(Collection indexes) throws DataStoreException;
-
- QueryResponse find(BSONObject query, BSONObject fields, BSONObject sort, BSONObject facet, int skip, int limit, boolean bulk,
- boolean fromMaster) throws DataStoreException;
-
- int count(BSONObject query, boolean fromMaster) throws DataStoreException;
-
- void remove(BSONObject query) throws DataStoreException;
-
- void update(BSONObject query, BSONObject data,String ip) throws DataStoreException;
-
- void save(List extends BSONObject> docList) throws DataStoreException;
+public interface DataStore extends Reloadable
+{
+ StoreConfig getStoreConfig();
+
+ Mongo getMongo();
+
+ Object getAttribute(final String p0);
+
+ Object setAttribute(final String p0, final Object p1);
+
+ Collection getIndexes() throws DataStoreException;
+
+ BSONObject reIndex(final BSONObject p0) throws DataStoreException;
+
+ void ensureIndex(final Collection p0) throws DataStoreException;
+
+ QueryResponse find(final BSONObject p0, final BSONObject p1, final BSONObject p2, final BSONObject p3, final int p4, final int p5, final boolean p6, final boolean p7) throws DataStoreException;
+
+ int count(final BSONObject p0, final boolean p1) throws DataStoreException;
+
+ void remove(final BSONObject p0) throws DataStoreException;
+
+ int update(final BSONObject p0, final BSONObject p1) throws DataStoreException;
+
+ void save(final List extends BSONObject> p0) throws DataStoreException;
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContext.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContext.java
index 7777ebef4..bb2f19617 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContext.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContext.java
@@ -3,15 +3,15 @@ package com.fangjia.datastore;
import com.fangjia.datastore.config.DataStoreConfig;
import com.fangjia.datastore.config.Reloadable;
-public interface DataStoreContext extends Reloadable {
-
- DataStoreConfig getDataStoreConfig();
-
- DataStore getDataStore(String storeName) throws DataStoreException;
-
- Cursor getCursor(long cursorId);
-
- void addCursor(Cursor cursor);
-
- Cursor removeCursor(long cursorId);
+public interface DataStoreContext extends Reloadable
+{
+ DataStoreConfig getDataStoreConfig();
+
+ DataStore getDataStore(final String p0) throws DataStoreException;
+
+ Cursor getCursor(final long p0);
+
+ void addCursor(final Cursor p0);
+
+ Cursor removeCursor(final long p0);
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContextImpl.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContextImpl.java
index 2e11d6ca7..54fde188a 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContextImpl.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContextImpl.java
@@ -1,56 +1,61 @@
package com.fangjia.datastore;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-import com.fangjia.datastore.config.DataStoreConfig;
import com.fangjia.datastore.config.StoreConfig;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.HashMap;
+import java.util.concurrent.TimeUnit;
import com.google.common.collect.MapMaker;
+import java.util.Map;
+import com.fangjia.datastore.config.DataStoreConfig;
-public class DataStoreContextImpl implements DataStoreContext {
-
- private final DataStoreConfig config;
- private final Map cursorMap;
- private final Map stores;
-
- public DataStoreContextImpl(DataStoreConfig config) throws ConfigurationException {
- this.config = config;
- cursorMap = new MapMaker().concurrencyLevel(50).expiration(config.getServerConfig().getCursorExpirationMinutes(),
- TimeUnit.MINUTES).makeMap();
- Set storeNames = config.getStoreNames();
- stores = new HashMap(storeNames.size());
- for (String storeName : storeNames) {
- StoreConfig storeConfig = this.config.getStoreConfig(storeName);
- stores.put(storeName, new DataStoreImpl(storeConfig, new Mongo(storeConfig.getMongoConfig())));
- }
- }
-
- public DataStoreConfig getDataStoreConfig() {
- return config;
- }
-
- public DataStore getDataStore(String storeName) throws DataStoreException {
- if (!stores.containsKey(storeName)) {
- throw new DataStoreException("unknown store name: \"" + storeName + "\"");
- }
- return stores.get(storeName);
- }
-
- public void reload() throws ConfigurationException {
- config.reload();
- }
-
- public Cursor getCursor(long cursorId) {
- return this.cursorMap.get(cursorId);
- }
-
- public void addCursor(Cursor cursor) {
- this.cursorMap.put(cursor.getId(), cursor);
- }
-
- public Cursor removeCursor(long cursorId) {
- return this.cursorMap.remove(cursorId);
- }
+public class DataStoreContextImpl implements DataStoreContext
+{
+ private final DataStoreConfig config;
+ private final Map cursorMap;
+ private final Map stores;
+
+ public DataStoreContextImpl(final DataStoreConfig config) throws ConfigurationException {
+ this.config = config;
+ this.cursorMap = new MapMaker().concurrencyLevel(50).expiration((long)config.getServerConfig().getCursorExpirationMinutes(), TimeUnit.MINUTES).makeMap();
+ final Set storeNames = config.getStoreNames();
+ this.stores = new HashMap(storeNames.size());
+ for (final String storeName : storeNames) {
+ final StoreConfig storeConfig = this.config.getStoreConfig(storeName);
+ this.stores.put(storeName, new DataStoreImpl(storeConfig, new Mongo(storeConfig.getMongoConfig())));
+ }
+ }
+
+ @Override
+ public DataStoreConfig getDataStoreConfig() {
+ return this.config;
+ }
+
+ @Override
+ public DataStore getDataStore(final String storeName) throws DataStoreException {
+ if (!this.stores.containsKey(storeName)) {
+ throw new DataStoreException("unknown store name: \"" + storeName + "\"");
+ }
+ return this.stores.get(storeName);
+ }
+
+ @Override
+ public void reload() throws ConfigurationException {
+ this.config.reload();
+ }
+
+ @Override
+ public Cursor getCursor(final long cursorId) {
+ return this.cursorMap.get(cursorId);
+ }
+
+ @Override
+ public void addCursor(final Cursor cursor) {
+ this.cursorMap.put(cursor.getId(), cursor);
+ }
+
+ @Override
+ public Cursor removeCursor(final long cursorId) {
+ return this.cursorMap.remove(cursorId);
+ }
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreException.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreException.java
index 034460ce5..23e8b15cb 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreException.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreException.java
@@ -1,23 +1,21 @@
package com.fangjia.datastore;
-public class DataStoreException extends Exception {
-
- private static final long serialVersionUID = -5492973985910379215L;
-
- public DataStoreException() {
- super();
- }
-
- public DataStoreException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public DataStoreException(String message) {
- super(message);
- }
-
- public DataStoreException(Throwable cause) {
- super(cause);
- }
-
+public class DataStoreException extends Exception
+{
+ private static final long serialVersionUID = -5492973985910379215L;
+
+ public DataStoreException() {
+ }
+
+ public DataStoreException(final String message, final Throwable cause) {
+ super(message, cause);
+ }
+
+ public DataStoreException(final String message) {
+ super(message);
+ }
+
+ public DataStoreException(final Throwable cause) {
+ super(cause);
+ }
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreImpl.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreImpl.java
index 9f10d1e4a..215b15b64 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreImpl.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreImpl.java
@@ -1,702 +1,637 @@
package com.fangjia.datastore;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.HashMap;
+import com.fangjia.datastore.utils.LoggerFactory;
+import com.mongodb.WriteResult;
+import com.mongodb.WriteConcern;
import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.commons.collections.CollectionUtils;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collections;
import org.apache.solr.client.solrj.response.FacetField;
-import org.apache.solr.client.solrj.response.FacetField.Count;
-import org.bson.BSONObject;
+import java.util.Arrays;
+import java.util.ArrayList;
+import com.fangjia.datastore.solr.SolrException;
+import java.util.Iterator;
import org.bson.BasicBSONObject;
import org.bson.types.ObjectId;
+import java.util.List;
+import com.mongodb.BasicDBObject;
+import java.util.Date;
+import org.apache.commons.collections.CollectionUtils;
+import org.bson.BSONObject;
+import com.fangjia.datastore.solr.Index;
+import java.util.Collection;
+import com.mongodb.DBCollection;
+import com.mongodb.gridfs.GridFS;
+import java.util.HashMap;
+import com.fangjia.datastore.solr.CoreImpl;
+import com.fangjia.datastore.utils.MongoUtils;
+import com.fangjia.datastore.config.StoreConfig;
+import java.util.Map;
+import com.fangjia.datastore.solr.Core;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.Set;
+import com.mongodb.DBObject;
+import java.util.Comparator;
import org.slf4j.Logger;
-import com.fangjia.datastore.config.StoreConfig;
-import com.fangjia.datastore.solr.Core;
-import com.fangjia.datastore.solr.CoreImpl;
-import com.fangjia.datastore.solr.Index;
-import com.fangjia.datastore.solr.SolrException;
-import com.fangjia.datastore.solr.SolrSchema;
-import com.fangjia.datastore.utils.LoggerFactory;
-import com.fangjia.datastore.utils.MongoUtils;
-import com.mongodb.BasicDBObject;
-import com.mongodb.Bytes;
-import com.mongodb.DBCollection;
-import com.mongodb.DBObject;
-import com.mongodb.WriteConcern;
-import com.mongodb.gridfs.GridFS;
-
-public class DataStoreImpl implements DataStore {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(DataStoreImpl.class);
- private final static Logger PERFORMANCE_LOGGER = LoggerFactory.getPerformanceLogger(DataStoreImpl.class);
- private final static String SEAWEED_UPDATE_LOG = "seaweed_update_log";
-
- private final static Comparator OBJECT_ID_COMPARATOR = new Comparator() {
-
- public int compare(DBObject o1, DBObject o2) {
- ObjectId id1 = (ObjectId) o1.get("_id");
- ObjectId id2 = (ObjectId) o2.get("_id");
- return id1.compareTo(id2);
- }
- };
- private final static Set SPECIAL_KEYS = new HashSet(Arrays.asList("_id", "$bulk", SolrSchema.VERTION));
- private final static int MAX_REMOVE_ROWS = 1000;
-
- private final AtomicBoolean indexing = new AtomicBoolean(false);
-
- private final Core core;
- private final Map attributes;
- private final StoreConfig storeConfig;
- private Mongo mongo;
- private String mongoDbName;
- private String mongoCollectionName;
- private final FileStore fileStore;
-
- public DataStoreImpl(StoreConfig storeConfig, Mongo mongo) throws ConfigurationException {
- String[] split = MongoUtils.parseFullCollectionName(storeConfig.getName());
- if (null == split) {
- throw new IllegalArgumentException("invalid store name: \"" + storeConfig.getName() + "\"");
- }
- this.storeConfig = storeConfig;
- this.mongo = mongo;
- mongoDbName = split[0];
- mongoCollectionName = split[1];
- core = new CoreImpl(storeConfig);
- attributes = new HashMap();
-
- if (storeConfig.isTextFieldToGridFS()) {
- fileStore = new FileStoreImpl(new GridFS(mongo.getDB(mongoDbName), mongoCollectionName), storeConfig.getFields());
- } else {
- fileStore = null;
- }
- }
-
- public void reload() throws ConfigurationException {
- this.core.reload();
- }
-
- public DBCollection getMongoCollection() {
- return mongo.getDB(mongoDbName).getCollection(mongoCollectionName);
- }
-
- public DBCollection getUpdateLogMongoCollection() {
- return mongo.getDB(mongoDbName).getCollection(SEAWEED_UPDATE_LOG);
- }
-
- public StoreConfig getStoreConfig() {
- return storeConfig;
- }
-
- public Mongo getMongo() {
- return mongo;
- }
-
- public Object getAttribute(String key) {
- return attributes.get(key);
- }
-
- public Object setAttribute(String key, Object attr) {
- return attributes.put(key, attr);
- }
-
- public Collection getIndexes() throws DataStoreException {
- try {
- return core.getIndexes();
- } catch (Exception e) {
- throw new DataStoreException("error get indexes", e);
- }
- }
-
- public BSONObject reIndex(BSONObject query) throws DataStoreException {
- try {
- // 如没有查询条件,则全部重建
- if (null == query || CollectionUtils.isEmpty(query.keySet())) {
- if (indexing.compareAndSet(false, true)) {
- String currentVersion = Core.VERSION_DATE_FORMAT.format(new Date()); // 操作发起的时间戳
- Iterator extends BSONObject> i = this.getMongoCollection().find().addOption(Bytes.QUERYOPTION_NOTIMEOUT)
- .sort(new BasicDBObject("_id", 1));
- core.reIndex(null, i);
- // 删除重建之前的索引
- query = new BasicBSONObject();
- BSONObject versionObj = new BasicBSONObject();
- versionObj.put("$lte", currentVersion);
- query.put(SolrSchema.VERTION, versionObj);
- core.deleteByQuery(query);
- return query;
- } else {
- throw new DataStoreException(storeConfig.getName() + ": the re-index is in progress.");
- }
- } else {
- doReindexByQuery(query);
- return null;
- }
- } catch (Exception e) {
- throw new DataStoreException("error re-index", e);
- } finally {
- indexing.compareAndSet(true, false);
- }
- }
-
- private interface Closure {
- /**
- * 返回false则跳出遍历
- *
- * @param input
- * @return
- * @throws SolrException
- * @throws DataStoreException
- */
- public boolean execute(List input) throws SolrException, DataStoreException;
- }
-
- /**
- * 根据查询条件重建索引
- *
- * @param query
- * @throws DataStoreException
- */
- private void doReindexByQuery(BSONObject query) throws DataStoreException {
- String operationName = this.storeConfig.getName() + "_re-index_" + System.currentTimeMillis();
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info(operationName + ": query=" + query);
- }
-
- try {
- iterateSolrQuery(operationName, query, this.storeConfig.getDataStoreConfig().getServerConfig().getReIndexBatchSize(),
- new Closure() {
-
- public boolean execute(List objIdList) throws SolrException, DataStoreException {
- DBObject seletor = new BasicDBObject();
- seletor.put("_id", new BasicDBObject("$in", objIdList));
- Iterator extends BSONObject> i = getMongoCollection().find(seletor);
- core.reIndex(objIdList, i);
- return true;
- }
-
- });
- core.deleteByQuery(query);
- } catch (SolrException e) {
- throw new DataStoreException(operationName + ": ", e);
- }
-
- }
-
- /**
- * 根据query遍历。
- * 这个方法只适用于reIndex, update, remove等会重建索引的操作, 不能简单的用来遍历
- *
- * @param operationName
- * @param query
- * @param batchSize
- * @param closure
- * @throws SolrException
- * @throws DataStoreException
- */
- private void iterateSolrQuery(String operationName, BSONObject query, int batchSize, Closure closure) throws SolrException,
- DataStoreException {
- if (null == query.get(SolrSchema.VERTION)) {
- String currentVersion = Core.VERSION_DATE_FORMAT.format(new Date()); // 操作发起的时间戳
- BSONObject versionObj = new BasicBSONObject();
- versionObj.put("$lte", currentVersion);
- query.put(SolrSchema.VERTION, versionObj); // 只遍历时间戳之前的数据
- }
-
- long startTime = System.currentTimeMillis();
- int currentPage = 0; // 当前遍历页面
- long c = 0;
- long lastC = c;
- int round = 0;
- long roundIndexCount = 0;
- QueryResponse response = null;
- int lastNumFound = -1;
- while (true) {
- response = findInSolr(query, null, null, currentPage * batchSize, batchSize, true, true);
- if (currentPage == 0) {// 每轮第一次迭代
- if (lastNumFound == -1) {
- // 第一轮
- lastNumFound = response.getNumFound();
- } else if (response.getNumFound() > 0 && lastNumFound == response.getNumFound()) {
- long sleep = 20000;
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("{}[{}]: wait solr commit, sleep {}s.", new Object[] { operationName, round, sleep / 1000 });
- }
- try {
- Thread.sleep(sleep);
- } catch (InterruptedException e) {
- throw new DataStoreException(operationName + ": ", e);
- }
- response = findInSolr(query, null, null, 0, 1, true, true);
- if (response.getNumFound() > 0 && lastNumFound == response.getNumFound()) {
- // TODO: 可能需要删除solr中有但mongodb中没有的索引数据
- // core.deleteByQuery(query);
- break;
- }
- lastNumFound = response.getNumFound();
- continue;
- } else {
- lastNumFound = response.getNumFound();
- }
- }
-
- int returnedSize = response.getNumReturned();
- List idList = response.getIdList();
- // 有可能为空
- if (null != idList && idList.size() > 0) {
- // 执行重建
- if (!closure.execute(idList)) {
- // 返回false则不再遍历
- break;
- }
- }
- c += returnedSize;
- roundIndexCount += returnedSize;
- if (c - lastC >= batchSize && LOGGER.isInfoEnabled()) {
- LOGGER.info("{}[{}]: {}/{}, {}/s", new Object[] { operationName, round, c, response.getNumFound(),
- (int) (c / ((System.currentTimeMillis() - startTime) / 1000.0)) });
- lastC = c;
- }
-
- // 因为重建后的索引已不在query条件内,所以query条件的整个集合大小随着遍历的往前进行,一直在变动
- // 而分页是顺序向前的,所以一次执行,不能保证全部遍历到,所以要多次执行
- if (currentPage == 0 && returnedSize >= lastNumFound) {
- // 第一页数量就小于BATCH_SIZE,则不用往下遍历
- break;
- } else if (returnedSize == 0 || roundIndexCount >= lastNumFound) {
- // 本轮重建数量大于总数量,从头再来过
- currentPage = 0;
- round++;
- roundIndexCount = 0;
- } else {
- // 当前页数量跟BATCH_SIZE一致,向前翻页
- currentPage++;
- }
- }
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("{}[{}] finished, total time: {}s, count: {}/{}", new Object[] { operationName, round,
- (System.currentTimeMillis() - startTime) / 1000, roundIndexCount, c });
- }
- }
-
- public QueryResponse find(BSONObject query, BSONObject fields, BSONObject sort, BSONObject facet, int skip, int limit,
- boolean bulk, boolean fromMaster) throws DataStoreException {
- checkFields(fields);
- try {
- QueryResponse response = findInSolr(query, sort, facet, skip, limit, fromMaster, bulk);
- List docList = find(response, fields);
- int numReturned = docList.size() - 1;
- if (numReturned > response.getNumFound()) {
- response.setNumFound(numReturned);
- docList.get(0).put("$numFound", numReturned);
- }
- if (numReturned != response.getNumReturned()) {
- response.setNumReturned(numReturned);
- }
- response.setDocList(docList);
- return response;
- } catch (SolrException e) {
- throw new DataStoreException("error execute find{query=" + query + ", fields=" + fields + ", sort=" + sort
- + ", facet=" + facet + ", skip=" + skip + ", limit=" + limit + "}: ", e);
- }
- }
-
- private QueryResponse findInSolr(BSONObject query, BSONObject sort, BSONObject facet, int skip, int limit,
- boolean fromMaster, boolean bulk) throws SolrException, DataStoreException {
- QueryResponse response = null;
- List idList = null;
- if (null == facet || facet.keySet().size() == 0) {
- idList = getObjectIdsInQuery(query);
- }
- if (null != idList && idList.size() > 0) {
- response = new QueryResponse();
- response.setIdList(idList);
- response.setNumReturned(idList.size());
- } else {
- response = core.find(query, sort, facet, skip, limit, fromMaster, bulk);
- }
- return response;
- }
-
- @SuppressWarnings("unchecked")
- private List getObjectIdsInQuery(BSONObject query) throws DataStoreException {
- List ret = null;
- Object value = query.get("_id");
- if (null == value) {
- ret = null;
- } else if (value instanceof BSONObject) {
- List> ids = (List>) ((BSONObject) value).get("$in");
- if (null != ids && ids.size() > 0) {
- if (ids.get(0) instanceof ObjectId) {
- ret = (List) ids;
- } else {
- ret = new ArrayList(ids.size());
- for (Object id : ids) {
- ret.add(new ObjectId(String.valueOf(id)));
- }
- }
- }
- } else if (value instanceof ObjectId) {
- ret = Arrays.asList((ObjectId) value);
- } else if (value instanceof String) {
- ret = Arrays.asList(new ObjectId((String) value));
- }
- if (ret != null && containsNoneIdQuery(query)) {
- throw new DataStoreException("id query with other query:" + query);
- }
- return ret;
- }
-
- private boolean containsNoneIdQuery(BSONObject query) {
- if (null == query) {
- return false;
- }
- for (String key : query.keySet()) {
- if (SPECIAL_KEYS.contains(key)) {
- continue;
- }
- return true;
- }
- return false;
- }
-
- public int count(BSONObject query, boolean fromMaster) throws DataStoreException {
- try {
- QueryResponse response = core.find(query, null, null, 0, 0, fromMaster, false);
- List docList = find(response, null);
- response.setDocList(docList);
- return response.getNumFound();
- } catch (SolrException e) {
- throw new DataStoreException("error execute count{query=" + query + "}: ", e);
- }
- }
-
- private List find(QueryResponse response, BSONObject fields) throws DataStoreException {
- List idList = response.getIdList();
- List newDocList = new ArrayList(null == idList ? 1 : (idList.size() + 1));
-
- DBObject meta = new BasicDBObject("$numFound", response.getNumFound());
- List facetFieldList = response.getFacetFieldList();
- if (null != facetFieldList && facetFieldList.size() > 0) {
- List facetList = new ArrayList(facetFieldList.size());
- for (FacetField facetField : facetFieldList) {
- DBObject facet = new BasicDBObject();
- facet.put("name", facetField.getName());
- facet.put("end", facetField.getEnd());
- facet.put("gap", facetField.getGap());
- facet.put("valueCount", facetField.getValueCount());
- List values = facetField.getValues();
- if (null != values && values.size() > 0) {
- List vs = new ArrayList(values.size());
- for (Count c : values) {
- DBObject v = new BasicDBObject();
- v.put("name", c.getName());
- v.put("count", c.getCount());
- vs.add(v);
- }
- facet.put("values", vs);
- }
- facetList.add(facet);
- }
- meta.put("$facet", facetList);
- }
- newDocList.add(meta);
-
- if (null != idList && idList.size() > 0) {
- /**
- * 此处是为了性能考虑
- * 当查询只需要_id的时候,直接返回从solr中检索出的idList即可
- * 但是由于存在solr中存在的_id在mongodb中不存在,所以把这段代码注释了,从而一定程度降低了只查询_id请求的性能,保证了数据一致性
- */
-// if (null != fields && fields.containsField("_id") && fields.keySet().size() == 1) {
-// for (ObjectId id : idList) {
-// newDocList.add(new BasicDBObject("_id", id));
-// }
-// } else {
- DBObject query = new BasicDBObject();
- if (idList.size() == 1) {
- query.put("_id", idList.get(0));
- } else {
- query.put("_id", new BasicDBObject("$in", idList));
- }
- DBObject fieldsQuery = null;
- if (null != fields) {
- fieldsQuery = new BasicDBObject();
- fieldsQuery.putAll(fields);
- }
-
- long start = System.currentTimeMillis();
- List docList = this.getMongoCollection().find(query, fieldsQuery).toArray();
- Collections.sort(docList, OBJECT_ID_COMPARATOR);
- long time = System.currentTimeMillis() - start;
- if (time > 100) {
- if (PERFORMANCE_LOGGER.isWarnEnabled()) {
- PERFORMANCE_LOGGER.warn(this.storeConfig.getName() + ": query mongodb, id size: " + idList.size()
- + ", time: " + time);
- }
- }
- if (null != docList && !docList.isEmpty()) {
- for (Iterator i = idList.iterator(); i.hasNext();) {
- int index = binarySearch(docList, i.next());
- if (index >= 0) {
- newDocList.add(docList.get(index));
- }
- }
- }
-
- if (storeConfig.isTextFieldToGridFS()) {
- newDocList = fileStore.find(newDocList, fields);
- }
-// }
- }
- return newDocList;
- }
-
- private static int binarySearch(List list, ObjectId id) {
- int low = 0;
- int high = list.size() - 1;
-
- while (low <= high) {
- int mid = (low + high) >>> 1;
- ObjectId midVal = (ObjectId) list.get(mid).get("_id");
- int cmp = midVal.compareTo(id);
-
- if (cmp < 0)
- low = mid + 1;
- else if (cmp > 0)
- high = mid - 1;
- else
- return mid; // key found
- }
- return -(low + 1); // key not found
- }
-
- private boolean isOpSupported(String op) {
- return "$inc".equals(op) || "$set".equals(op) || "$unset".equals(op);
- }
-
- private void checkUpdateFields(BSONObject data) throws DataStoreException {
- for (String op : data.keySet()) {
- if (!isOpSupported(op)) {
- throw new DataStoreException("modifier operation " + op + " is not supported!");
- }
- BSONObject update = (BSONObject) data.get(op);
- if (null != update) {
- for (String field : update.keySet()) {
- if (!storeConfig.containsField(field)) {
- throw new DataStoreException("unknown field: " + field + " !");
- }
- if (!storeConfig.isFieldValueLegal(field, update.get(field))) {
- throw new DataStoreException("invalid value " + update.get(field) + " of field '" + field + "'!");
- }
- }
- }
- }
- }
-
- private void checkFields(BSONObject fields) throws DataStoreException {
- if (null != fields) {
- for (String field : fields.keySet()) {
- if (!storeConfig.containsField(field)) {
- throw new DataStoreException("unknown field: " + field + " !");
- }
- }
- }
- }
-
- public void update(final BSONObject query, final BSONObject data,final String ip) throws DataStoreException {
- checkUpdateFields(data);
- String operationName = this.storeConfig.getName() + "_update_" + System.currentTimeMillis();
-
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info(operationName + ": query=" + query);
- }
-
- try {
- iterateSolrQuery(operationName, query, 1000, new Closure() {
-
- public boolean execute(List objIdList) throws SolrException, DataStoreException {
- final DBObject update = new BasicDBObject();
- final String ipAddress = ip;
- update.putAll(data);
-
- // 更新大字段关联文件
- if (storeConfig.isTextFieldToGridFS()) {
- fileStore.update(objIdList, update);
-
- // 把要更新的大字段数据,置为null,不需要把数据插入主表
- BSONObject realData = (BSONObject) update.get("$set");
- if (null != realData) {
- Set removeField = new HashSet();
- for (String field : realData.keySet()) {
- if (storeConfig.isBigField(field)) {
- // realData.removeField(field);
- removeField.add(field);
- }
- }
- realData.keySet().removeAll(removeField);
- }
- }
-
- DBObject seletor = new BasicDBObject();
- if (objIdList.size() == 1) {
- seletor.put("_id", objIdList.get(0));
- } else {
- seletor.put("_id", new BasicDBObject("$in", objIdList));
- }
- getMongoCollection().update(seletor, update, false, true, WriteConcern.NORMAL);
- //修改小区状态为1时记录状态
- if(update.get("$set")!=null){
- BasicBSONObject updateData = new BasicBSONObject();
- updateData = (BasicBSONObject) update.get("$set");
- String status = updateData.getString("status");
- if(status!=null && "1".equals(status)){
- DBObject saveObj = new BasicDBObject();
- saveObj.put("ip", ipAddress);
- saveObj.put("districtIp", objIdList.get(0).toString());
-
- saveObj.put("type", "remove");
- saveObj.put("c_date", new Date());
- getUpdateLogMongoCollection().save(saveObj);
- }
+public class DataStoreImpl implements DataStore
+{
+ private static final Logger LOGGER;
+ private static final Logger PERFORMANCE_LOGGER;
+ private static final String SEAWEED_UPDATE_LOG = "seaweed_update_log";
+ private static final Comparator OBJECT_ID_COMPARATOR;
+ private static final Set SPECIAL_KEYS;
+ private static final int MAX_REMOVE_ROWS = 10;
+ private final AtomicBoolean indexing;
+ private final Core core;
+ private final Map attributes;
+ private final StoreConfig storeConfig;
+ private Mongo mongo;
+ private String mongoDbName;
+ private String mongoCollectionName;
+ private final FileStore fileStore;
+
+ public DataStoreImpl(final StoreConfig storeConfig, final Mongo mongo) throws ConfigurationException {
+ this.indexing = new AtomicBoolean(false);
+ final String[] split = MongoUtils.parseFullCollectionName(storeConfig.getName());
+ if (null == split) {
+ throw new IllegalArgumentException("invalid store name: \"" + storeConfig.getName() + "\"");
+ }
+ this.storeConfig = storeConfig;
+ this.mongo = mongo;
+ this.mongoDbName = split[0];
+ this.mongoCollectionName = split[1];
+ this.core = new CoreImpl(storeConfig);
+ this.attributes = new HashMap();
+ if (storeConfig.isTextFieldToGridFS()) {
+ this.fileStore = new FileStoreImpl(new GridFS(mongo.getDB(this.mongoDbName), this.mongoCollectionName), storeConfig.getFields());
+ }
+ else {
+ this.fileStore = null;
+ }
+ }
+
+ @Override
+ public void reload() throws ConfigurationException {
+ this.core.reload();
+ }
+
+ public DBCollection getMongoCollection() {
+ return this.mongo.getDB(this.mongoDbName).getCollection(this.mongoCollectionName);
+ }
+
+ public DBCollection getUpdateLogMongoCollection() {
+ return this.mongo.getDB(this.mongoDbName).getCollection("seaweed_update_log");
+ }
+
+ @Override
+ public StoreConfig getStoreConfig() {
+ return this.storeConfig;
+ }
+
+ @Override
+ public Mongo getMongo() {
+ return this.mongo;
+ }
+
+ @Override
+ public Object getAttribute(final String key) {
+ return this.attributes.get(key);
+ }
+
+ @Override
+ public Object setAttribute(final String key, final Object attr) {
+ return this.attributes.put(key, attr);
+ }
+
+ @Override
+ public Collection getIndexes() throws DataStoreException {
+ try {
+ return this.core.getIndexes();
+ }
+ catch (Exception e) {
+ throw new DataStoreException("error get indexes", e);
+ }
+ }
+
+ @Override
+ public BSONObject reIndex(BSONObject query) throws DataStoreException {
+ try {
+ if (null != query && !CollectionUtils.isEmpty((Collection)query.keySet())) {
+ this.doReindexByQuery(query);
+ return null;
+ }
+ if (this.indexing.compareAndSet(false, true)) {
+ final String currentVersion = Core.VERSION_DATE_FORMAT.format(new Date());
+ final Iterator extends BSONObject> i = (Iterator extends BSONObject>)this.getMongoCollection().find().addOption(16).sort((DBObject)new BasicDBObject("_id", (Object)1));
+ this.core.reIndex(null, i);
+ query = (BSONObject)new BasicBSONObject();
+ final BSONObject versionObj = (BSONObject)new BasicBSONObject();
+ versionObj.put("$lte", (Object)currentVersion);
+ query.put("_version", (Object)versionObj);
+ this.core.deleteByQuery(query);
+ return query;
+ }
+ throw new DataStoreException(this.storeConfig.getName() + ": the re-index is in progress.");
+ }
+ catch (Exception e) {
+ throw new DataStoreException("error re-index", e);
+ }
+ finally {
+ this.indexing.compareAndSet(true, false);
+ }
+ }
+
+ private void doReindexByQuery(final BSONObject query) throws DataStoreException {
+ final String operationName = this.storeConfig.getName() + "_re-index_" + System.currentTimeMillis();
+ if (DataStoreImpl.LOGGER.isInfoEnabled()) {
+ DataStoreImpl.LOGGER.info(operationName + ": query=" + query);
+ }
+ try {
+ this.iterateSolrQuery(operationName, query, this.storeConfig.getDataStoreConfig().getServerConfig().getReIndexBatchSize(), new Closure() {
+ @Override
+ public boolean execute(final List objIdList) throws SolrException, DataStoreException {
+ final DBObject seletor = (DBObject)new BasicDBObject();
+ seletor.put("_id", (Object)new BasicDBObject("$in", (Object)objIdList));
+ final Iterator extends BSONObject> i = (Iterator extends BSONObject>)DataStoreImpl.this.getMongoCollection().find(seletor);
+ DataStoreImpl.this.core.reIndex(objIdList, i);
+ return true;
+ }
+ });
+ this.core.deleteByQuery(query);
+ }
+ catch (SolrException e) {
+ throw new DataStoreException(operationName + ": ", e);
+ }
+ }
+
+ private void iterateSolrQuery(final String operationName, final BSONObject query, final int batchSize, final Closure closure) throws SolrException, DataStoreException {
+ if (null == query.get("_version")) {
+ final String currentVersion = Core.VERSION_DATE_FORMAT.format(new Date());
+ final BSONObject versionObj = new BasicBSONObject();
+ versionObj.put("$lte", currentVersion);
+ query.put("_version", versionObj);
+ }
+ final long startTime = System.currentTimeMillis();
+ int currentPage = 0;
+ long lastC;
+ long c = lastC = 0L;
+ int round = 0;
+ long roundIndexCount = 0L;
+ QueryResponse response = null;
+ int lastNumFound = -1;
+ while (true) {
+ response = this.findInSolr(query, null, null, currentPage * batchSize, batchSize, true, true);
+ if (currentPage == 0) {
+ if (lastNumFound == -1) {
+ lastNumFound = response.getNumFound();
+ }
+ else if (response.getNumFound() > 0 && lastNumFound == response.getNumFound()) {
+ final long sleep = 20000L;
+ if (DataStoreImpl.LOGGER.isInfoEnabled()) {
+ DataStoreImpl.LOGGER.info("{}[{}]: wait solr commit, sleep {}s.", new Object[] { operationName, round, sleep / 1000L });
}
- // 重建索引
- core.save(getMongoCollection().find(seletor).addOption(Bytes.QUERYOPTION_NOTIMEOUT));
- return true;
- }
-
- });
- } catch (SolrException e) {
- throw new DataStoreException(operationName + ": ", e);
- }
- }
-
- public void remove(final BSONObject query) throws DataStoreException {
-
- String operationName = this.storeConfig.getName() + "_remove_" + System.currentTimeMillis();
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info(operationName + ": query=" + query);
- }
- int count = this.count(query, true);
- if (count > MAX_REMOVE_ROWS && !query.containsField("$remove")) {
- throw new DataStoreException("can not remove more than 1000 rows at single time, actual rows: " + count);
- }
- try {
- List removeIdList = getObjectIdsInQuery(query);
- if(removeIdList != null) {
- doExecuteDelete(removeIdList);
- }else {
- iterateSolrQuery(operationName, query, 1000, new Closure() {
- public boolean execute(List objIdList) throws SolrException, DataStoreException {
- return doExecuteDelete(objIdList);
- }
- });
- }
- } catch (SolrException e) {
- throw new DataStoreException(operationName + ": ", e);
- }
- }
-
- private boolean doExecuteDelete(List objIdList) throws SolrException, DataStoreException {
- DBObject seletor = new BasicDBObject();
- if (objIdList.size() == 1) {
- seletor.put("_id", objIdList.get(0));
- } else {
- seletor.put("_id", new BasicDBObject("$in", objIdList));
- }
- getMongoCollection().remove(seletor, WriteConcern.NORMAL);
-
- // 删除大字段的关联文件
- if (storeConfig.isTextFieldToGridFS()) {
- fileStore.remove(objIdList);
- }
-
- core.deleteByQuery(seletor);
- return true;
- }
-
- public void save(List extends BSONObject> docList) throws DataStoreException {
- if (null == docList || docList.isEmpty()) {
- return;
- }
- if (storeConfig.isTextFieldToGridFS()) {
- List insertList = new ArrayList(docList.size());
- List fileList = new ArrayList(docList.size());
- boolean hasBigField = false;
- for (BSONObject bson : docList) {
- DBObject fileObj = new BasicDBObject();
- for (String field : bson.keySet()) {
- if (!storeConfig.containsField(field)) {
- throw new DataStoreException("unknown field: " + field + " !");
- }
- if (!storeConfig.isFieldValueLegal(field, bson.get(field))) {
- throw new DataStoreException("invalid value: " + bson.get(field) + " of field '" + field + "'!");
- }
-
- // 若是大字段,将内容保存至fileObj后存入fileList
- if (storeConfig.isBigField(field)) {
- hasBigField = true;
-
- fileObj.put(field, bson.get(field));
-
- // 主表中不保存该字段内容
- bson.put(field, null);
- }
- }
- fileList.add(fileObj);
-
- DBObject dbObj = new BasicDBObject();
- dbObj.putAll(bson);
- insertList.add(dbObj);
- this.getMongoCollection().save(dbObj, WriteConcern.NORMAL);
- }
-
- // 保存大字段
- if (hasBigField)
- fileStore.save(fileList, insertList);
- } else {
- for (BSONObject bson : docList) {
- for (String field : bson.keySet()) {
- if (!storeConfig.containsField(field)) {
- throw new DataStoreException("unknown field: " + field + " !");
- }
- if (!storeConfig.isFieldValueLegal(field, bson.get(field))) {
- throw new DataStoreException("invalid value: " + bson.get(field) + " of field '" + field + "'!");
- }
- }
- DBObject dbObj = new BasicDBObject();
- dbObj.putAll(bson);
- this.getMongoCollection().save(dbObj, WriteConcern.NORMAL);
- }
- }
-
- try {
- core.save(docList.iterator());
- } catch (SolrException e) {
- throw new DataStoreException("error execute save{docList=" + docList + "}: ", e);
- }
- }
-
- public void ensureIndex(Collection indexes) throws DataStoreException {
- if (null == indexes || indexes.isEmpty()) {
- return;
- }
- try {
- Iterator extends BSONObject> i = this.getMongoCollection().find().addOption(Bytes.QUERYOPTION_NOTIMEOUT);
- core.ensureIndex(indexes, i);
- } catch (SolrException e) {
- throw new DataStoreException("error execute ensureIndex{indexes=" + indexes + "}:", e);
- }
- }
-
+ try {
+ Thread.sleep(sleep);
+ }
+ catch (InterruptedException e) {
+ throw new DataStoreException(operationName + ": ", e);
+ }
+ response = this.findInSolr(query, null, null, 0, 1, true, true);
+ if (response.getNumFound() > 0 && lastNumFound == response.getNumFound()) {
+ break;
+ }
+ lastNumFound = response.getNumFound();
+ continue;
+ }
+ else {
+ lastNumFound = response.getNumFound();
+ }
+ }
+ final int returnedSize = response.getNumReturned();
+ final List idList = response.getIdList();
+ if (null != idList && idList.size() > 0 && !closure.execute(idList)) {
+ break;
+ }
+ c += returnedSize;
+ roundIndexCount += returnedSize;
+ if (c - lastC >= batchSize && DataStoreImpl.LOGGER.isInfoEnabled()) {
+ DataStoreImpl.LOGGER.info("{}[{}]: {}/{}, {}/s", new Object[] { operationName, round, c, response.getNumFound(), (int)(c / ((System.currentTimeMillis() - startTime) / 1000.0)) });
+ lastC = c;
+ }
+ if (currentPage == 0 && returnedSize >= lastNumFound) {
+ break;
+ }
+ if (returnedSize == 0 || roundIndexCount >= lastNumFound) {
+ currentPage = 0;
+ ++round;
+ roundIndexCount = 0L;
+ }
+ else {
+ ++currentPage;
+ }
+ }
+ if (DataStoreImpl.LOGGER.isInfoEnabled()) {
+ DataStoreImpl.LOGGER.info("{}[{}] finished, total time: {}s, count: {}/{}", new Object[] { operationName, round, (System.currentTimeMillis() - startTime) / 1000L, roundIndexCount, c });
+ }
+ }
+
+ @Override
+ public QueryResponse find(final BSONObject query, final BSONObject fields, final BSONObject sort, final BSONObject facet, final int skip, final int limit, final boolean bulk, final boolean fromMaster) throws DataStoreException {
+ this.checkFields(fields);
+ try {
+ final QueryResponse response = this.findInSolr(query, sort, facet, skip, limit, fromMaster, bulk);
+ final List docList = this.find(response, fields);
+ final int numReturned = docList.size() - 1;
+ if (numReturned > response.getNumFound()) {
+ response.setNumFound(numReturned);
+ docList.get(0).put("$numFound", (Object)numReturned);
+ }
+ if (numReturned != response.getNumReturned()) {
+ response.setNumReturned(numReturned);
+ }
+ response.setDocList(docList);
+ return response;
+ }
+ catch (SolrException e) {
+ throw new DataStoreException("error execute find{query=" + query + ", fields=" + fields + ", sort=" + sort + ", facet=" + facet + ", skip=" + skip + ", limit=" + limit + "}: ", e);
+ }
+ }
+
+ private QueryResponse findInSolr(final BSONObject query, final BSONObject sort, final BSONObject facet, final int skip, final int limit, final boolean fromMaster, final boolean bulk) throws SolrException, DataStoreException {
+ QueryResponse response = null;
+ List idList = null;
+ if (null == facet || facet.keySet().size() == 0) {
+ idList = this.getObjectIdsInQuery(query);
+ }
+ if (null != idList && idList.size() > 0) {
+ response = new QueryResponse();
+ response.setIdList(idList);
+ response.setNumReturned(idList.size());
+ }
+ else {
+ response = this.core.find(query, sort, facet, skip, limit, fromMaster, bulk);
+ }
+ return response;
+ }
+
+ private List getObjectIdsInQuery(final BSONObject query) throws DataStoreException {
+ List ret = null;
+ final Object value = query.get("_id");
+ if (null == value) {
+ ret = null;
+ }
+ else if (value instanceof BSONObject) {
+ final List> ids = (List>)((BSONObject)value).get("$in");
+ if (null != ids && ids.size() > 0) {
+ if (ids.get(0) instanceof ObjectId) {
+ ret = (List)ids;
+ }
+ else {
+ ret = new ArrayList(ids.size());
+ for (final Object id : ids) {
+ ret.add(new ObjectId(String.valueOf(id)));
+ }
+ }
+ }
+ }
+ else if (value instanceof ObjectId) {
+ ret = Arrays.asList((ObjectId)value);
+ }
+ else if (value instanceof String) {
+ ret = Arrays.asList(new ObjectId((String)value));
+ }
+ if (ret != null && this.containsNoneIdQuery(query)) {
+ throw new DataStoreException("id query with other query:" + query);
+ }
+ return ret;
+ }
+
+ private boolean containsNoneIdQuery(final BSONObject query) {
+ if (null == query) {
+ return false;
+ }
+ for (final String key : query.keySet()) {
+ if (DataStoreImpl.SPECIAL_KEYS.contains(key)) {
+ continue;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public int count(final BSONObject query, final boolean fromMaster) throws DataStoreException {
+ try {
+ final QueryResponse response = this.core.find(query, null, null, 0, 0, fromMaster, false);
+ final List docList = this.find(response, null);
+ response.setDocList(docList);
+ return response.getNumFound();
+ }
+ catch (SolrException e) {
+ throw new DataStoreException("error execute count{query=" + query + "}: ", e);
+ }
+ }
+
+ private List find(final QueryResponse response, final BSONObject fields) throws DataStoreException {
+ final List idList = response.getIdList();
+ List newDocList = new ArrayList((null == idList) ? 1 : (idList.size() + 1));
+ final DBObject meta = new BasicDBObject("$numFound", response.getNumFound());
+ final List facetFieldList = response.getFacetFieldList();
+ if (null != facetFieldList && facetFieldList.size() > 0) {
+ final List facetList = new ArrayList(facetFieldList.size());
+ for (final FacetField facetField : facetFieldList) {
+ final DBObject facet = new BasicDBObject();
+ facet.put("name", facetField.getName());
+ facet.put("end", facetField.getEnd());
+ facet.put("gap", facetField.getGap());
+ facet.put("valueCount", facetField.getValueCount());
+ final List values = facetField.getValues();
+ if (null != values && values.size() > 0) {
+ final List vs = new ArrayList(values.size());
+ for (final FacetField.Count c : values) {
+ final DBObject v = new BasicDBObject();
+ v.put("name", c.getName());
+ v.put("count", c.getCount());
+ vs.add(v);
+ }
+ facet.put("values", vs);
+ }
+ facetList.add(facet);
+ }
+ meta.put("$facet", facetList);
+ }
+ newDocList.add(meta);
+ if (null != idList && idList.size() > 0) {
+ final DBObject query = new BasicDBObject();
+ if (idList.size() == 1) {
+ query.put("_id", idList.get(0));
+ }
+ else {
+ query.put("_id", new BasicDBObject("$in", idList));
+ }
+ DBObject fieldsQuery = null;
+ if (null != fields) {
+ fieldsQuery = new BasicDBObject();
+ fieldsQuery.putAll(fields);
+ }
+ final long start = System.currentTimeMillis();
+ final List docList = this.getMongoCollection().find(query, fieldsQuery).toArray();
+ Collections.sort(docList, DataStoreImpl.OBJECT_ID_COMPARATOR);
+ final long time = System.currentTimeMillis() - start;
+ if (time > 100L && DataStoreImpl.PERFORMANCE_LOGGER.isWarnEnabled()) {
+ DataStoreImpl.PERFORMANCE_LOGGER.warn(this.storeConfig.getName() + ": query mongodb, id size: " + idList.size() + ", time: " + time);
+ }
+ if (null != docList && !docList.isEmpty()) {
+ final Iterator i = idList.iterator();
+ while (i.hasNext()) {
+ final int index = binarySearch(docList, i.next());
+ if (index >= 0) {
+ newDocList.add(docList.get(index));
+ }
+ }
+ }
+ if (this.storeConfig.isTextFieldToGridFS()) {
+ newDocList = this.fileStore.find(newDocList, fields);
+ }
+ }
+ return newDocList;
+ }
+
+ private static int binarySearch(final List list, final ObjectId id) {
+ int low = 0;
+ int high = list.size() - 1;
+ while (low <= high) {
+ final int mid = low + high >>> 1;
+ final ObjectId midVal = (ObjectId)list.get(mid).get("_id");
+ final int cmp = midVal.compareTo(id);
+ if (cmp < 0) {
+ low = mid + 1;
+ }
+ else {
+ if (cmp <= 0) {
+ return mid;
+ }
+ high = mid - 1;
+ }
+ }
+ return -(low + 1);
+ }
+
+ private boolean isOpSupported(final String op) {
+ return "$inc".equals(op) || "$set".equals(op) || "$unset".equals(op);
+ }
+
+ private void checkUpdateFields(final BSONObject data) throws DataStoreException {
+ for (final String op : data.keySet()) {
+ if (!this.isOpSupported(op)) {
+ throw new DataStoreException("modifier operation " + op + " is not supported!");
+ }
+ final BSONObject update = (BSONObject)data.get(op);
+ if (null == update) {
+ continue;
+ }
+ for (final String field : update.keySet()) {
+ if (!this.storeConfig.containsField(field)) {
+ throw new DataStoreException("unknown field: " + field + " !");
+ }
+ if (!this.storeConfig.isFieldValueLegal(field, update.get(field))) {
+ throw new DataStoreException("invalid value " + update.get(field) + " of field '" + field + "'!");
+ }
+ }
+ }
+ }
+
+ private void checkFields(final BSONObject fields) throws DataStoreException {
+ if (null != fields) {
+ for (final String field : fields.keySet()) {
+ if (!this.storeConfig.containsField(field)) {
+ throw new DataStoreException("unknown field: " + field + " !");
+ }
+ }
+ }
+ }
+
+ @Override
+ public int update(final BSONObject query, final BSONObject data) throws DataStoreException {
+ final AtomicInteger count = new AtomicInteger(0);
+ this.checkUpdateFields(data);
+ final String operationName = this.storeConfig.getName() + "_update_" + System.currentTimeMillis();
+ if (DataStoreImpl.LOGGER.isInfoEnabled()) {
+ DataStoreImpl.LOGGER.info(operationName + ": query=" + query);
+ }
+ try {
+ this.iterateSolrQuery(operationName, query, 1000, new Closure() {
+ @Override
+ public boolean execute(final List objIdList) throws SolrException, DataStoreException {
+ final DBObject update = (DBObject)new BasicDBObject();
+ update.putAll(data);
+ if (DataStoreImpl.this.storeConfig.isTextFieldToGridFS()) {
+ DataStoreImpl.this.fileStore.update(objIdList, (BSONObject)update);
+ final BSONObject realData = (BSONObject)update.get("$set");
+ if (null != realData) {
+ final Set removeField = new HashSet();
+ for (final String field : realData.keySet()) {
+ if (DataStoreImpl.this.storeConfig.isBigField(field)) {
+ removeField.add(field);
+ }
+ }
+ realData.keySet().removeAll(removeField);
+ }
+ }
+ final DBObject seletor = (DBObject)new BasicDBObject();
+ if (objIdList.size() == 1) {
+ seletor.put("_id", (Object)objIdList.get(0));
+ }
+ else {
+ seletor.put("_id", (Object)new BasicDBObject("$in", (Object)objIdList));
+ }
+ final WriteResult writeResult = DataStoreImpl.this.getMongoCollection().update(seletor, update, false, true, WriteConcern.ACKNOWLEDGED);
+ count.set(writeResult.getN());
+ DataStoreImpl.this.core.save((Iterator extends BSONObject>)DataStoreImpl.this.getMongoCollection().find(seletor).addOption(16));
+ return true;
+ }
+ });
+ return count.get();
+ }
+ catch (SolrException e) {
+ throw new DataStoreException(operationName + ": ", e);
+ }
+ }
+
+ @Override
+ public void remove(final BSONObject query) throws DataStoreException {
+ final String operationName = this.storeConfig.getName() + "_remove_" + System.currentTimeMillis();
+ if (DataStoreImpl.LOGGER.isInfoEnabled()) {
+ DataStoreImpl.LOGGER.info(operationName + ": query=" + query);
+ }
+ final int count = this.count(query, true);
+ if (count > 10 && !query.containsField("$remove")) {
+ throw new DataStoreException("can not remove more than 10 rows at single time, actual rows: " + count);
+ }
+ try {
+ final List removeIdList = this.getObjectIdsInQuery(query);
+ if (removeIdList != null) {
+ this.doExecuteDelete(removeIdList);
+ }
+ else {
+ this.iterateSolrQuery(operationName, query, 1000, new Closure() {
+ @Override
+ public boolean execute(final List objIdList) throws SolrException, DataStoreException {
+ return DataStoreImpl.this.doExecuteDelete(objIdList);
+ }
+ });
+ }
+ }
+ catch (SolrException e) {
+ throw new DataStoreException(operationName + ": ", e);
+ }
+ }
+
+ private boolean doExecuteDelete(final List objIdList) throws SolrException, DataStoreException {
+ final DBObject seletor = (DBObject)new BasicDBObject();
+ if (objIdList.size() == 1) {
+ seletor.put("_id", (Object)objIdList.get(0));
+ }
+ else {
+ seletor.put("_id", (Object)new BasicDBObject("$in", (Object)objIdList));
+ }
+ this.getMongoCollection().remove(seletor, WriteConcern.NORMAL);
+ if (this.storeConfig.isTextFieldToGridFS()) {
+ this.fileStore.remove(objIdList);
+ }
+ this.core.deleteByQuery((BSONObject)seletor);
+ return true;
+ }
+
+ @Override
+ public void save(final List extends BSONObject> docList) throws DataStoreException {
+ if (null == docList || docList.isEmpty()) {
+ return;
+ }
+ if (this.storeConfig.isTextFieldToGridFS()) {
+ final List insertList = new ArrayList(docList.size());
+ final List fileList = new ArrayList(docList.size());
+ boolean hasBigField = false;
+ for (final BSONObject bson : docList) {
+ final DBObject fileObj = (DBObject)new BasicDBObject();
+ for (final String field : bson.keySet()) {
+ if (!this.storeConfig.containsField(field)) {
+ throw new DataStoreException("unknown field: " + field + " !");
+ }
+ if (!this.storeConfig.isFieldValueLegal(field, bson.get(field))) {
+ throw new DataStoreException("invalid value: " + bson.get(field) + " of field '" + field + "'!");
+ }
+ if (!this.storeConfig.isBigField(field)) {
+ continue;
+ }
+ hasBigField = true;
+ fileObj.put(field, bson.get(field));
+ bson.put(field, (Object)null);
+ }
+ fileList.add(fileObj);
+ final DBObject dbObj = (DBObject)new BasicDBObject();
+ dbObj.putAll(bson);
+ insertList.add(dbObj);
+ this.getMongoCollection().save(dbObj, WriteConcern.NORMAL);
+ }
+ if (hasBigField) {
+ this.fileStore.save(fileList, insertList);
+ }
+ }
+ else {
+ for (final BSONObject bson2 : docList) {
+ for (final String field2 : bson2.keySet()) {
+ if (!this.storeConfig.containsField(field2)) {
+ throw new DataStoreException("unknown field: " + field2 + " !");
+ }
+ if (!this.storeConfig.isFieldValueLegal(field2, bson2.get(field2))) {
+ throw new DataStoreException("invalid value: " + bson2.get(field2) + " of field '" + field2 + "'!");
+ }
+ }
+ final DBObject dbObj2 = (DBObject)new BasicDBObject();
+ dbObj2.putAll(bson2);
+ this.getMongoCollection().save(dbObj2, WriteConcern.NORMAL);
+ }
+ }
+ try {
+ this.core.save(docList.iterator());
+ }
+ catch (SolrException e) {
+ throw new DataStoreException("error execute save{docList=" + docList + "}: ", e);
+ }
+ }
+
+ @Override
+ public void ensureIndex(final Collection indexes) throws DataStoreException {
+ if (null == indexes || indexes.isEmpty()) {
+ return;
+ }
+ try {
+ final Iterator extends BSONObject> i = (Iterator extends BSONObject>)this.getMongoCollection().find().addOption(16);
+ this.core.ensureIndex(indexes, i);
+ }
+ catch (SolrException e) {
+ throw new DataStoreException("error execute ensureIndex{indexes=" + indexes + "}:", e);
+ }
+ }
+
+ static {
+ LOGGER = LoggerFactory.getLogger(DataStoreImpl.class);
+ PERFORMANCE_LOGGER = LoggerFactory.getPerformanceLogger(DataStoreImpl.class);
+ OBJECT_ID_COMPARATOR = new Comparator() {
+ @Override
+ public int compare(final DBObject o1, final DBObject o2) {
+ final ObjectId id1 = (ObjectId)o1.get("_id");
+ final ObjectId id2 = (ObjectId)o2.get("_id");
+ return id1.compareTo(id2);
+ }
+ };
+ SPECIAL_KEYS = new HashSet(Arrays.asList("_id", "$bulk", "_version"));
+ }
+
+ private interface Closure
+ {
+ boolean execute(final List p0) throws SolrException, DataStoreException;
+ }
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreServer.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreServer.java
index ccab8e57f..959b16cfc 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreServer.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreServer.java
@@ -1,193 +1,178 @@
package com.fangjia.datastore;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import org.apache.mina.core.buffer.IoBuffer;
-import org.apache.mina.core.service.IoHandlerAdapter;
+import org.slf4j.LoggerFactory;
import org.apache.mina.core.service.IoService;
-import org.apache.mina.core.service.IoServiceListener;
-import org.apache.mina.core.service.SimpleIoProcessorPool;
-import org.apache.mina.core.session.IdleStatus;
-import org.apache.mina.core.session.IoSession;
-import org.apache.mina.filter.codec.ProtocolCodecException;
-import org.apache.mina.filter.codec.ProtocolCodecFilter;
-import org.apache.mina.filter.logging.LogLevel;
-import org.apache.mina.filter.logging.LoggingFilter;
-import org.apache.mina.transport.socket.SocketSessionConfig;
-import org.apache.mina.transport.socket.nio.NioProcessor;
-import org.apache.mina.transport.socket.nio.NioSession;
-import org.apache.mina.transport.socket.nio.NioSocketAcceptor;
+import java.util.Arrays;
import org.bson.BSONObject;
import org.bson.BasicBSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fangjia.datastore.config.DataStoreConfig;
-import com.fangjia.datastore.handler.DeleteMessageHandler;
-import com.fangjia.datastore.handler.GetMoreMessageHandler;
-import com.fangjia.datastore.handler.InsertMessageHandler;
-import com.fangjia.datastore.handler.KillCursorsMessageHandler;
-import com.fangjia.datastore.handler.MessageHandler;
-import com.fangjia.datastore.handler.QueryMessageHandler;
+import com.fangjia.datastore.mongo.protocol.ReplyMessage;
+import org.apache.mina.filter.codec.ProtocolCodecException;
+import java.io.IOException;
+import org.apache.mina.core.session.IoSession;
+import org.apache.mina.transport.socket.SocketSessionConfig;
+import java.net.SocketAddress;
+import java.net.InetSocketAddress;
+import org.apache.mina.core.session.IdleStatus;
+import org.apache.mina.core.service.IoHandler;
import com.fangjia.datastore.handler.UpdateMessageHandler;
-import com.fangjia.datastore.mongo.codec.BSONIoBufferAllocator;
+import com.fangjia.datastore.mongo.protocol.UpdateMessage;
+import com.fangjia.datastore.handler.DeleteMessageHandler;
+import com.fangjia.datastore.mongo.protocol.DeleteMessage;
+import com.fangjia.datastore.handler.KillCursorsMessageHandler;
+import com.fangjia.datastore.mongo.protocol.KillCursorsMessage;
+import com.fangjia.datastore.handler.GetMoreMessageHandler;
+import com.fangjia.datastore.mongo.protocol.GetMoreMessage;
+import com.fangjia.datastore.handler.QueryMessageHandler;
+import com.fangjia.datastore.mongo.protocol.QueryMessage;
+import com.fangjia.datastore.handler.InsertMessageHandler;
+import com.fangjia.datastore.mongo.protocol.InsertMessage;
+import java.util.concurrent.ExecutorService;
+import org.apache.mina.filter.logging.LogLevel;
+import org.apache.mina.filter.logging.LoggingFilter;
+import org.apache.mina.core.filterchain.IoFilter;
+import org.apache.mina.filter.codec.ProtocolDecoder;
+import org.apache.mina.filter.codec.ProtocolEncoder;
+import org.apache.mina.filter.codec.ProtocolCodecFilter;
import com.fangjia.datastore.mongo.codec.MongoProtocolDecoder;
import com.fangjia.datastore.mongo.codec.MongoProtocolEncoder;
-import com.fangjia.datastore.mongo.protocol.DeleteMessage;
-import com.fangjia.datastore.mongo.protocol.GetMoreMessage;
-import com.fangjia.datastore.mongo.protocol.InsertMessage;
-import com.fangjia.datastore.mongo.protocol.KillCursorsMessage;
+import org.apache.mina.core.service.IoProcessor;
+import java.util.concurrent.Executor;
+import org.apache.mina.core.service.SimpleIoProcessorPool;
+import org.apache.mina.transport.socket.nio.NioProcessor;
+import java.util.concurrent.Executors;
+import org.apache.mina.core.buffer.IoBufferAllocator;
+import org.apache.mina.core.buffer.IoBuffer;
+import com.fangjia.datastore.mongo.codec.BSONIoBufferAllocator;
+import com.fangjia.datastore.config.DataStoreConfig;
+import java.util.HashMap;
+import com.fangjia.datastore.handler.MessageHandler;
+import java.util.Map;
import com.fangjia.datastore.mongo.protocol.MongoMessage;
-import com.fangjia.datastore.mongo.protocol.QueryMessage;
-import com.fangjia.datastore.mongo.protocol.ReplyMessage;
-import com.fangjia.datastore.mongo.protocol.UpdateMessage;
-
-public class DataStoreServer extends IoHandlerAdapter implements IoServiceListener {
-
- private final NioSocketAcceptor acceptor;
- private final DataStoreContext context;
- private static final Logger LOGGER = LoggerFactory.getLogger(DataStoreServer.class);
- private static ThreadLocal CURRENT_MESSAGE = new ThreadLocal();
- private final Map, MessageHandler extends MongoMessage>> handlerMap = new HashMap, MessageHandler extends MongoMessage>>();
-
- public DataStoreServer(String[] args) throws Exception {
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("starting datastore...");
- }
-
- DataStoreConfig config = new DataStoreConfig(System.getProperty("datastore.home", "d:\\goojiaconfig\\datastore"));
- config.setReadOnly(Boolean.getBoolean("datastore.readOnly"));
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info(config.toString());
- }
- context = new DataStoreContextImpl(config);
-
- IoBuffer.setAllocator(new BSONIoBufferAllocator());
-
- final ExecutorService executorService = Executors.newCachedThreadPool();
- acceptor = new NioSocketAcceptor(executorService, new SimpleIoProcessorPool(NioProcessor.class, config
- .getServerConfig().getMaxConnections()));
-
- acceptor.getFilterChain().addLast("codec",
- new ProtocolCodecFilter(new MongoProtocolEncoder(), new MongoProtocolDecoder()));
- acceptor.getFilterChain().addLast("performance",
- new PerformanceLogFilter(config.getServerConfig().getPerformanceLogFilterMaxTime()));
- if (!config.getServerConfig().isDisableLogging()) {
- LoggingFilter loggingFilter = new LoggingFilter(DataStoreServer.class);
- loggingFilter.setExceptionCaughtLogLevel(LogLevel.NONE);
- acceptor.getFilterChain().addLast("logger", loggingFilter);
- }
-
- Runtime.getRuntime().addShutdownHook(new Thread() {
-
- @Override
- public void run() {
- LOGGER.info("shutting down...");
- acceptor.unbind();
- executorService.shutdown();
- LOGGER.info("shut down");
- }
-
- });
-
- // register message handlers
- register(InsertMessage.class, new InsertMessageHandler(context));
- register(QueryMessage.class, new QueryMessageHandler(context));
- register(GetMoreMessage.class, new GetMoreMessageHandler(context));
- register(KillCursorsMessage.class, new KillCursorsMessageHandler(context));
- register(DeleteMessage.class, new DeleteMessageHandler(context));
- register(UpdateMessage.class, new UpdateMessageHandler(context));
-
- acceptor.setHandler(this);
- acceptor.addListener(this);
- acceptor.setReuseAddress(true);
- SocketSessionConfig sessionConfig = acceptor.getSessionConfig();
- sessionConfig.setTcpNoDelay(true);
- sessionConfig.setKeepAlive(true);
- sessionConfig.setReuseAddress(true);
- sessionConfig.setReadBufferSize(config.getServerConfig().getMinaReadBufferSize());
- sessionConfig.setIdleTime(IdleStatus.BOTH_IDLE, 300);
- acceptor.bind(new InetSocketAddress(config.getServerConfig().getPort()));
- }
-
- private void register(Class extends MongoMessage> messageClass, MessageHandler extends MongoMessage> handler) {
- this.handlerMap.put(messageClass, handler);
- }
-
- /**
- * @param args
- * @throws Exception
- * @throws
- */
- public static void main(String[] args) throws Exception {
- new DataStoreServer(args);
- }
-
- @Override
- public void exceptionCaught(IoSession session, Throwable cause) throws Exception {
- if (cause instanceof IOException && null != cause.getMessage() && cause.getMessage().contains("Connection reset by peer")) {
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("connection closed on session: {}", session);
- }
- } else if (cause instanceof ProtocolCodecException) {
- // ProtocolDecoderException or ProtocolEncoderException
- MongoMessage message = CURRENT_MESSAGE.get();
- if (null != message) {
- if (message instanceof QueryMessage) {
- QueryMessage queryMessage = (QueryMessage) message;
- ReplyMessage reply = new ReplyMessage();
- reply.setNumberReturned(1);
- reply.setResponseFlag(8);
- reply.getMessageHeader().setResponseTo(queryMessage.getMessageHeader().getRequestID());
- BSONObject msg = new BasicBSONObject("$err", cause.getMessage());
- reply.setDocuments(Arrays.asList(msg));
- session.write(reply);
- }
- }
- } else if (LOGGER.isErrorEnabled()) {
- LOGGER.error("exception caught on session:{" + session + "}", cause);
- }
- }
-
- @Override
- public void messageReceived(IoSession session, Object message) throws Exception {
- if (message instanceof MongoMessage) {
- MongoMessage mongoMessage = (MongoMessage) message;
- CURRENT_MESSAGE.set(mongoMessage);
- MessageHandler extends MongoMessage> handler = this.handlerMap.get(mongoMessage.getClass());
- if (null != handler) {
- handler.messageReceived(session, mongoMessage);
- } else {
- throw new RuntimeException("unknown message received:" + message);
- }
- } else {
- throw new RuntimeException("unknown message received:" + message);
- }
- }
-
- public void serviceActivated(IoService service) throws Exception {
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("datastore started");
- LOGGER.info("listening at " + this.context.getDataStoreConfig().getServerConfig().getPort());
- }
- }
-
- public void serviceIdle(IoService service, IdleStatus idleStatus) throws Exception {
- // empty
- }
-
- public void serviceDeactivated(IoService service) throws Exception {
- // empty
- }
-
- public void sessionDestroyed(IoSession session) throws Exception {
- // empty
- }
+import org.slf4j.Logger;
+import org.apache.mina.transport.socket.nio.NioSocketAcceptor;
+import org.apache.mina.core.service.IoServiceListener;
+import org.apache.mina.core.service.IoHandlerAdapter;
+public class DataStoreServer extends IoHandlerAdapter implements IoServiceListener
+{
+ private final NioSocketAcceptor acceptor;
+ private final DataStoreContext context;
+ private static final Logger LOGGER;
+ private static ThreadLocal CURRENT_MESSAGE;
+ private final Map, MessageHandler extends MongoMessage>> handlerMap;
+
+ public DataStoreServer(final String[] args) throws Exception {
+ this.handlerMap = new HashMap, MessageHandler extends MongoMessage>>();
+ if (DataStoreServer.LOGGER.isInfoEnabled()) {
+ DataStoreServer.LOGGER.info("starting datastore...");
+ }
+ final DataStoreConfig config = new DataStoreConfig(System.getProperty("datastore.home", "D:\\goojiaconfig\\seaweedsolr6"));
+ config.setReadOnly(Boolean.getBoolean("datastore.readOnly"));
+ if (DataStoreServer.LOGGER.isInfoEnabled()) {
+ DataStoreServer.LOGGER.info(config.toString());
+ }
+ this.context = new DataStoreContextImpl(config);
+ IoBuffer.setAllocator(new BSONIoBufferAllocator());
+ final ExecutorService executorService = Executors.newCachedThreadPool();
+ this.acceptor = new NioSocketAcceptor(executorService, new SimpleIoProcessorPool(NioProcessor.class, config.getServerConfig().getMaxConnections()));
+ this.acceptor.getFilterChain().addLast("codec", new ProtocolCodecFilter(new MongoProtocolEncoder(), new MongoProtocolDecoder()));
+ this.acceptor.getFilterChain().addLast("performance", new PerformanceLogFilter(config.getServerConfig().getPerformanceLogFilterMaxTime()));
+ if (!config.getServerConfig().isDisableLogging()) {
+ final LoggingFilter loggingFilter = new LoggingFilter(DataStoreServer.class);
+ loggingFilter.setExceptionCaughtLogLevel(LogLevel.NONE);
+ this.acceptor.getFilterChain().addLast("logger", loggingFilter);
+ }
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ DataStoreServer.LOGGER.info("shutting down...");
+ DataStoreServer.this.acceptor.unbind();
+ executorService.shutdown();
+ DataStoreServer.LOGGER.info("shut down");
+ }
+ });
+ this.register(InsertMessage.class, new InsertMessageHandler(this.context));
+ this.register(QueryMessage.class, new QueryMessageHandler(this.context));
+ this.register(GetMoreMessage.class, new GetMoreMessageHandler(this.context));
+ this.register(KillCursorsMessage.class, new KillCursorsMessageHandler(this.context));
+ this.register(DeleteMessage.class, new DeleteMessageHandler(this.context));
+ this.register(UpdateMessage.class, new UpdateMessageHandler(this.context));
+ this.acceptor.setHandler(this);
+ this.acceptor.addListener(this);
+ this.acceptor.setReuseAddress(true);
+ final SocketSessionConfig sessionConfig = this.acceptor.getSessionConfig();
+ sessionConfig.setTcpNoDelay(true);
+ sessionConfig.setKeepAlive(true);
+ sessionConfig.setReuseAddress(true);
+ sessionConfig.setReadBufferSize(config.getServerConfig().getMinaReadBufferSize());
+ sessionConfig.setIdleTime(IdleStatus.BOTH_IDLE, 300);
+ this.acceptor.bind(new InetSocketAddress(config.getServerConfig().getPort()));
+ }
+
+ private void register(final Class extends MongoMessage> messageClass, final MessageHandler extends MongoMessage> handler) {
+ this.handlerMap.put(messageClass, handler);
+ }
+
+ public static void main(final String[] args) throws Exception {
+ new DataStoreServer(args);
+ }
+
+ public void exceptionCaught(final IoSession session, final Throwable cause) throws Exception {
+ if (cause instanceof IOException && null != cause.getMessage() && cause.getMessage().contains("Connection reset by peer")) {
+ if (DataStoreServer.LOGGER.isInfoEnabled()) {
+ DataStoreServer.LOGGER.info("connection closed on session: {}", (Object)session);
+ }
+ }
+ else if (cause instanceof ProtocolCodecException) {
+ final MongoMessage message = DataStoreServer.CURRENT_MESSAGE.get();
+ if (null != message && message instanceof QueryMessage) {
+ final QueryMessage queryMessage = (QueryMessage)message;
+ final ReplyMessage reply = new ReplyMessage();
+ reply.setNumberReturned(1);
+ reply.setResponseFlag(8);
+ reply.getMessageHeader().setResponseTo(queryMessage.getMessageHeader().getRequestID());
+ final BSONObject msg = (BSONObject)new BasicBSONObject("$err", cause.getMessage());
+ reply.setDocuments(Arrays.asList(msg));
+ session.write((Object)reply);
+ }
+ }
+ else if (DataStoreServer.LOGGER.isErrorEnabled()) {
+ DataStoreServer.LOGGER.error("exception caught on session:{" + session + "}", cause);
+ }
+ }
+
+ public void messageReceived(final IoSession session, final Object message) throws Exception {
+ if (!(message instanceof MongoMessage)) {
+ throw new RuntimeException("unknown message received:" + message);
+ }
+ final MongoMessage mongoMessage = (MongoMessage)message;
+ DataStoreServer.CURRENT_MESSAGE.set(mongoMessage);
+ final MessageHandler extends MongoMessage> handler = this.handlerMap.get(mongoMessage.getClass());
+ if (null != handler) {
+ handler.messageReceived(session, mongoMessage);
+ return;
+ }
+ throw new RuntimeException("unknown message received:" + message);
+ }
+
+ public void serviceActivated(final IoService service) throws Exception {
+ if (DataStoreServer.LOGGER.isInfoEnabled()) {
+ DataStoreServer.LOGGER.info("datastore started");
+ DataStoreServer.LOGGER.info("listening at " + this.context.getDataStoreConfig().getServerConfig().getPort());
+ }
+ }
+
+ public void serviceIdle(final IoService service, final IdleStatus idleStatus) throws Exception {
+ }
+
+ public void serviceDeactivated(final IoService service) throws Exception {
+ }
+
+ public void sessionDestroyed(final IoSession session) throws Exception {
+ }
+
+ static {
+ LOGGER = LoggerFactory.getLogger((Class)DataStoreServer.class);
+ DataStoreServer.CURRENT_MESSAGE = new ThreadLocal();
+ }
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStore.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStore.java
index 0ec78a5c7..daf576ffa 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStore.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStore.java
@@ -1,64 +1,22 @@
package com.fangjia.datastore;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.util.List;
-import java.util.Set;
-
import org.bson.BSONObject;
+import java.util.Set;
import org.bson.types.ObjectId;
-
import com.mongodb.DBObject;
+import java.util.List;
-public interface FileStore {
- public final static String DEFAULT_CHARSET = "UTF-8";
-
- /**
- *
- * @param fileList
- * 仅包含大字段的文档列表
- * @param dbObjList
- * 已经保存到数据库的主对象列表
- * @throws IllegalArgumentException
- * 大字段数据不支持默认编码时抛出
- */
- void save(List fileList, List dbObjList) throws DataStoreException;
-
- /**
- * 删除指定的大字段内容
- *
- * @param objIdList
- * 删除的主对象的id列表
- * @param removeFields
- * 需要删除的大字段名集合
- */
- public void remove(List objIdList, Set removeFields);
-
- /**
- * 删除所有大字段内容
- *
- * @param objIdList
- */
- public void remove(List objIdList);
-
- /**
- * @param objIdList
- * 待更新的主对象id列表
- * @param data
- * 更新文档
- * @throws UnsupportedEncodingException
- * 大字段数据不支持默认编码时抛出
- */
- public void update(List objIdList, BSONObject data) throws DataStoreException;
-
- /**
- * @param docList
- * 查询的结果文档
- * @param fields
- * 查询字段对象
- * @return
- * @throws IOException
- * 读取大字段数据异常时、大字段数据不支持默认编码时抛出
- */
- public List find(List docList, BSONObject fields) throws DataStoreException;
+public interface FileStore
+{
+ public static final String DEFAULT_CHARSET = "UTF-8";
+
+ void save(final List p0, final List p1) throws DataStoreException;
+
+ void remove(final List p0, final Set p1);
+
+ void remove(final List p0);
+
+ void update(final List p0, final BSONObject p1) throws DataStoreException;
+
+ List find(final List p0, final BSONObject p1) throws DataStoreException;
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStoreImpl.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStoreImpl.java
index 4285d99b7..9cef6152f 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStoreImpl.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStoreImpl.java
@@ -1,249 +1,229 @@
package com.fangjia.datastore;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.zip.GZIPOutputStream;
-
import org.apache.commons.io.IOUtils;
+import com.mongodb.gridfs.GridFSDBFile;
+import java.util.HashSet;
+import com.mongodb.BasicDBObject;
+import java.util.ArrayList;
+import java.util.Set;
+import java.io.OutputStream;
+import java.util.zip.GZIPOutputStream;
+import java.io.ByteArrayOutputStream;
+import com.mongodb.gridfs.GridFSFile;
+import java.util.Date;
+import java.io.IOException;
import org.apache.commons.lang.StringUtils;
+import com.fangjia.datastore.config.FieldTypeEnum;
import org.bson.BSONObject;
import org.bson.types.ObjectId;
-
-import com.fangjia.datastore.config.Field;
-import com.fangjia.datastore.config.FieldTypeEnum;
-import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
+import java.util.List;
+import java.util.Iterator;
+import java.util.HashMap;
+import java.util.Collection;
+import com.fangjia.datastore.config.Field;
+import java.util.Map;
import com.mongodb.gridfs.GridFS;
-import com.mongodb.gridfs.GridFSDBFile;
-import com.mongodb.gridfs.GridFSFile;
-public class FileStoreImpl implements FileStore {
-
- private GridFS gridFS;
- private Map bigFieldSet;
-
- public FileStoreImpl(GridFS gridFS, Collection fields) {
- this.gridFS = gridFS;
-
- // 把大字段的字段名放入set
- bigFieldSet = new HashMap();
- for (Field field : fields) {
- if (field.isBigField()) {
- bigFieldSet.put(field.getName(), field);
- }
- }
- }
-
- public void save(List fileList, List dbObjList) {
- if (null == fileList || null == dbObjList || fileList.size() != dbObjList.size())
- throw new IllegalArgumentException("empty list or unequal length list is not allowed!");
-
- Iterator dbObjIt = dbObjList.iterator();
- for (Iterator fileIt = fileList.iterator(); fileIt.hasNext();) {
- // 取得已保存主对象的id
- String id = ((ObjectId) dbObjIt.next().get("_id")).toString();
-
- BSONObject file = fileIt.next();
- for (String fieldName : file.keySet()) {
- byte[] bytes = null;
-
- Field field = bigFieldSet.get(fieldName);
- if (null == field) {
- continue;
- }
- FieldTypeEnum fieldType = field.getType();
- if (FieldTypeEnum.TEXT == fieldType) {
- String value = (String) file.get(fieldName);
- if (!StringUtils.isBlank(value)) {
- try {
- bytes = gzip(value);
- } catch (IOException e) {
- throw new IllegalArgumentException("unsupported encoding value!");
- }
- }
- }
-
- if (null != bytes && bytes.length > 0) {
- GridFSFile mongofile = gridFS.createFile(bytes);
- mongofile.put("uploadDate", new Date());
- mongofile.put("filename", id);
- mongofile.put("contentType", fieldName);
- mongofile.save();
- }
- }
- }
- }
-
- private byte[] gzip(String string) throws IOException {
- ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
- GZIPOutputStream gzipOutputStream = new GZIPOutputStream(outputStream);
- gzipOutputStream.write(string.getBytes(DEFAULT_CHARSET));
- return outputStream.toByteArray();
- }
-
- public void remove(List objIdList) {
- remove(objIdList, null);
- }
-
- public void remove(List objIdList, Set removeFields) {
- // 删除表的所有大字段
- if (null == removeFields)
- removeFields = bigFieldSet.keySet();
-
- // 若到删除的字段中没有大字段,直接返回
- if (removeFields.size() == 0)
- return;
-
- List idList = new ArrayList(objIdList.size());
- for (ObjectId objId : objIdList) {
- idList.add(objId.toString());
- }
-
- // 删除大字段关联的GridFS
- DBObject fileSeletor = new BasicDBObject();
- fileSeletor.put("filename", new BasicDBObject("$in", idList));
- fileSeletor.put("contentType", new BasicDBObject("$in", removeFields));
- gridFS.remove(fileSeletor);
- }
-
- public void update(List objIdList, BSONObject data) throws DataStoreException {
- // 待删除的字段
- BSONObject unset = (BSONObject) data.get("$unset");
- if (null != unset) {
- // 待删除的大字段set
- Set removeSet = new HashSet();
- for (String field : unset.keySet()) {
- if (bigFieldSet.containsKey(field))
- removeSet.add(field);
- }
-
- if (removeSet.size() > 0)
- remove(objIdList, removeSet);
- }
-
- // 更新数据
- BSONObject set = (BSONObject) data.get("$set");
- if (null == set)
- return;
-
- Map fileMap = new HashMap();
- for (String field : set.keySet()) {
- if (bigFieldSet.containsKey(field)) {
- fileMap.put(field, set.get(field));
- }
- }
-
- // 更新数据中没有大字段
- if (fileMap.size() == 0)
- return;
-
- // 删除大字段关联的GridFS
- remove(objIdList, fileMap.keySet());
-
- // 将新的内容存入GridFS
- for (ObjectId objId : objIdList) {
- String id = objId.toString();
- for (String fieldName : fileMap.keySet()) {
- byte[] bytes = null;
-
- Field field = bigFieldSet.get(fieldName);
- if (null == field) {
- continue;
- }
- FieldTypeEnum fieldType = field.getType();
- if (FieldTypeEnum.TEXT == fieldType) {
- String value = (String) fileMap.get(fieldName);
- if (!StringUtils.isBlank(value)) {
- try {
- bytes = gzip(value);
- } catch (IOException e) {
- throw new IllegalArgumentException("unsupported encoding value!");
- }
- }
- }
-
- if (null != bytes && bytes.length > 0) {
- GridFSFile mongofile = gridFS.createFile(bytes);
- mongofile.put("uploadDate", new Date());
- mongofile.put("filename", id);
- mongofile.put("contentType", fieldName);
- mongofile.save();
- }
- }
- }
-
- }
-
- public List find(List docList, BSONObject fields) throws DataStoreException {
- // 判断查询的字段中有没有大字段
- Set queryBigField = new HashSet();
- if (null == fields) {
- queryBigField = bigFieldSet.keySet();
- } else {
- for (String field : fields.keySet()) {
- if (bigFieldSet.containsKey(field)) {
- queryBigField.add(field);
- }
- }
- }
-
- // 没有大字段,则返回原结果文档
- if (queryBigField.size() == 0)
- return docList;
-
- Map docMap = new HashMap(docList.size());
- List idList = new ArrayList(docList.size());
- for (DBObject dbObj : docList) {
- ObjectId objId = (ObjectId) dbObj.get("_id");
- if (null != objId) {
- String id = objId.toString();
- idList.add(id);
- docMap.put(id, dbObj);
- }
- }
-
- DBObject fileSeletor = new BasicDBObject();
- fileSeletor.put("filename", new BasicDBObject("$in", idList));
- fileSeletor.put("contentType", new BasicDBObject("$in", queryBigField));
- List fileList = gridFS.find(fileSeletor);
-
- for (GridFSDBFile file : fileList) {
- // 对应主记录id
- String id = file.getFilename();
- // 对应主记录大字段名
- String fieldName = file.getContentType();
-
- DBObject obj = docMap.get(id);
- if (obj.keySet().contains(fieldName)) {
-
- try {
- ByteArrayOutputStream data = new ByteArrayOutputStream();
- GZIPOutputStream out = new GZIPOutputStream(data);
- IOUtils.copy(file.getInputStream(), out);
-
- Field field = bigFieldSet.get(fieldName);
- if (null == field) {
- continue;
- }
- FieldTypeEnum fieldType = field.getType();
- if (FieldTypeEnum.TEXT == fieldType) {
- obj.put(fieldName, new String(data.toByteArray(), DEFAULT_CHARSET));
- }
- } catch (IOException e) {
- throw new DataStoreException("error extract text field[" + fieldName + "]:", e);
- }
- }
- }
-
- return docList;
- }
+public class FileStoreImpl implements FileStore
+{
+ private GridFS gridFS;
+ private Map bigFieldSet;
+
+ public FileStoreImpl(final GridFS gridFS, final Collection fields) {
+ this.gridFS = gridFS;
+ this.bigFieldSet = new HashMap();
+ for (final Field field : fields) {
+ if (field.isBigField()) {
+ this.bigFieldSet.put(field.getName(), field);
+ }
+ }
+ }
+
+ @Override
+ public void save(final List fileList, final List dbObjList) {
+ if (null == fileList || null == dbObjList || fileList.size() != dbObjList.size()) {
+ throw new IllegalArgumentException("empty list or unequal length list is not allowed!");
+ }
+ final Iterator dbObjIt = dbObjList.iterator();
+ final Iterator fileIt = fileList.iterator();
+ while (fileIt.hasNext()) {
+ final String id = ((ObjectId)dbObjIt.next().get("_id")).toString();
+ final BSONObject file = (BSONObject)fileIt.next();
+ for (final String fieldName : file.keySet()) {
+ byte[] bytes = null;
+ final Field field = this.bigFieldSet.get(fieldName);
+ if (null == field) {
+ continue;
+ }
+ final FieldTypeEnum fieldType = field.getType();
+ if (FieldTypeEnum.TEXT == fieldType) {
+ final String value = (String)file.get(fieldName);
+ if (!StringUtils.isBlank(value)) {
+ try {
+ bytes = this.gzip(value);
+ }
+ catch (IOException e) {
+ throw new IllegalArgumentException("unsupported encoding value!");
+ }
+ }
+ }
+ if (null == bytes || bytes.length <= 0) {
+ continue;
+ }
+ final GridFSFile mongofile = (GridFSFile)this.gridFS.createFile(bytes);
+ mongofile.put("uploadDate", (Object)new Date());
+ mongofile.put("filename", (Object)id);
+ mongofile.put("contentType", (Object)fieldName);
+ mongofile.save();
+ }
+ }
+ }
+
+ private byte[] gzip(final String string) throws IOException {
+ final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ final GZIPOutputStream gzipOutputStream = new GZIPOutputStream(outputStream);
+ gzipOutputStream.write(string.getBytes("UTF-8"));
+ return outputStream.toByteArray();
+ }
+
+ @Override
+ public void remove(final List objIdList) {
+ this.remove(objIdList, null);
+ }
+
+ @Override
+ public void remove(final List objIdList, Set removeFields) {
+ if (null == removeFields) {
+ removeFields = this.bigFieldSet.keySet();
+ }
+ if (removeFields.size() == 0) {
+ return;
+ }
+ final List idList = new ArrayList(objIdList.size());
+ for (final ObjectId objId : objIdList) {
+ idList.add(objId.toString());
+ }
+ final DBObject fileSeletor = (DBObject)new BasicDBObject();
+ fileSeletor.put("filename", (Object)new BasicDBObject("$in", (Object)idList));
+ fileSeletor.put("contentType", (Object)new BasicDBObject("$in", (Object)removeFields));
+ this.gridFS.remove(fileSeletor);
+ }
+
+ @Override
+ public void update(final List objIdList, final BSONObject data) throws DataStoreException {
+ final BSONObject unset = (BSONObject)data.get("$unset");
+ if (null != unset) {
+ final Set removeSet = new HashSet();
+ for (final String field : unset.keySet()) {
+ if (this.bigFieldSet.containsKey(field)) {
+ removeSet.add(field);
+ }
+ }
+ if (removeSet.size() > 0) {
+ this.remove(objIdList, removeSet);
+ }
+ }
+ final BSONObject set = (BSONObject)data.get("$set");
+ if (null == set) {
+ return;
+ }
+ final Map fileMap = new HashMap();
+ for (final String field2 : set.keySet()) {
+ if (this.bigFieldSet.containsKey(field2)) {
+ fileMap.put(field2, set.get(field2));
+ }
+ }
+ if (fileMap.size() == 0) {
+ return;
+ }
+ this.remove(objIdList, fileMap.keySet());
+ for (final ObjectId objId : objIdList) {
+ final String id = objId.toString();
+ for (final String fieldName : fileMap.keySet()) {
+ byte[] bytes = null;
+ final Field field3 = this.bigFieldSet.get(fieldName);
+ if (null == field3) {
+ continue;
+ }
+ final FieldTypeEnum fieldType = field3.getType();
+ if (FieldTypeEnum.TEXT == fieldType) {
+ final String value = (String)fileMap.get(fieldName);
+ if (!StringUtils.isBlank(value)) {
+ try {
+ bytes = this.gzip(value);
+ }
+ catch (IOException e) {
+ throw new IllegalArgumentException("unsupported encoding value!");
+ }
+ }
+ }
+ if (null == bytes || bytes.length <= 0) {
+ continue;
+ }
+ final GridFSFile mongofile = (GridFSFile)this.gridFS.createFile(bytes);
+ mongofile.put("uploadDate", (Object)new Date());
+ mongofile.put("filename", (Object)id);
+ mongofile.put("contentType", (Object)fieldName);
+ mongofile.save();
+ }
+ }
+ }
+
+ @Override
+ public List find(final List docList, final BSONObject fields) throws DataStoreException {
+ Set queryBigField = new HashSet();
+ if (null == fields) {
+ queryBigField = this.bigFieldSet.keySet();
+ }
+ else {
+ for (final String field : fields.keySet()) {
+ if (this.bigFieldSet.containsKey(field)) {
+ queryBigField.add(field);
+ }
+ }
+ }
+ if (queryBigField.size() == 0) {
+ return docList;
+ }
+ final Map docMap = new HashMap(docList.size());
+ final List idList = new ArrayList(docList.size());
+ for (final DBObject dbObj : docList) {
+ final ObjectId objId = (ObjectId)dbObj.get("_id");
+ if (null != objId) {
+ final String id = objId.toString();
+ idList.add(id);
+ docMap.put(id, dbObj);
+ }
+ }
+ final DBObject fileSeletor = (DBObject)new BasicDBObject();
+ fileSeletor.put("filename", (Object)new BasicDBObject("$in", (Object)idList));
+ fileSeletor.put("contentType", (Object)new BasicDBObject("$in", (Object)queryBigField));
+ final List fileList = (List)this.gridFS.find(fileSeletor);
+ for (final GridFSDBFile file : fileList) {
+ final String id2 = file.getFilename();
+ final String fieldName = file.getContentType();
+ final DBObject obj = docMap.get(id2);
+ if (obj.keySet().contains(fieldName)) {
+ try {
+ final ByteArrayOutputStream data = new ByteArrayOutputStream();
+ final GZIPOutputStream out = new GZIPOutputStream(data);
+ IOUtils.copy(file.getInputStream(), (OutputStream)out);
+ final Field field2 = this.bigFieldSet.get(fieldName);
+ if (null == field2) {
+ continue;
+ }
+ final FieldTypeEnum fieldType = field2.getType();
+ if (FieldTypeEnum.TEXT != fieldType) {
+ continue;
+ }
+ obj.put(fieldName, (Object)new String(data.toByteArray(), "UTF-8"));
+ }
+ catch (IOException e) {
+ throw new DataStoreException("error extract text field[" + fieldName + "]:", e);
+ }
+ }
+ }
+ return docList;
+ }
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Mongo.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Mongo.java
index 1955bee2c..bb424447a 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Mongo.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Mongo.java
@@ -1,46 +1,46 @@
package com.fangjia.datastore;
-import java.util.List;
-
-import com.fangjia.datastore.config.MongoConfig;
-import com.mongodb.DB;
-import com.mongodb.MongoException;
import com.mongodb.WriteConcern;
+import com.mongodb.MongoException;
+import java.util.List;
+import com.mongodb.DB;
+import com.fangjia.datastore.config.MongoConfig;
-public class Mongo {
-
- private final MongoConfig mongoConfig;
-
- public Mongo(MongoConfig mongoConfig) {
- this.mongoConfig = mongoConfig;
- }
-
- private com.mongodb.Mongo getMongo() {
- try {
- return mongoConfig.getMongo();
- } catch (ConfigurationException e) {
- throw new RuntimeException(e);
- }
- }
-
- public DB getDB(String dbname) {
- return getMongo().getDB(dbname);
- }
-
- public List getDatabaseNames() throws MongoException {
- return getMongo().getDatabaseNames();
- }
-
- public void setWriteConcern(WriteConcern concern) {
- getMongo().setWriteConcern(concern);
- }
-
- public WriteConcern getWriteConcern() {
- return getMongo().getWriteConcern();
- }
-
- public String toString() {
- return getMongo().toString();
- }
-
+public class Mongo
+{
+ private final MongoConfig mongoConfig;
+
+ public Mongo(final MongoConfig mongoConfig) {
+ this.mongoConfig = mongoConfig;
+ }
+
+ private com.mongodb.Mongo getMongo() {
+ try {
+ return this.mongoConfig.getMongo();
+ }
+ catch (ConfigurationException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public DB getDB(final String dbname) {
+ return this.getMongo().getDB(dbname);
+ }
+
+ public List getDatabaseNames() throws MongoException {
+ return (List)this.getMongo().getDatabaseNames();
+ }
+
+ public void setWriteConcern(final WriteConcern concern) {
+ this.getMongo().setWriteConcern(concern);
+ }
+
+ public WriteConcern getWriteConcern() {
+ return this.getMongo().getWriteConcern();
+ }
+
+ @Override
+ public String toString() {
+ return this.getMongo().toString();
+ }
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/PerformanceLogFilter.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/PerformanceLogFilter.java
index 26d62b527..7ae1f812f 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/PerformanceLogFilter.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/PerformanceLogFilter.java
@@ -1,114 +1,119 @@
package com.fangjia.datastore;
-import org.apache.mina.core.filterchain.IoFilterAdapter;
-import org.apache.mina.core.session.IdleStatus;
-import org.apache.mina.core.session.IoSession;
-import org.apache.mina.core.write.WriteRequest;
-import org.slf4j.Logger;
-
-import com.fangjia.datastore.mongo.protocol.MongoMessage;
import com.fangjia.datastore.utils.LoggerFactory;
+import org.apache.mina.core.session.IdleStatus;
+import org.apache.mina.core.write.WriteRequest;
+import org.apache.mina.core.filterchain.IoFilter;
+import com.fangjia.datastore.mongo.protocol.MongoMessage;
+import org.apache.mina.core.session.IoSession;
+import org.slf4j.Logger;
+import org.apache.mina.core.filterchain.IoFilterAdapter;
-public class PerformanceLogFilter extends IoFilterAdapter {
- private final long maxTime;
-
- private static final Logger LOGGER = LoggerFactory.getPerformanceLogger(PerformanceLogFilter.class);
-
- public PerformanceLogFilter(long maxTime) {
- this.maxTime = maxTime;
- }
-
- private void logPerformace(long time, String methodName, IoSession session, Object message) {
- Object[] params;
- if (message instanceof MongoMessage) {
- MongoMessage mongoMessage = (MongoMessage) message;
- params = new Object[] { time, methodName, session, mongoMessage.toPrettyString() };
- } else {
- params = new Object[] { time, methodName, session, message };
- }
- if (time < maxTime * 3) {
- LOGGER.warn("time={}, methodName={}, session={}, message={}", params);
- } else {
- LOGGER.error("time={}, methodName={}, session={}, message={}", params);
- }
- }
-
- @Override
- public void messageReceived(NextFilter nextFilter, IoSession session, Object message) throws Exception {
- long time = System.currentTimeMillis();
- try {
- nextFilter.messageReceived(session, message);
- } finally {
- time = System.currentTimeMillis() - time;
- if (time >= maxTime) {
- logPerformace(time, "messageReceived", session, message);
- }
- }
- }
-
- @Override
- public void messageSent(NextFilter nextFilter, IoSession session, WriteRequest writeRequest) throws Exception {
- long time = System.currentTimeMillis();
- try {
- nextFilter.messageSent(session, writeRequest);
- } finally {
- time = System.currentTimeMillis() - time;
- if (time >= maxTime) {
- logPerformace(time, "messageSent", session, writeRequest);
- }
- }
- }
-
- @Override
- public void sessionCreated(NextFilter nextFilter, IoSession session) throws Exception {
- long time = System.currentTimeMillis();
- try {
- nextFilter.sessionCreated(session);
- } finally {
- time = System.currentTimeMillis() - time;
- if (time >= maxTime) {
- logPerformace(time, "sessionCreated", session, null);
- }
- }
- }
-
- @Override
- public void sessionOpened(NextFilter nextFilter, IoSession session) throws Exception {
- long time = System.currentTimeMillis();
- try {
- nextFilter.sessionOpened(session);
- } finally {
- time = System.currentTimeMillis() - time;
- if (time >= maxTime) {
- logPerformace(time, "sessionOpened", session, null);
- }
- }
- }
-
- @Override
- public void sessionIdle(NextFilter nextFilter, IoSession session, IdleStatus status) throws Exception {
- long time = System.currentTimeMillis();
- try {
- nextFilter.sessionIdle(session, status);
- } finally {
- time = System.currentTimeMillis() - time;
- if (time >= maxTime) {
- logPerformace(time, "sessionIdle", session, status);
- }
- }
- }
-
- @Override
- public void sessionClosed(NextFilter nextFilter, IoSession session) throws Exception {
- long time = System.currentTimeMillis();
- try {
- nextFilter.sessionClosed(session);
- } finally {
- time = System.currentTimeMillis() - time;
- if (time >= maxTime) {
- logPerformace(time, "sessionClosed", session, null);
- }
- }
- }
-
+public class PerformanceLogFilter extends IoFilterAdapter
+{
+ private final long maxTime;
+ private static final Logger LOGGER;
+
+ public PerformanceLogFilter(final long maxTime) {
+ this.maxTime = maxTime;
+ }
+
+ private void logPerformace(final long time, final String methodName, final IoSession session, final Object message) {
+ Object[] params;
+ if (message instanceof MongoMessage) {
+ final MongoMessage mongoMessage = (MongoMessage)message;
+ params = new Object[] { time, methodName, session, mongoMessage.toPrettyString() };
+ }
+ else {
+ params = new Object[] { time, methodName, session, message };
+ }
+ if (time < this.maxTime * 3L) {
+ PerformanceLogFilter.LOGGER.warn("time={}, methodName={}, session={}, message={}", params);
+ }
+ else {
+ PerformanceLogFilter.LOGGER.error("time={}, methodName={}, session={}, message={}", params);
+ }
+ }
+
+ public void messageReceived(final NextFilter nextFilter, final IoSession session, final Object message) throws Exception {
+ long time = System.currentTimeMillis();
+ try {
+ nextFilter.messageReceived(session, message);
+ }
+ finally {
+ time = System.currentTimeMillis() - time;
+ if (time >= this.maxTime) {
+ this.logPerformace(time, "messageReceived", session, message);
+ }
+ }
+ }
+
+ public void messageSent(final NextFilter nextFilter, final IoSession session, final WriteRequest writeRequest) throws Exception {
+ long time = System.currentTimeMillis();
+ try {
+ nextFilter.messageSent(session, writeRequest);
+ }
+ finally {
+ time = System.currentTimeMillis() - time;
+ if (time >= this.maxTime) {
+ this.logPerformace(time, "messageSent", session, writeRequest);
+ }
+ }
+ }
+
+ public void sessionCreated(final NextFilter nextFilter, final IoSession session) throws Exception {
+ long time = System.currentTimeMillis();
+ try {
+ nextFilter.sessionCreated(session);
+ }
+ finally {
+ time = System.currentTimeMillis() - time;
+ if (time >= this.maxTime) {
+ this.logPerformace(time, "sessionCreated", session, null);
+ }
+ }
+ }
+
+ public void sessionOpened(final NextFilter nextFilter, final IoSession session) throws Exception {
+ long time = System.currentTimeMillis();
+ try {
+ nextFilter.sessionOpened(session);
+ }
+ finally {
+ time = System.currentTimeMillis() - time;
+ if (time >= this.maxTime) {
+ this.logPerformace(time, "sessionOpened", session, null);
+ }
+ }
+ }
+
+ public void sessionIdle(final NextFilter nextFilter, final IoSession session, final IdleStatus status) throws Exception {
+ long time = System.currentTimeMillis();
+ try {
+ nextFilter.sessionIdle(session, status);
+ }
+ finally {
+ time = System.currentTimeMillis() - time;
+ if (time >= this.maxTime) {
+ this.logPerformace(time, "sessionIdle", session, status);
+ }
+ }
+ }
+
+ public void sessionClosed(final NextFilter nextFilter, final IoSession session) throws Exception {
+ long time = System.currentTimeMillis();
+ try {
+ nextFilter.sessionClosed(session);
+ }
+ finally {
+ time = System.currentTimeMillis() - time;
+ if (time >= this.maxTime) {
+ this.logPerformace(time, "sessionClosed", session, null);
+ }
+ }
+ }
+
+ static {
+ LOGGER = LoggerFactory.getPerformanceLogger(PerformanceLogFilter.class);
+ }
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/QueryResponse.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/QueryResponse.java
index 224844bef..f5392909b 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/QueryResponse.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/QueryResponse.java
@@ -1,72 +1,80 @@
package com.fangjia.datastore;
import java.util.Collections;
+import org.apache.solr.client.solrj.response.FacetField;
+import com.mongodb.DBObject;
+import org.bson.types.ObjectId;
import java.util.List;
-import org.apache.solr.client.solrj.response.FacetField;
-import org.bson.types.ObjectId;
-
-import com.mongodb.DBObject;
-
-public class QueryResponse {
- private List idList = Collections.emptyList();
- private List docList = Collections.emptyList();
- private List facetFieldList = null;
- private int numFound;
- private int numReturned;
- private int numToSkip;
- public final static QueryResponse EMPTY_RESPONSE = new QueryResponse();
-
- public List getIdList() {
- return idList;
- }
-
- public void setIdList(List idList) {
- this.idList = idList;
- }
-
- public boolean hasMore() {
- return this.numFound > numReturned + numToSkip;
- }
-
- public int getNumFound() {
- return numFound;
- }
-
- public void setNumFound(int numFound) {
- this.numFound = numFound;
- }
-
- public int getNumReturned() {
- return numReturned;
- }
-
- public void setNumReturned(int numReturned) {
- this.numReturned = numReturned;
- }
-
- public int getNumToSkip() {
- return numToSkip;
- }
-
- public void setNumToSkip(int numToSkip) {
- this.numToSkip = numToSkip;
- }
-
- public List getFacetFieldList() {
- return facetFieldList;
- }
-
- public void setFacetFieldList(List facetFieldList) {
- this.facetFieldList = facetFieldList;
- }
-
- public List getDocList() {
- return docList;
- }
-
- public void setDocList(List docList) {
- this.docList = docList;
- }
-
+public class QueryResponse
+{
+ private List idList;
+ private List docList;
+ private List facetFieldList;
+ private int numFound;
+ private int numReturned;
+ private int numToSkip;
+ public static final QueryResponse EMPTY_RESPONSE;
+
+ public QueryResponse() {
+ this.idList = Collections.emptyList();
+ this.docList = Collections.emptyList();
+ this.facetFieldList = null;
+ }
+
+ public List getIdList() {
+ return this.idList;
+ }
+
+ public void setIdList(final List idList) {
+ this.idList = idList;
+ }
+
+ public boolean hasMore() {
+ return this.numFound > this.numReturned + this.numToSkip;
+ }
+
+ public int getNumFound() {
+ return this.numFound;
+ }
+
+ public void setNumFound(final int numFound) {
+ this.numFound = numFound;
+ }
+
+ public int getNumReturned() {
+ return this.numReturned;
+ }
+
+ public void setNumReturned(final int numReturned) {
+ this.numReturned = numReturned;
+ }
+
+ public int getNumToSkip() {
+ return this.numToSkip;
+ }
+
+ public void setNumToSkip(final int numToSkip) {
+ this.numToSkip = numToSkip;
+ }
+
+ public List getFacetFieldList() {
+ return this.facetFieldList;
+ }
+
+ public void setFacetFieldList(final List facetFieldList) {
+ this.facetFieldList = facetFieldList;
+ }
+
+ public List getDocList() {
+ return this.docList;
+ }
+
+ public void setDocList(final List docList) {
+ this.docList = docList;
+ }
+
+ static {
+ EMPTY_RESPONSE = new QueryResponse();
+ }
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecuteException.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecuteException.java
index f2e1546b4..590bb13d9 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecuteException.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecuteException.java
@@ -1,22 +1,21 @@
package com.fangjia.datastore;
-public class TriggerExecuteException extends Exception {
-
- private static final long serialVersionUID = -1165411196743370433L;
-
- public TriggerExecuteException() {
- }
-
- public TriggerExecuteException(String message) {
- super(message);
- }
-
- public TriggerExecuteException(Throwable cause) {
- super(cause);
- }
-
- public TriggerExecuteException(String message, Throwable cause) {
- super(message, cause);
- }
-
+public class TriggerExecuteException extends Exception
+{
+ private static final long serialVersionUID = -1165411196743370433L;
+
+ public TriggerExecuteException() {
+ }
+
+ public TriggerExecuteException(final String message) {
+ super(message);
+ }
+
+ public TriggerExecuteException(final Throwable cause) {
+ super(cause);
+ }
+
+ public TriggerExecuteException(final String message, final Throwable cause) {
+ super(message, cause);
+ }
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecutor.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecutor.java
index b998e420e..ce37f7653 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecutor.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecutor.java
@@ -1,165 +1,161 @@
package com.fangjia.datastore;
-import java.io.IOException;
+import org.slf4j.LoggerFactory;
+import org.apache.http.HttpRequest;
import java.io.InputStream;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.List;
-
-import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.params.AllClientPNames;
-import org.apache.http.conn.scheme.PlainSocketFactory;
-import org.apache.http.conn.scheme.Scheme;
-import org.apache.http.conn.scheme.SchemeRegistry;
+import com.mongodb.DBObject;
+import org.bson.BasicBSONDecoder;
+import org.apache.http.protocol.HttpContext;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpEntity;
import org.apache.http.entity.ByteArrayEntity;
+import org.sothis.core.util.GZIPUtils;
+import org.bson.BasicBSONEncoder;
+import com.mongodb.BasicDBObject;
+import org.apache.http.client.methods.HttpPost;
+import java.net.URL;
+import java.net.URISyntaxException;
+import java.io.IOException;
+import org.bson.BSONObject;
+import java.util.List;
+import com.fangjia.datastore.config.Trigger;
+import org.apache.http.params.HttpParams;
+import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.params.BasicHttpParams;
-import org.apache.http.params.CoreProtocolPNames;
-import org.apache.http.params.HttpParams;
-import org.apache.http.protocol.HttpContext;
-import org.bson.BSONEncoder;
-import org.bson.BSONObject;
-import org.bson.BasicBSONDecoder;
+import org.apache.http.conn.scheme.SchemeSocketFactory;
+import org.apache.http.conn.scheme.Scheme;
+import org.apache.http.conn.scheme.PlainSocketFactory;
+import org.apache.http.conn.scheme.SchemeRegistry;
+import org.apache.http.client.HttpClient;
import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.sothis.core.util.GZIPUtils;
-import com.fangjia.datastore.config.Trigger;
-import com.mongodb.BasicDBObject;
-import com.mongodb.DBObject;
-
-public class TriggerExecutor {
-
- private final static int MAX_TOTAL = 200;
- private final static int MAX_FOR_ROUTE = 200;
- private final static String DEFAULT_CHARSET = "UTF-8";
- private final static int DEFAULT_PORT = 80;
- private final static int MAX_RETRY_COUNT = 10;
-
- private final static Logger LOGGER = LoggerFactory.getLogger(TriggerExecutor.class);
- private final static TriggerExecutor TRIGGER_EXECUTOR = new TriggerExecutor();
-
- private final HttpClient httpClient;
-
- private TriggerExecutor() {
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("init trigger executor");
- }
- SchemeRegistry schemeRegistry = new SchemeRegistry();
- schemeRegistry.register(new Scheme("http", DEFAULT_PORT, new PlainSocketFactory()));
-
- HttpParams httpParams = new BasicHttpParams();
- httpParams.setParameter(CoreProtocolPNames.HTTP_CONTENT_CHARSET, DEFAULT_CHARSET);
- httpParams.setParameter(AllClientPNames.SO_TIMEOUT, 300000);
- httpParams.setParameter(AllClientPNames.CONNECTION_TIMEOUT, 60000);
-
- ThreadSafeClientConnManager connManager = new ThreadSafeClientConnManager(schemeRegistry);
- connManager.setMaxTotal(MAX_TOTAL);
- connManager.setDefaultMaxPerRoute(MAX_FOR_ROUTE);
-
- httpClient = new DefaultHttpClient(connManager, httpParams);
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("trigger executor initialized[{}]", toString());
- }
- }
-
- public static TriggerExecutor getInstance() {
- return TRIGGER_EXECUTOR;
- }
-
- public List execute(Trigger trigger, List docList) throws TriggerExecuteException {
- try {
- List ret = doExecute(trigger, docList);
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("execute trigger: " + trigger + ", doc size:" + docList.size() + ", return size:" + ret.size());
- }
- return ret;
- } catch (IOException e) {
- throw new TriggerExecuteException("error execute trigger: " + trigger, e);
- } catch (URISyntaxException e) {
- throw new TriggerExecuteException("error execute trigger: " + trigger, e);
- }
- }
-
- private List doExecute(Trigger trigger, List docList) throws IOException, URISyntaxException,
- TriggerExecuteException {
- URL url = new URL(trigger.getInterface());
- HttpPost httpPost = new HttpPost(url.toURI());
- DBObject object = new BasicDBObject();
- object.put("doc_list_size", docList.size());
- object.put("doc_list", docList);
- httpPost.setEntity(new ByteArrayEntity(GZIPUtils.makeAsByteArray(new BSONEncoder().encode(object))));
- HttpHost host = new HttpHost(url.getHost(), url.getPort() == -1 ? DEFAULT_PORT : url.getPort());
- HttpResponse httpResponse = executeWithRetry(host, httpPost, null, trigger);
- InputStream inputStream = httpResponse.getEntity().getContent();
- try {
- if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- BSONObject returnObject = new BasicBSONDecoder().readObject(GZIPUtils.wrap(inputStream));
- if (null != returnObject) {
- List ret = (List) returnObject.get("doc_list");
- if (ret == null) {
- throw new TriggerExecuteException("trigger" + trigger + " return an null doc_list[" + ret + "]");
- }
- return ret;
- } else {
- throw new TriggerExecuteException("trigger" + trigger + " returns nothing at all !");
- }
- } else {
- throw new TriggerExecuteException("error execute trigger:" + trigger + ", get http code:"
- + httpResponse.getStatusLine().getStatusCode());
- }
- } finally {
- if (null != inputStream) {
- inputStream.close();
- }
- }
- }
-
- private HttpResponse executeWithRetry(HttpHost host, HttpPost httpPost, HttpContext httpContext, Trigger trigger)
- throws IOException, TriggerExecuteException {
- int retryCounter = 0;
- HttpResponse httpResponse = null;
- while (true) {
- try {
- httpResponse = httpClient.execute(host, httpPost, (HttpContext) null);
- if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- return httpResponse;
- } else {
- retryCounter++;
- sleepAndRetry(new IllegalStateException("invalid http status code: "
- + httpResponse.getStatusLine().getStatusCode()), retryCounter, trigger);
- }
- } catch (Exception e) {
- retryCounter++;
- sleepAndRetry(e, retryCounter, trigger);
- } finally {
- if (null != httpResponse && httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
- httpResponse.getEntity().getContent().close();
- }
- }
- }
- }
-
- private void sleepAndRetry(Exception e, int retryCounter, Trigger trigger) throws TriggerExecuteException {
- if (retryCounter > MAX_RETRY_COUNT) {
- throw new TriggerExecuteException(
- "error execute trigger[" + trigger + "] after " + MAX_RETRY_COUNT + " times retry!", e);
- }
- if (LOGGER.isErrorEnabled()) {
- LOGGER.error("error execute trigger[" + trigger + "]:(" + e.getMessage() + "), retry for the " + retryCounter
- + "th time after " + (retryCounter * 30) + " seconds...");
- }
- try {
- Thread.sleep(retryCounter * 30000);
- } catch (InterruptedException e1) {
- if (LOGGER.isWarnEnabled()) {
- LOGGER.warn("", e);
- }
- }
- }
+public class TriggerExecutor
+{
+ private static final int MAX_TOTAL = 200;
+ private static final int MAX_FOR_ROUTE = 200;
+ private static final String DEFAULT_CHARSET = "UTF-8";
+ private static final int DEFAULT_PORT = 80;
+ private static final int MAX_RETRY_COUNT = 10;
+ private static final Logger LOGGER;
+ private static final TriggerExecutor TRIGGER_EXECUTOR;
+ private final HttpClient httpClient;
+
+ private TriggerExecutor() {
+ if (TriggerExecutor.LOGGER.isDebugEnabled()) {
+ TriggerExecutor.LOGGER.debug("init trigger executor");
+ }
+ final SchemeRegistry schemeRegistry = new SchemeRegistry();
+ schemeRegistry.register(new Scheme("http", 80, (SchemeSocketFactory)new PlainSocketFactory()));
+ final HttpParams httpParams = (HttpParams)new BasicHttpParams();
+ httpParams.setParameter("http.protocol.content-charset", (Object)"UTF-8");
+ httpParams.setParameter("http.socket.timeout", (Object)300000);
+ httpParams.setParameter("http.connection.timeout", (Object)60000);
+ final ThreadSafeClientConnManager connManager = new ThreadSafeClientConnManager(schemeRegistry);
+ connManager.setMaxTotal(200);
+ connManager.setDefaultMaxPerRoute(200);
+ this.httpClient = (HttpClient)new DefaultHttpClient((ClientConnectionManager)connManager, httpParams);
+ if (TriggerExecutor.LOGGER.isDebugEnabled()) {
+ TriggerExecutor.LOGGER.debug("trigger executor initialized[{}]", (Object)this.toString());
+ }
+ }
+
+ public static TriggerExecutor getInstance() {
+ return TriggerExecutor.TRIGGER_EXECUTOR;
+ }
+
+ public List execute(final Trigger trigger, final List docList) throws TriggerExecuteException {
+ try {
+ final List ret = this.doExecute(trigger, docList);
+ if (TriggerExecutor.LOGGER.isDebugEnabled()) {
+ TriggerExecutor.LOGGER.debug("execute trigger: " + trigger + ", doc size:" + docList.size() + ", return size:" + ret.size());
+ }
+ return ret;
+ }
+ catch (IOException e) {
+ throw new TriggerExecuteException("error execute trigger: " + trigger, e);
+ }
+ catch (URISyntaxException e2) {
+ throw new TriggerExecuteException("error execute trigger: " + trigger, e2);
+ }
+ }
+
+ private List doExecute(final Trigger trigger, final List docList) throws IOException, URISyntaxException, TriggerExecuteException {
+ final URL url = new URL(trigger.getInterface());
+ final HttpPost httpPost = new HttpPost(url.toURI());
+ final DBObject object = (DBObject)new BasicDBObject();
+ object.put("doc_list_size", (Object)docList.size());
+ object.put("doc_list", (Object)docList);
+ httpPost.setEntity((HttpEntity)new ByteArrayEntity(GZIPUtils.makeAsByteArray(new BasicBSONEncoder().encode((BSONObject)object))));
+ final HttpHost host = new HttpHost(url.getHost(), (url.getPort() == -1) ? 80 : url.getPort());
+ final HttpResponse httpResponse = this.executeWithRetry(host, httpPost, null, trigger);
+ final InputStream inputStream = httpResponse.getEntity().getContent();
+ try {
+ if (httpResponse.getStatusLine().getStatusCode() != 200) {
+ throw new TriggerExecuteException("error execute trigger:" + trigger + ", get http code:" + httpResponse.getStatusLine().getStatusCode());
+ }
+ final BSONObject returnObject = new BasicBSONDecoder().readObject(GZIPUtils.wrap(inputStream));
+ if (null == returnObject) {
+ throw new TriggerExecuteException("trigger" + trigger + " returns nothing at all !");
+ }
+ final List ret = (List)returnObject.get("doc_list");
+ if (ret == null) {
+ throw new TriggerExecuteException("trigger" + trigger + " return an null doc_list[" + ret + "]");
+ }
+ return ret;
+ }
+ finally {
+ if (null != inputStream) {
+ inputStream.close();
+ }
+ }
+ }
+
+ private HttpResponse executeWithRetry(final HttpHost host, final HttpPost httpPost, final HttpContext httpContext, final Trigger trigger) throws IOException, TriggerExecuteException {
+ int retryCounter = 0;
+ HttpResponse httpResponse = null;
+ while (true) {
+ try {
+ httpResponse = this.httpClient.execute(host, (HttpRequest)httpPost, (HttpContext)null);
+ if (httpResponse.getStatusLine().getStatusCode() == 200) {
+ return httpResponse;
+ }
+ ++retryCounter;
+ this.sleepAndRetry(new IllegalStateException("invalid http status code: " + httpResponse.getStatusLine().getStatusCode()), retryCounter, trigger);
+ }
+ catch (Exception e) {
+ ++retryCounter;
+ this.sleepAndRetry(e, retryCounter, trigger);
+ }
+ finally {
+ if (null != httpResponse && httpResponse.getStatusLine().getStatusCode() != 200) {
+ httpResponse.getEntity().getContent().close();
+ }
+ }
+ }
+ }
+
+ private void sleepAndRetry(final Exception e, final int retryCounter, final Trigger trigger) throws TriggerExecuteException {
+ if (retryCounter > 10) {
+ throw new TriggerExecuteException("error execute trigger[" + trigger + "] after " + 10 + " times retry!", e);
+ }
+ if (TriggerExecutor.LOGGER.isErrorEnabled()) {
+ TriggerExecutor.LOGGER.error("error execute trigger[" + trigger + "]:(" + e.getMessage() + "), retry for the " + retryCounter + "th time after " + retryCounter * 30 + " seconds...");
+ }
+ try {
+ Thread.sleep(retryCounter * 30000);
+ }
+ catch (InterruptedException e2) {
+ if (TriggerExecutor.LOGGER.isWarnEnabled()) {
+ TriggerExecutor.LOGGER.warn("", (Throwable)e);
+ }
+ }
+ }
+
+ static {
+ LOGGER = LoggerFactory.getLogger((Class)TriggerExecutor.class);
+ TRIGGER_EXECUTOR = new TriggerExecutor();
+ }
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/DataStoreConfig.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/DataStoreConfig.java
index b70a6f73a..f532a2655 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/DataStoreConfig.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/DataStoreConfig.java
@@ -1,170 +1,172 @@
package com.fangjia.datastore.config;
-import java.io.File;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.dom4j.Document;
import org.dom4j.DocumentException;
+import java.io.File;
+import java.util.Iterator;
+import java.util.List;
import org.dom4j.Element;
import org.dom4j.Node;
+import org.dom4j.Document;
+import java.util.Collections;
+import java.util.Set;
+import com.fangjia.datastore.ConfigurationException;
+import java.util.HashMap;
+import java.util.Map;
import org.dom4j.io.SAXReader;
-import com.fangjia.datastore.ConfigurationException;
-
-public class DataStoreConfig implements Reloadable {
- private final SAXReader reader = new SAXReader();
- private final String serverConfigFile;
- private final String storeConfigFile;
- private final ServerConfig serverConfig;
- private final Map storeConfigs;
- private final Map solrConfigs;
- private final Map mongoConfigs;
- private boolean readOnly = false;
-
- public DataStoreConfig(String datastoreHome) throws ConfigurationException {
- this.serverConfigFile = datastoreHome + "/conf/server.xml";
- this.storeConfigFile = datastoreHome + "/conf/store.xml";
- this.serverConfig = new ServerConfig();
- this.storeConfigs = new HashMap();
- this.solrConfigs = new HashMap();
- this.mongoConfigs = new HashMap();
- load();
- }
-
- public ServerConfig getServerConfig() {
- return serverConfig;
- }
-
- public SolrConfig getSolrConfigById(String id) {
- return solrConfigs.get(id);
- }
-
- public MongoConfig getMongoConfigById(String id) {
- return this.mongoConfigs.get(id);
- }
-
- public StoreConfig getStoreConfig(String storeName) {
- return this.storeConfigs.get(storeName);
- }
-
- public Set getStoreNames() {
- return Collections.unmodifiableSet(this.storeConfigs.keySet());
- }
-
- public void reload() throws ConfigurationException {
- load();
- }
-
- private synchronized void load() throws ConfigurationException {
- Document document = read(this.serverConfigFile);
- serverConfig.reload(document.selectSingleNode("//datastore//server"));
- initSolrConfigs(document);
- initMongoConfigs(document);
- document = read(this.storeConfigFile);
- initStoreConfigs(document, solrConfigs, mongoConfigs);
- }
-
- synchronized StoreConfig getOrCreateStoreConfig(String storeName) {
- StoreConfig ret = this.storeConfigs.get(storeName);
- if (null == ret) {
- ret = new StoreConfig(this, storeName);
- storeConfigs.put(storeName, ret);
- }
- return ret;
- }
-
- private synchronized SolrConfig getOrCreateSolrConfig(String solrId) {
- SolrConfig ret = this.solrConfigs.get(solrId);
- if (null == ret) {
- ret = new SolrConfig(this, solrId);
- solrConfigs.put(solrId, ret);
- }
- return ret;
- }
-
- private synchronized MongoConfig getOrCreateMongoConfig(String mongoId) throws ConfigurationException {
- MongoConfig ret = this.mongoConfigs.get(mongoId);
- if (null == ret) {
- ret = new MongoConfig(mongoId);
- mongoConfigs.put(mongoId, ret);
- }
- return ret;
- }
-
- private void initSolrConfigs(Document document) throws ConfigurationException {
- List solrNodes = document.selectNodes("//datastore//solr");
- if (null == solrNodes || solrNodes.isEmpty()) {
- throw new ConfigurationException("no solr is configured in " + serverConfigFile + " !");
- }
- Map newSolrConfigs = new HashMap();
- for (Node solrNode : solrNodes) {
- String id = ((Element) solrNode).attributeValue("id");
- SolrConfig config = getOrCreateSolrConfig(id);
- config.reload(solrNode);
- newSolrConfigs.put(id, config);
- }
- this.solrConfigs.clear();
- this.solrConfigs.putAll(newSolrConfigs);
- }
-
- private void initMongoConfigs(Document document) throws ConfigurationException {
- List mongoNodes = document.selectNodes("//datastore//mongo");
- if (null == mongoNodes || mongoNodes.isEmpty()) {
- throw new ConfigurationException("no mongo is configured in " + serverConfigFile + " !");
- }
- Map newMongoConfigs = new HashMap();
- for (Node mongoNode : mongoNodes) {
- String id = ((Element) mongoNode).attributeValue("id");
- MongoConfig mongoConfig = getOrCreateMongoConfig(id);
- mongoConfig.reload(mongoNode);
- newMongoConfigs.put(id, mongoConfig);
- }
- this.mongoConfigs.clear();
- this.mongoConfigs.putAll(newMongoConfigs);
- }
-
- private void initStoreConfigs(Document document, Map solrConfigMap,
- Map mongoConfigMap) throws ConfigurationException {
- List storeNodes = document.selectNodes("//datastore//store");
- if (null == storeNodes || storeNodes.isEmpty()) {
- throw new ConfigurationException("no store is configured in " + storeConfigFile + " !");
- }
- Map newStoreConfigs = new HashMap();
- for (Node storeNode : storeNodes) {
- String name = ((Element) storeNode).attributeValue("name");
- final StoreConfig storeConfig = getOrCreateStoreConfig(name);
- storeConfig.reload(storeNode);
- newStoreConfigs.put(name, storeConfig);
- }
- this.storeConfigs.clear();
- this.storeConfigs.putAll(newStoreConfigs);
- }
-
- private Document read(String configFile) throws ConfigurationException {
- try {
- return reader.read(new File(configFile));
- } catch (DocumentException e) {
- throw new ConfigurationException("error read datastore config file: " + configFile, e);
- }
- }
-
- @Override
- public String toString() {
- StringBuilder ret = new StringBuilder();
- ret.append("server config file:").append(this.serverConfigFile).append(", store config file: ").append(
- this.storeConfigFile).append(", readOnly:").append(this.readOnly);
- return ret.toString();
- }
-
- public boolean isReadOnly() {
- return readOnly;
- }
-
- public void setReadOnly(boolean readOnly) {
- this.readOnly = readOnly;
- }
+public class DataStoreConfig implements Reloadable
+{
+ private final SAXReader reader;
+ private final String serverConfigFile;
+ private final String storeConfigFile;
+ private final ServerConfig serverConfig;
+ private final Map storeConfigs;
+ private final Map solrConfigs;
+ private final Map mongoConfigs;
+ private boolean readOnly;
+
+ public DataStoreConfig(final String datastoreHome) throws ConfigurationException {
+ this.reader = new SAXReader();
+ this.readOnly = false;
+ this.serverConfigFile = datastoreHome + "/conf/server.xml";
+ this.storeConfigFile = datastoreHome + "/conf/store.xml";
+ this.serverConfig = new ServerConfig();
+ this.storeConfigs = new HashMap();
+ this.solrConfigs = new HashMap();
+ this.mongoConfigs = new HashMap();
+ this.load();
+ }
+
+ public ServerConfig getServerConfig() {
+ return this.serverConfig;
+ }
+
+ public SolrConfig getSolrConfigById(final String id) {
+ return this.solrConfigs.get(id);
+ }
+
+ public MongoConfig getMongoConfigById(final String id) {
+ return this.mongoConfigs.get(id);
+ }
+
+ public StoreConfig getStoreConfig(final String storeName) {
+ return this.storeConfigs.get(storeName);
+ }
+
+ public Set getStoreNames() {
+ return Collections.unmodifiableSet((Set extends String>)this.storeConfigs.keySet());
+ }
+
+ @Override
+ public void reload() throws ConfigurationException {
+ this.load();
+ }
+
+ private synchronized void load() throws ConfigurationException {
+ Document document = this.read(this.serverConfigFile);
+ this.serverConfig.reload(document.selectSingleNode("//datastore//server"));
+ this.initSolrConfigs(document);
+ this.initMongoConfigs(document);
+ document = this.read(this.storeConfigFile);
+ this.initStoreConfigs(document, this.solrConfigs, this.mongoConfigs);
+ }
+
+ synchronized StoreConfig getOrCreateStoreConfig(final String storeName) {
+ StoreConfig ret = this.storeConfigs.get(storeName);
+ if (null == ret) {
+ ret = new StoreConfig(this, storeName);
+ this.storeConfigs.put(storeName, ret);
+ }
+ return ret;
+ }
+
+ private synchronized SolrConfig getOrCreateSolrConfig(final String solrId) {
+ SolrConfig ret = this.solrConfigs.get(solrId);
+ if (null == ret) {
+ ret = new SolrConfig(this, solrId);
+ this.solrConfigs.put(solrId, ret);
+ }
+ return ret;
+ }
+
+ private synchronized MongoConfig getOrCreateMongoConfig(final String mongoId) throws ConfigurationException {
+ MongoConfig ret = this.mongoConfigs.get(mongoId);
+ if (null == ret) {
+ ret = new MongoConfig(mongoId);
+ this.mongoConfigs.put(mongoId, ret);
+ }
+ return ret;
+ }
+
+ private void initSolrConfigs(final Document document) throws ConfigurationException {
+ final List solrNodes = (List)document.selectNodes("//datastore//solr");
+ if (null == solrNodes || solrNodes.isEmpty()) {
+ throw new ConfigurationException("no solr is configured in " + this.serverConfigFile + " !");
+ }
+ final Map newSolrConfigs = new HashMap();
+ for (final Node solrNode : solrNodes) {
+ final String id = ((Element)solrNode).attributeValue("id");
+ final SolrConfig config = this.getOrCreateSolrConfig(id);
+ config.reload(solrNode);
+ newSolrConfigs.put(id, config);
+ }
+ this.solrConfigs.clear();
+ this.solrConfigs.putAll(newSolrConfigs);
+ }
+
+ private void initMongoConfigs(final Document document) throws ConfigurationException {
+ final List mongoNodes = (List)document.selectNodes("//datastore//mongo");
+ if (null == mongoNodes || mongoNodes.isEmpty()) {
+ throw new ConfigurationException("no mongo is configured in " + this.serverConfigFile + " !");
+ }
+ final Map newMongoConfigs = new HashMap();
+ for (final Node mongoNode : mongoNodes) {
+ final String id = ((Element)mongoNode).attributeValue("id");
+ final MongoConfig mongoConfig = this.getOrCreateMongoConfig(id);
+ mongoConfig.reload(mongoNode);
+ newMongoConfigs.put(id, mongoConfig);
+ }
+ this.mongoConfigs.clear();
+ this.mongoConfigs.putAll(newMongoConfigs);
+ }
+
+ private void initStoreConfigs(final Document document, final Map solrConfigMap, final Map mongoConfigMap) throws ConfigurationException {
+ final List storeNodes = (List)document.selectNodes("//datastore//store");
+ if (null == storeNodes || storeNodes.isEmpty()) {
+ throw new ConfigurationException("no store is configured in " + this.storeConfigFile + " !");
+ }
+ final Map newStoreConfigs = new HashMap();
+ for (final Node storeNode : storeNodes) {
+ final String name = ((Element)storeNode).attributeValue("name");
+ final StoreConfig storeConfig = this.getOrCreateStoreConfig(name);
+ storeConfig.reload(storeNode);
+ newStoreConfigs.put(name, storeConfig);
+ }
+ this.storeConfigs.clear();
+ this.storeConfigs.putAll(newStoreConfigs);
+ }
+
+ private Document read(final String configFile) throws ConfigurationException {
+ try {
+ return this.reader.read(new File(configFile));
+ }
+ catch (DocumentException e) {
+ throw new ConfigurationException("error read datastore config file: " + configFile, (Throwable)e);
+ }
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder ret = new StringBuilder();
+ ret.append("server config file:").append(this.serverConfigFile).append(", store config file: ").append(this.storeConfigFile).append(", readOnly:").append(this.readOnly);
+ return ret.toString();
+ }
+
+ public boolean isReadOnly() {
+ return this.readOnly;
+ }
+
+ public void setReadOnly(final boolean readOnly) {
+ this.readOnly = readOnly;
+ }
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Field.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Field.java
index 00c58cb2a..c1fad3456 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Field.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Field.java
@@ -1,127 +1,118 @@
package com.fangjia.datastore.config;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.regex.Pattern;
-
-import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.bson.types.ObjectId;
+import java.util.regex.Pattern;
+import java.util.Date;
+import java.util.Map;
+import java.util.List;
+import org.apache.commons.lang.StringUtils;
-public class Field {
- private String name;
- private FieldTypeEnum type;
- private boolean required;
-
- Field() {
- }
-
- public String getName() {
- return name;
- }
-
- public FieldTypeEnum getType() {
- return type;
- }
-
- void setName(String name) {
- if (StringUtils.isEmpty(name)) {
- throw new IllegalArgumentException("invalid field name:" + name);
- }
- this.name = name;
- }
-
- void setType(FieldTypeEnum type) {
- if (null == type) {
- throw new IllegalArgumentException("invalid type of field[" + name + "]:" + type);
- }
- this.type = type;
- }
-
- public boolean isRequired() {
- return required;
- }
-
- void setRequired(boolean required) {
- this.required = required;
- }
-
- public boolean isBigField() {
- return this.type == FieldTypeEnum.TEXT;
- }
-
- /**
- * 验证一个值是否匹配其对应的类型
- *
- * @param value
- * @return 匹配则返回true,不匹配返回false
- */
- public boolean validate(Object value) {
- if (this.required && value == null) {
- return false;
- }
- if (value == null) {
- return true;
- }
- Class> valueClass = value.getClass();
- switch (this.type) {
- case BOOLEAN:
- return valueClass == boolean.class || valueClass == Boolean.class;
- case BYTE:
- return valueClass == byte.class || valueClass == Byte.class;
- case INTEGER:
- return valueClass == int.class || valueClass == Integer.class;
- case LONG:
- return valueClass == long.class || valueClass == Long.class;
- case DOUBLE:
- return valueClass == double.class || valueClass == Double.class;
- case STRING:
- case TEXT:
- return valueClass == String.class;
- case LIST:
- return value instanceof List;
- case MAP:
- return value instanceof Map;
- case DATETIME:
- return value instanceof Date;
- case BINARY:
- return valueClass == byte[].class;
- case REGEXP:
- return valueClass == Pattern.class;
- case OBJECTID:
- return valueClass == ObjectId.class;
- default:
- throw new RuntimeException("un cased type:" + this);
- }
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder().append(this.name).append(this.type).append(this.required).toHashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this) {
- return true;
- }
- if (null == obj) {
- return false;
- }
- if (!(obj instanceof Field)) {
- return false;
- }
- Field other = (Field) obj;
- return new EqualsBuilder().append(this.name, other.name).append(this.type, other.type)
- .append(this.required, other.required).isEquals();
- }
-
- @Override
- public String toString() {
- return new StringBuilder().append("{field:").append(name).append(", type:").append(type).append(", required:")
- .append(required).append("}").toString();
- }
-
+public class Field
+{
+ private String name;
+ private FieldTypeEnum type;
+ private boolean required;
+
+ Field() {
+ }
+
+ public String getName() {
+ return this.name;
+ }
+
+ public FieldTypeEnum getType() {
+ return this.type;
+ }
+
+ void setName(final String name) {
+ if (StringUtils.isEmpty(name)) {
+ throw new IllegalArgumentException("invalid field name:" + name);
+ }
+ this.name = name;
+ }
+
+ void setType(final FieldTypeEnum type) {
+ if (null == type) {
+ throw new IllegalArgumentException("invalid type of field[" + this.name + "]:" + type);
+ }
+ this.type = type;
+ }
+
+ public boolean isRequired() {
+ return this.required;
+ }
+
+ void setRequired(final boolean required) {
+ this.required = required;
+ }
+
+ public boolean isBigField() {
+ return this.type == FieldTypeEnum.TEXT;
+ }
+
+ public boolean validate(final Object value) {
+ if (this.required && value == null) {
+ return false;
+ }
+ if (value == null) {
+ return true;
+ }
+ final Class> valueClass = value.getClass();
+ switch (this.type) {
+ case BOOLEAN:
+ return valueClass == Boolean.TYPE || valueClass == Boolean.class;
+ case BYTE:
+ return valueClass == Byte.TYPE || valueClass == Byte.class;
+ case INTEGER:
+ return valueClass == Integer.TYPE || valueClass == Integer.class;
+ case LONG:
+ return valueClass == Long.TYPE || valueClass == Long.class;
+ case DOUBLE:
+ return valueClass == Double.TYPE || valueClass == Double.class;
+ case STRING:
+ case TEXT:
+ return valueClass == String.class;
+ case LIST:
+ return value instanceof List;
+ case MAP:
+ return value instanceof Map;
+ case DATETIME:
+ return value instanceof Date;
+ case BINARY:
+ return valueClass == byte[].class;
+ case REGEXP:
+ return valueClass == Pattern.class;
+ case OBJECTID:
+ return valueClass == ObjectId.class;
+ default:
+ throw new RuntimeException("un cased type:" + this);
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder().append((Object)this.name).append((Object)this.type).append(this.required).toHashCode();
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (null == obj) {
+ return false;
+ }
+ if (!(obj instanceof Field)) {
+ return false;
+ }
+ final Field other = (Field)obj;
+ return new EqualsBuilder().append((Object)this.name, (Object)other.name).append((Object)this.type, (Object)other.type).append(this.required, other.required).isEquals();
+ }
+
+ @Override
+ public String toString() {
+ return "{field:" + this.name + ", type:" + this.type + ", required:" + this.required + "}";
+ }
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/FieldTypeEnum.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/FieldTypeEnum.java
index a75f63b43..ffd4b0bb2 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/FieldTypeEnum.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/FieldTypeEnum.java
@@ -1,22 +1,31 @@
package com.fangjia.datastore.config;
-public enum FieldTypeEnum {
- BOOLEAN, BYTE, INTEGER, LONG, DOUBLE, STRING, TEXT, LIST, MAP, DATETIME, BINARY, REGEXP, OBJECTID;
-
- private FieldTypeEnum() {
- }
-
- public String getType() {
- return this.name().toLowerCase();
- }
-
- public static FieldTypeEnum parse(String type) {
- for (FieldTypeEnum typeEnum : FieldTypeEnum.values()) {
- if (typeEnum.name().toLowerCase().equals(type)) {
- return typeEnum;
- }
- }
- return null;
- }
-
+public enum FieldTypeEnum
+{
+ BOOLEAN,
+ BYTE,
+ INTEGER,
+ LONG,
+ DOUBLE,
+ STRING,
+ TEXT,
+ LIST,
+ MAP,
+ DATETIME,
+ BINARY,
+ REGEXP,
+ OBJECTID;
+
+ public String getType() {
+ return this.name().toLowerCase();
+ }
+
+ public static FieldTypeEnum parse(final String type) {
+ for (final FieldTypeEnum typeEnum : values()) {
+ if (typeEnum.name().toLowerCase().equals(type)) {
+ return typeEnum;
+ }
+ }
+ return null;
+ }
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/MongoConfig.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/MongoConfig.java
index a3d352e24..98aee8f1d 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/MongoConfig.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/MongoConfig.java
@@ -2,87 +2,98 @@ package com.fangjia.datastore.config;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.dom4j.Node;
-
-import com.fangjia.datastore.ConfigurationException;
import com.fangjia.datastore.utils.XmlUtils;
+import org.dom4j.Node;
+import com.mongodb.MongoURI;
+import org.apache.commons.lang.StringUtils;
+import com.fangjia.datastore.ConfigurationException;
import com.mongodb.Mongo;
-public final class MongoConfig {
-
- private String host;
- private int port;
- private final String id;
- private Mongo mongo;
-
- MongoConfig(String id) throws ConfigurationException {
- this.id = id;
- }
-
- private synchronized void init() throws ConfigurationException {
- if (null == mongo) {
- try {
- mongo = new Mongo(host, port);
- } catch (Exception e) {
- throw new ConfigurationException("error init mongo[" + host + ":" + port, e);
- }
- }
- }
-
- public Mongo getMongo() throws ConfigurationException {
- if (null == mongo) {
- init();
- }
- return mongo;
- }
-
- synchronized void reload(Node mongoNode) throws ConfigurationException {
- String host = XmlUtils.getNodeValue(mongoNode, "host", "datastore.mongo.db.fangjia.com");
- int port = XmlUtils.getNodeValueAsInt(mongoNode, "port", 27017);
- if (host.equals(this.host) && this.port == port) {
- return;
- }
- this.host = host;
- this.port = port;
- if (null != mongo) {
- try {
- init();
- } catch (Exception e) {
- throw new ConfigurationException("error init mongo[" + host + ":" + port, e);
- }
- }
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder().append(this.host).append(this.port).append(this.id).toHashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this) {
- return true;
- }
- if (null == obj) {
- return false;
- }
- if (!(obj instanceof MongoConfig)) {
- return false;
- }
- MongoConfig other = (MongoConfig) obj;
- return new EqualsBuilder().append(this.id, other.id).append(this.host, other.host).append(this.port, other.port)
- .isEquals();
- }
-
- @Override
- public String toString() {
- StringBuilder ret = new StringBuilder();
- ret.append('{');
- ret.append("id:").append(this.id);
- ret.append(", host:").append(this.host);
- ret.append(", port:").append(this.port);
- ret.append('}');
- return ret.toString();
- }
-
+public final class MongoConfig
+{
+ private String host;
+ private String uri;
+ private int port;
+ private final String id;
+ private Mongo mongo;
+
+ MongoConfig(final String id) throws ConfigurationException {
+ this.id = id;
+ }
+
+ private synchronized void init() throws ConfigurationException {
+ if (null == this.mongo) {
+ try {
+ MongoURI mongoURI;
+ if (StringUtils.isNotBlank(this.uri)) {
+ mongoURI = new MongoURI(this.uri);
+ }
+ else {
+ mongoURI = new MongoURI("mongodb://" + this.host + ":" + this.port);
+ }
+ this.mongo = new Mongo(mongoURI);
+ }
+ catch (Exception e) {
+ throw new ConfigurationException("error init mongo[" + this.host + ":" + this.port, e);
+ }
+ }
+ }
+
+ public Mongo getMongo() throws ConfigurationException {
+ if (null == this.mongo) {
+ this.init();
+ }
+ return this.mongo;
+ }
+
+ synchronized void reload(final Node mongoNode) throws ConfigurationException {
+ final String host = XmlUtils.getNodeValue(mongoNode, "host", "datastore.mongo.db.fangjia.com");
+ final int port = XmlUtils.getNodeValueAsInt(mongoNode, "port", 27017);
+ final String uri = XmlUtils.getNodeValue(mongoNode, "uri", "");
+ if (host.equals(this.host) && this.port == port) {
+ return;
+ }
+ this.host = host;
+ this.port = port;
+ this.uri = uri;
+ if (null != this.mongo) {
+ try {
+ this.init();
+ }
+ catch (Exception e) {
+ throw new ConfigurationException("error init mongo[" + host + ":" + port, e);
+ }
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder().append((Object)this.host).append(this.port).append((Object)this.id).toHashCode();
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (null == obj) {
+ return false;
+ }
+ if (!(obj instanceof MongoConfig)) {
+ return false;
+ }
+ final MongoConfig other = (MongoConfig)obj;
+ return new EqualsBuilder().append((Object)this.id, (Object)other.id).append((Object)this.host, (Object)other.host).append(this.port, other.port).isEquals();
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder ret = new StringBuilder();
+ ret.append('{');
+ ret.append("id:").append(this.id);
+ ret.append(", host:").append(this.host);
+ ret.append(", port:").append(this.port);
+ ret.append('}');
+ return ret.toString();
+ }
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Reloadable.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Reloadable.java
index 07bd3e51b..ba98d1688 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Reloadable.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Reloadable.java
@@ -2,6 +2,7 @@ package com.fangjia.datastore.config;
import com.fangjia.datastore.ConfigurationException;
-public interface Reloadable {
- void reload() throws ConfigurationException;
+public interface Reloadable
+{
+ void reload() throws ConfigurationException;
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/ServerConfig.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/ServerConfig.java
index 9159563dc..3bd447036 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/ServerConfig.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/ServerConfig.java
@@ -1,74 +1,72 @@
package com.fangjia.datastore.config;
+import com.fangjia.datastore.utils.XmlUtils;
+import com.fangjia.datastore.ConfigurationException;
import org.dom4j.Node;
-import com.fangjia.datastore.ConfigurationException;
-import com.fangjia.datastore.utils.XmlUtils;
-
-public final class ServerConfig {
- private int port;
- private boolean disableLogging;
- private long performanceLogFilterMaxTime;
- private int minaReadBufferSize;
- private int cursorExpirationMinutes;
- private int defaultCursorLimit;
- private int maxConnections;
- private int triggerBatchSize;
- private int reIndexBatchSize;
-
- ServerConfig() {
- super();
- }
-
- void reload(Node server) throws ConfigurationException {
- if (null == server) {
- throw new ConfigurationException("no server is configured!");
- }
- this.port = XmlUtils.getNodeValueAsInt(server, "port", 27017);
- this.disableLogging = XmlUtils.getNodeValueAsBoolean(server, "disableLogging", false);
- this.performanceLogFilterMaxTime = XmlUtils.getNodeValueAsLong(server, "performanceLogFilterMaxTime", 1000);
- this.minaReadBufferSize = XmlUtils.getNodeValueAsInt(server, "minaReadBufferSize", 20480);
- this.cursorExpirationMinutes = XmlUtils.getNodeValueAsInt(server, "cursorExpirationMinutes", 5);
- this.defaultCursorLimit = XmlUtils.getNodeValueAsInt(server, "defaultCursorLimit", 20);
- this.maxConnections = XmlUtils.getNodeValueAsInt(server, "maxConnections", 10);
- this.triggerBatchSize = XmlUtils.getNodeValueAsInt(server, "triggerBatchSize", 200);
- this.reIndexBatchSize = XmlUtils.getNodeValueAsInt(server, "reIndexBatchSize", 200);
- }
-
- public int getPort() {
- return port;
- }
-
- public boolean isDisableLogging() {
- return disableLogging;
- }
-
- public long getPerformanceLogFilterMaxTime() {
- return performanceLogFilterMaxTime;
- }
-
- public int getMinaReadBufferSize() {
- return minaReadBufferSize;
- }
-
- public int getCursorExpirationMinutes() {
- return cursorExpirationMinutes;
- }
-
- public int getDefaultCursorLimit() {
- return defaultCursorLimit;
- }
-
- public int getMaxConnections() {
- return maxConnections;
- }
-
- public int getTriggerBatchSize() {
- return triggerBatchSize;
- }
-
- public int getReIndexBatchSize() {
- return reIndexBatchSize;
- }
-
+public final class ServerConfig
+{
+ private int port;
+ private boolean disableLogging;
+ private long performanceLogFilterMaxTime;
+ private int minaReadBufferSize;
+ private int cursorExpirationMinutes;
+ private int defaultCursorLimit;
+ private int maxConnections;
+ private int triggerBatchSize;
+ private int reIndexBatchSize;
+
+ ServerConfig() {
+ }
+
+ void reload(final Node server) throws ConfigurationException {
+ if (null == server) {
+ throw new ConfigurationException("no server is configured!");
+ }
+ this.port = XmlUtils.getNodeValueAsInt(server, "port", 27017);
+ this.disableLogging = XmlUtils.getNodeValueAsBoolean(server, "disableLogging", false);
+ this.performanceLogFilterMaxTime = XmlUtils.getNodeValueAsLong(server, "performanceLogFilterMaxTime", 1000L);
+ this.minaReadBufferSize = XmlUtils.getNodeValueAsInt(server, "minaReadBufferSize", 20480);
+ this.cursorExpirationMinutes = XmlUtils.getNodeValueAsInt(server, "cursorExpirationMinutes", 5);
+ this.defaultCursorLimit = XmlUtils.getNodeValueAsInt(server, "defaultCursorLimit", 20);
+ this.maxConnections = XmlUtils.getNodeValueAsInt(server, "maxConnections", 10);
+ this.triggerBatchSize = XmlUtils.getNodeValueAsInt(server, "triggerBatchSize", 200);
+ this.reIndexBatchSize = XmlUtils.getNodeValueAsInt(server, "reIndexBatchSize", 200);
+ }
+
+ public int getPort() {
+ return this.port;
+ }
+
+ public boolean isDisableLogging() {
+ return this.disableLogging;
+ }
+
+ public long getPerformanceLogFilterMaxTime() {
+ return this.performanceLogFilterMaxTime;
+ }
+
+ public int getMinaReadBufferSize() {
+ return this.minaReadBufferSize;
+ }
+
+ public int getCursorExpirationMinutes() {
+ return this.cursorExpirationMinutes;
+ }
+
+ public int getDefaultCursorLimit() {
+ return this.defaultCursorLimit;
+ }
+
+ public int getMaxConnections() {
+ return this.maxConnections;
+ }
+
+ public int getTriggerBatchSize() {
+ return this.triggerBatchSize;
+ }
+
+ public int getReIndexBatchSize() {
+ return this.reIndexBatchSize;
+ }
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/SolrConfig.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/SolrConfig.java
index 283d169e0..f08c37efd 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/SolrConfig.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/SolrConfig.java
@@ -1,132 +1,132 @@
package com.fangjia.datastore.config;
+import com.fangjia.datastore.ConfigurationException;
import org.apache.commons.lang.StringUtils;
+import com.fangjia.datastore.utils.XmlUtils;
import org.dom4j.Node;
-import com.fangjia.datastore.ConfigurationException;
-import com.fangjia.datastore.utils.XmlUtils;
-
-public class SolrConfig {
- private final String id;
- private String host;
- private String home;
- private final DataStoreConfig dataStoreConfig;
- private String shards;
- private String slaves;
- private SolrConfig[] shardConfigs;
- private SolrConfig[] slaveConfigs;
- private String filter;
-
- SolrConfig(DataStoreConfig dataStoreConfig, String id) {
- this.dataStoreConfig = dataStoreConfig;
- this.id = id;
- }
-
- synchronized void reload(Node solrNode) throws ConfigurationException {
- this.host = removeLastSlash(XmlUtils.getNodeValue(solrNode, "host", null));
- this.shards = XmlUtils.getNodeValue(solrNode, "shards", null);
- this.slaves = XmlUtils.getNodeValue(solrNode, "slaves", null);
- this.home = XmlUtils.getNodeValue(solrNode, "home", null);
- if (StringUtils.isNotEmpty(shards) && StringUtils.isEmpty(home)) {
- throw new ConfigurationException("[" + id + "] invalid home attribute:" + home);
- }
- if (StringUtils.isEmpty(shards) && StringUtils.isEmpty(slaves) && StringUtils.isEmpty(host)) {
- throw new ConfigurationException("[" + id + "] invalid host attribute:" + host);
- }
- this.filter = removeLastSlash(XmlUtils.getNodeValue(solrNode, "filter", null));
- }
-
- private SolrConfig[] getConfigs(String ids) {
- SolrConfig[] ret;
- if (StringUtils.isBlank(ids)) {
- ret = new SolrConfig[0];
- } else {
- String[] idSplit = ids.split(",");
- ret = new SolrConfig[idSplit.length];
- for (int i = 0; i < idSplit.length; i++) {
- ret[i] = dataStoreConfig.getSolrConfigById(idSplit[i]);
- }
- }
- return ret;
- }
-
- public SolrConfig[] getShardsConfigs() {
- if (null == shardConfigs) {
- shardConfigs = this.getConfigs(shards);
- }
- return shardConfigs;
- }
-
- public SolrConfig[] getSlaveConfigs() {
- if (null == slaveConfigs) {
- slaveConfigs = this.getConfigs(slaves);
- }
- return slaveConfigs;
- }
-
- private String removeLastSlash(String s) {
- if (StringUtils.isEmpty(s)) {
- return s;
- }
- if (s.charAt(s.length() - 1) == '/') {
- return s.substring(0, s.length() - 1);
- }
- return s;
- }
-
- public String getShardsParam(String coreName) {
- SolrConfig[] shardConfigs = getShardsConfigs();
- if (shardConfigs.length == 0) {
- return "";
- }
- StringBuilder ret = new StringBuilder();
- for (int i = 0; i < shardConfigs.length; i++) {
- ret.append(shardConfigs[i].host).append('/').append(coreName);
- if (i < shardConfigs.length - 1) {
- ret.append(',');
- }
- }
- return ret.toString();
- }
-
- public String getId() {
- return id;
- }
-
- public String getHost() {
- return host;
- }
-
- public DataStoreConfig getDataStoreConfig() {
- return dataStoreConfig;
- }
-
- public String getShards() {
- return shards;
- }
-
- public String getFilter() {
- return filter;
- }
-
- public String getHome() {
- return home;
- }
-
- @Override
- public String toString() {
- StringBuilder ret = new StringBuilder();
- ret.append('{');
- ret.append("id: ").append(this.id);
- ret.append(", host: ").append(this.host);
- ret.append(", shards: ").append(this.shards);
- ret.append(", slaves: ").append(this.slaves);
- ret.append('}');
- return ret.toString();
- }
-
- public String getSlaves() {
- return slaves;
- }
-
+public class SolrConfig
+{
+ private final String id;
+ private String host;
+ private String home;
+ private final DataStoreConfig dataStoreConfig;
+ private String shards;
+ private String slaves;
+ private SolrConfig[] shardConfigs;
+ private SolrConfig[] slaveConfigs;
+ private String filter;
+
+ SolrConfig(final DataStoreConfig dataStoreConfig, final String id) {
+ this.dataStoreConfig = dataStoreConfig;
+ this.id = id;
+ }
+
+ synchronized void reload(final Node solrNode) throws ConfigurationException {
+ this.host = this.removeLastSlash(XmlUtils.getNodeValue(solrNode, "host", null));
+ this.shards = XmlUtils.getNodeValue(solrNode, "shards", null);
+ this.slaves = XmlUtils.getNodeValue(solrNode, "slaves", null);
+ this.home = XmlUtils.getNodeValue(solrNode, "home", null);
+ if (StringUtils.isNotEmpty(this.shards) && StringUtils.isEmpty(this.home)) {
+ throw new ConfigurationException("[" + this.id + "] invalid home attribute:" + this.home);
+ }
+ if (StringUtils.isEmpty(this.shards) && StringUtils.isEmpty(this.slaves) && StringUtils.isEmpty(this.host)) {
+ throw new ConfigurationException("[" + this.id + "] invalid host attribute:" + this.host);
+ }
+ this.filter = this.removeLastSlash(XmlUtils.getNodeValue(solrNode, "filter", null));
+ }
+
+ private SolrConfig[] getConfigs(final String ids) {
+ SolrConfig[] ret;
+ if (StringUtils.isBlank(ids)) {
+ ret = new SolrConfig[0];
+ }
+ else {
+ final String[] idSplit = ids.split(",");
+ ret = new SolrConfig[idSplit.length];
+ for (int i = 0; i < idSplit.length; ++i) {
+ ret[i] = this.dataStoreConfig.getSolrConfigById(idSplit[i]);
+ }
+ }
+ return ret;
+ }
+
+ public SolrConfig[] getShardsConfigs() {
+ if (null == this.shardConfigs) {
+ this.shardConfigs = this.getConfigs(this.shards);
+ }
+ return this.shardConfigs;
+ }
+
+ public SolrConfig[] getSlaveConfigs() {
+ if (null == this.slaveConfigs) {
+ this.slaveConfigs = this.getConfigs(this.slaves);
+ }
+ return this.slaveConfigs;
+ }
+
+ private String removeLastSlash(final String s) {
+ if (StringUtils.isEmpty(s)) {
+ return s;
+ }
+ if (s.charAt(s.length() - 1) == '/') {
+ return s.substring(0, s.length() - 1);
+ }
+ return s;
+ }
+
+ public String getShardsParam(final String coreName) {
+ final SolrConfig[] shardConfigs = this.getShardsConfigs();
+ if (shardConfigs.length == 0) {
+ return "";
+ }
+ final StringBuilder ret = new StringBuilder();
+ for (int i = 0; i < shardConfigs.length; ++i) {
+ ret.append(shardConfigs[i].host).append('/').append(coreName);
+ if (i < shardConfigs.length - 1) {
+ ret.append(',');
+ }
+ }
+ return ret.toString();
+ }
+
+ public String getId() {
+ return this.id;
+ }
+
+ public String getHost() {
+ return this.host;
+ }
+
+ public DataStoreConfig getDataStoreConfig() {
+ return this.dataStoreConfig;
+ }
+
+ public String getShards() {
+ return this.shards;
+ }
+
+ public String getFilter() {
+ return this.filter;
+ }
+
+ public String getHome() {
+ return this.home;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder ret = new StringBuilder();
+ ret.append('{');
+ ret.append("id: ").append(this.id);
+ ret.append(", host: ").append(this.host);
+ ret.append(", shards: ").append(this.shards);
+ ret.append(", slaves: ").append(this.slaves);
+ ret.append('}');
+ return ret.toString();
+ }
+
+ public String getSlaves() {
+ return this.slaves;
+ }
}
diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/StoreConfig.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/StoreConfig.java
index fbeed7114..2d289057a 100644
--- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/StoreConfig.java
+++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/StoreConfig.java
@@ -1,229 +1,208 @@
package com.fangjia.datastore.config;
-import java.util.Collection;
import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
+import java.util.Collection;
import org.bson.types.ObjectId;
+import java.util.Iterator;
+import java.util.List;
+import com.fangjia.datastore.utils.XmlUtils;
+import com.fangjia.datastore.ConfigurationException;
+import org.apache.commons.lang.StringUtils;
import org.dom4j.Element;
import org.dom4j.Node;
+import java.util.HashMap;
+import java.util.Map;
-import com.fangjia.datastore.ConfigurationException;
-import com.fangjia.datastore.utils.XmlUtils;
-
-public final class StoreConfig {
-
- private final String name;
- private int solrMaxTotalConnections;
- private int solrMaxConnectionsPerHost;
- private SolrConfig[] masterSolrConfigs;
- private SolrConfig masterSolrConfig;
- private SolrConfig slaveSolrConfig;
- private MongoConfig mongoConfig;
- private boolean textFieldToGridFS;
- private final Trigger trigger;
- private final Map