diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java index e11cd2246..adfb2c7c8 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java @@ -1847,6 +1847,9 @@ public class DataUtil { // // System.out.println(FourUpSixInto("258.393", 2)); + String href = encodePicUrl("62afd81d303c3f827833bb03", "o", "upload"); + String link = "http://static.fangjia.com/stc/fs/"+href; + System.out.println(link); } } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java index 41941514a..4b72f411c 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java @@ -105,6 +105,7 @@ public class FjtEvaluation implements SqlEntity{ private String certificateAddress; //产证地址 private Date downPdfDate;//下载报告时间 + private Integer loan ;//是否续贷 1:是 0:否 @GeneratedValue(strategy = GenerationType.AUTO) @Id @@ -773,6 +774,15 @@ public class FjtEvaluation implements SqlEntity{ this.downPdfDate = downPdfDate; } + @Column(name = "loan") + public Integer getLoan() { + return loan; + } + + public void setLoan(Integer loan) { + this.loan = loan; + } + @Override public String toString() { return "FjtEvaluation{" + diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ReportFileAttach.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ReportFileAttach.java index a22ef8332..acc9fcd36 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ReportFileAttach.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ReportFileAttach.java @@ -11,7 +11,7 @@ public class ReportFileAttach implements SqlEntity { private Integer userId; private Integer evaluationId; private String queryId; - private Integer fileClass; + private Integer fileClass;//1:询价单 2:产证 3:房屋照片 4:其它附件 5:产调报告或权籍 private String fileName; private String fileId; diff --git a/projects/fangjia-datastore/pom.xml b/projects/fangjia-datastore/pom.xml index e86529d96..5c5fb279c 100644 --- a/projects/fangjia-datastore/pom.xml +++ b/projects/fangjia-datastore/pom.xml @@ -33,7 +33,7 @@ true lib/ com.fangjia.datastore.DataStoreServer - + @@ -94,8 +94,8 @@ org.mongodb - mongo-java-driver - 2.6.5 + mongodb-driver-legacy + 3.9.1 log4j diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/ConfigurationException.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/ConfigurationException.java index 1f740a992..9631409cf 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/ConfigurationException.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/ConfigurationException.java @@ -1,22 +1,21 @@ package com.fangjia.datastore; -public class ConfigurationException extends Exception { - - private static final long serialVersionUID = 6570405263317311822L; - - public ConfigurationException() { - } - - public ConfigurationException(String message) { - super(message); - } - - public ConfigurationException(Throwable cause) { - super(cause); - } - - public ConfigurationException(String message, Throwable cause) { - super(message, cause); - } - +public class ConfigurationException extends Exception +{ + private static final long serialVersionUID = 6570405263317311822L; + + public ConfigurationException() { + } + + public ConfigurationException(final String message) { + super(message); + } + + public ConfigurationException(final Throwable cause) { + super(cause); + } + + public ConfigurationException(final String message, final Throwable cause) { + super(message, cause); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Cursor.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Cursor.java index ab27690ef..1303b2372 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Cursor.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Cursor.java @@ -1,80 +1,83 @@ package com.fangjia.datastore; +import com.fangjia.datastore.mongo.protocol.QueryMessage; import java.util.concurrent.atomic.AtomicLong; -import com.fangjia.datastore.mongo.protocol.QueryMessage; - -public class Cursor { - private static final AtomicLong ID_GENERATOR = new AtomicLong(1); - private long id; - private int limit; - private int skip; - private long totalCount; - private QueryMessage queryMessage; - - public Cursor(int skip, int limit, QueryMessage queryMessage, long totalCount) { - id = ID_GENERATOR.getAndIncrement(); - this.skip = skip; - this.limit = limit; - this.totalCount = totalCount; - this.queryMessage = queryMessage; - } - - public long getId() { - return id; - } - - public int getLimit() { - return limit; - } - - public void setLimit(int limit) { - this.limit = limit; - } - - public int nextSkip() { - skip += limit; - return skip; - } - - public void setSkip(int skip) { - this.skip = skip; - } - - public boolean hasMore() { - return this.totalCount > this.skip; - } - - public long getTotalCount() { - return totalCount; - } - - public void setTotalCount(long totalCount) { - this.totalCount = totalCount; - } - - public QueryMessage getQueryMessage() { - return queryMessage; - } - - public void setQueryMessage(QueryMessage queryMessage) { - this.queryMessage = queryMessage; - } - - public int getSkip() { - return skip; - } - - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("id:").append(id); - builder.append(", limit:").append(limit); - builder.append(", skip:").append(skip); - builder.append(", totalCount:").append(totalCount); - builder.append(", queryMessage:").append(queryMessage); - builder.append("}"); - return builder.toString(); - } - +public class Cursor +{ + private static final AtomicLong ID_GENERATOR; + private long id; + private int limit; + private int skip; + private long totalCount; + private QueryMessage queryMessage; + + public Cursor(final int skip, final int limit, final QueryMessage queryMessage, final long totalCount) { + this.id = Cursor.ID_GENERATOR.getAndIncrement(); + this.skip = skip; + this.limit = limit; + this.totalCount = totalCount; + this.queryMessage = queryMessage; + } + + public long getId() { + return this.id; + } + + public int getLimit() { + return this.limit; + } + + public void setLimit(final int limit) { + this.limit = limit; + } + + public int nextSkip() { + return this.skip += this.limit; + } + + public void setSkip(final int skip) { + this.skip = skip; + } + + public boolean hasMore() { + return this.totalCount > this.skip; + } + + public long getTotalCount() { + return this.totalCount; + } + + public void setTotalCount(final long totalCount) { + this.totalCount = totalCount; + } + + public QueryMessage getQueryMessage() { + return this.queryMessage; + } + + public void setQueryMessage(final QueryMessage queryMessage) { + this.queryMessage = queryMessage; + } + + public int getSkip() { + return this.skip; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("id:").append(this.id); + builder.append(", limit:").append(this.limit); + builder.append(", skip:").append(this.skip); + builder.append(", totalCount:").append(this.totalCount); + builder.append(", queryMessage:").append(this.queryMessage); + builder.append("}"); + return builder.toString(); + } + + static { + ID_GENERATOR = new AtomicLong(1L); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStore.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStore.java index 52f2196ed..619d7f038 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStore.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStore.java @@ -1,44 +1,35 @@ package com.fangjia.datastore; -import java.util.Collection; import java.util.List; - import org.bson.BSONObject; - -import com.fangjia.datastore.config.Reloadable; -import com.fangjia.datastore.config.StoreConfig; import com.fangjia.datastore.solr.Index; +import java.util.Collection; +import com.fangjia.datastore.config.StoreConfig; +import com.fangjia.datastore.config.Reloadable; -public interface DataStore extends Reloadable { - - StoreConfig getStoreConfig(); - - Mongo getMongo(); - - Object getAttribute(String key); - - Object setAttribute(String key, Object attr); - - Collection getIndexes() throws DataStoreException; - - /** - * 按查询条件重建索引 - * - * @param query - * @throws DataStoreException - */ - BSONObject reIndex(BSONObject query) throws DataStoreException; - - void ensureIndex(Collection indexes) throws DataStoreException; - - QueryResponse find(BSONObject query, BSONObject fields, BSONObject sort, BSONObject facet, int skip, int limit, boolean bulk, - boolean fromMaster) throws DataStoreException; - - int count(BSONObject query, boolean fromMaster) throws DataStoreException; - - void remove(BSONObject query) throws DataStoreException; - - void update(BSONObject query, BSONObject data,String ip) throws DataStoreException; - - void save(List docList) throws DataStoreException; +public interface DataStore extends Reloadable +{ + StoreConfig getStoreConfig(); + + Mongo getMongo(); + + Object getAttribute(final String p0); + + Object setAttribute(final String p0, final Object p1); + + Collection getIndexes() throws DataStoreException; + + BSONObject reIndex(final BSONObject p0) throws DataStoreException; + + void ensureIndex(final Collection p0) throws DataStoreException; + + QueryResponse find(final BSONObject p0, final BSONObject p1, final BSONObject p2, final BSONObject p3, final int p4, final int p5, final boolean p6, final boolean p7) throws DataStoreException; + + int count(final BSONObject p0, final boolean p1) throws DataStoreException; + + void remove(final BSONObject p0) throws DataStoreException; + + int update(final BSONObject p0, final BSONObject p1) throws DataStoreException; + + void save(final List p0) throws DataStoreException; } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContext.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContext.java index 7777ebef4..bb2f19617 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContext.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContext.java @@ -3,15 +3,15 @@ package com.fangjia.datastore; import com.fangjia.datastore.config.DataStoreConfig; import com.fangjia.datastore.config.Reloadable; -public interface DataStoreContext extends Reloadable { - - DataStoreConfig getDataStoreConfig(); - - DataStore getDataStore(String storeName) throws DataStoreException; - - Cursor getCursor(long cursorId); - - void addCursor(Cursor cursor); - - Cursor removeCursor(long cursorId); +public interface DataStoreContext extends Reloadable +{ + DataStoreConfig getDataStoreConfig(); + + DataStore getDataStore(final String p0) throws DataStoreException; + + Cursor getCursor(final long p0); + + void addCursor(final Cursor p0); + + Cursor removeCursor(final long p0); } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContextImpl.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContextImpl.java index 2e11d6ca7..54fde188a 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContextImpl.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContextImpl.java @@ -1,56 +1,61 @@ package com.fangjia.datastore; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import com.fangjia.datastore.config.DataStoreConfig; import com.fangjia.datastore.config.StoreConfig; +import java.util.Iterator; +import java.util.Set; +import java.util.HashMap; +import java.util.concurrent.TimeUnit; import com.google.common.collect.MapMaker; +import java.util.Map; +import com.fangjia.datastore.config.DataStoreConfig; -public class DataStoreContextImpl implements DataStoreContext { - - private final DataStoreConfig config; - private final Map cursorMap; - private final Map stores; - - public DataStoreContextImpl(DataStoreConfig config) throws ConfigurationException { - this.config = config; - cursorMap = new MapMaker().concurrencyLevel(50).expiration(config.getServerConfig().getCursorExpirationMinutes(), - TimeUnit.MINUTES).makeMap(); - Set storeNames = config.getStoreNames(); - stores = new HashMap(storeNames.size()); - for (String storeName : storeNames) { - StoreConfig storeConfig = this.config.getStoreConfig(storeName); - stores.put(storeName, new DataStoreImpl(storeConfig, new Mongo(storeConfig.getMongoConfig()))); - } - } - - public DataStoreConfig getDataStoreConfig() { - return config; - } - - public DataStore getDataStore(String storeName) throws DataStoreException { - if (!stores.containsKey(storeName)) { - throw new DataStoreException("unknown store name: \"" + storeName + "\""); - } - return stores.get(storeName); - } - - public void reload() throws ConfigurationException { - config.reload(); - } - - public Cursor getCursor(long cursorId) { - return this.cursorMap.get(cursorId); - } - - public void addCursor(Cursor cursor) { - this.cursorMap.put(cursor.getId(), cursor); - } - - public Cursor removeCursor(long cursorId) { - return this.cursorMap.remove(cursorId); - } +public class DataStoreContextImpl implements DataStoreContext +{ + private final DataStoreConfig config; + private final Map cursorMap; + private final Map stores; + + public DataStoreContextImpl(final DataStoreConfig config) throws ConfigurationException { + this.config = config; + this.cursorMap = new MapMaker().concurrencyLevel(50).expiration((long)config.getServerConfig().getCursorExpirationMinutes(), TimeUnit.MINUTES).makeMap(); + final Set storeNames = config.getStoreNames(); + this.stores = new HashMap(storeNames.size()); + for (final String storeName : storeNames) { + final StoreConfig storeConfig = this.config.getStoreConfig(storeName); + this.stores.put(storeName, new DataStoreImpl(storeConfig, new Mongo(storeConfig.getMongoConfig()))); + } + } + + @Override + public DataStoreConfig getDataStoreConfig() { + return this.config; + } + + @Override + public DataStore getDataStore(final String storeName) throws DataStoreException { + if (!this.stores.containsKey(storeName)) { + throw new DataStoreException("unknown store name: \"" + storeName + "\""); + } + return this.stores.get(storeName); + } + + @Override + public void reload() throws ConfigurationException { + this.config.reload(); + } + + @Override + public Cursor getCursor(final long cursorId) { + return this.cursorMap.get(cursorId); + } + + @Override + public void addCursor(final Cursor cursor) { + this.cursorMap.put(cursor.getId(), cursor); + } + + @Override + public Cursor removeCursor(final long cursorId) { + return this.cursorMap.remove(cursorId); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreException.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreException.java index 034460ce5..23e8b15cb 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreException.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreException.java @@ -1,23 +1,21 @@ package com.fangjia.datastore; -public class DataStoreException extends Exception { - - private static final long serialVersionUID = -5492973985910379215L; - - public DataStoreException() { - super(); - } - - public DataStoreException(String message, Throwable cause) { - super(message, cause); - } - - public DataStoreException(String message) { - super(message); - } - - public DataStoreException(Throwable cause) { - super(cause); - } - +public class DataStoreException extends Exception +{ + private static final long serialVersionUID = -5492973985910379215L; + + public DataStoreException() { + } + + public DataStoreException(final String message, final Throwable cause) { + super(message, cause); + } + + public DataStoreException(final String message) { + super(message); + } + + public DataStoreException(final Throwable cause) { + super(cause); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreImpl.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreImpl.java index 9f10d1e4a..215b15b64 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreImpl.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreImpl.java @@ -1,702 +1,637 @@ package com.fangjia.datastore; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; +import com.fangjia.datastore.utils.LoggerFactory; +import com.mongodb.WriteResult; +import com.mongodb.WriteConcern; import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.commons.collections.CollectionUtils; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.Collections; import org.apache.solr.client.solrj.response.FacetField; -import org.apache.solr.client.solrj.response.FacetField.Count; -import org.bson.BSONObject; +import java.util.Arrays; +import java.util.ArrayList; +import com.fangjia.datastore.solr.SolrException; +import java.util.Iterator; import org.bson.BasicBSONObject; import org.bson.types.ObjectId; +import java.util.List; +import com.mongodb.BasicDBObject; +import java.util.Date; +import org.apache.commons.collections.CollectionUtils; +import org.bson.BSONObject; +import com.fangjia.datastore.solr.Index; +import java.util.Collection; +import com.mongodb.DBCollection; +import com.mongodb.gridfs.GridFS; +import java.util.HashMap; +import com.fangjia.datastore.solr.CoreImpl; +import com.fangjia.datastore.utils.MongoUtils; +import com.fangjia.datastore.config.StoreConfig; +import java.util.Map; +import com.fangjia.datastore.solr.Core; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.Set; +import com.mongodb.DBObject; +import java.util.Comparator; import org.slf4j.Logger; -import com.fangjia.datastore.config.StoreConfig; -import com.fangjia.datastore.solr.Core; -import com.fangjia.datastore.solr.CoreImpl; -import com.fangjia.datastore.solr.Index; -import com.fangjia.datastore.solr.SolrException; -import com.fangjia.datastore.solr.SolrSchema; -import com.fangjia.datastore.utils.LoggerFactory; -import com.fangjia.datastore.utils.MongoUtils; -import com.mongodb.BasicDBObject; -import com.mongodb.Bytes; -import com.mongodb.DBCollection; -import com.mongodb.DBObject; -import com.mongodb.WriteConcern; -import com.mongodb.gridfs.GridFS; - -public class DataStoreImpl implements DataStore { - - private final static Logger LOGGER = LoggerFactory.getLogger(DataStoreImpl.class); - private final static Logger PERFORMANCE_LOGGER = LoggerFactory.getPerformanceLogger(DataStoreImpl.class); - private final static String SEAWEED_UPDATE_LOG = "seaweed_update_log"; - - private final static Comparator OBJECT_ID_COMPARATOR = new Comparator() { - - public int compare(DBObject o1, DBObject o2) { - ObjectId id1 = (ObjectId) o1.get("_id"); - ObjectId id2 = (ObjectId) o2.get("_id"); - return id1.compareTo(id2); - } - }; - private final static Set SPECIAL_KEYS = new HashSet(Arrays.asList("_id", "$bulk", SolrSchema.VERTION)); - private final static int MAX_REMOVE_ROWS = 1000; - - private final AtomicBoolean indexing = new AtomicBoolean(false); - - private final Core core; - private final Map attributes; - private final StoreConfig storeConfig; - private Mongo mongo; - private String mongoDbName; - private String mongoCollectionName; - private final FileStore fileStore; - - public DataStoreImpl(StoreConfig storeConfig, Mongo mongo) throws ConfigurationException { - String[] split = MongoUtils.parseFullCollectionName(storeConfig.getName()); - if (null == split) { - throw new IllegalArgumentException("invalid store name: \"" + storeConfig.getName() + "\""); - } - this.storeConfig = storeConfig; - this.mongo = mongo; - mongoDbName = split[0]; - mongoCollectionName = split[1]; - core = new CoreImpl(storeConfig); - attributes = new HashMap(); - - if (storeConfig.isTextFieldToGridFS()) { - fileStore = new FileStoreImpl(new GridFS(mongo.getDB(mongoDbName), mongoCollectionName), storeConfig.getFields()); - } else { - fileStore = null; - } - } - - public void reload() throws ConfigurationException { - this.core.reload(); - } - - public DBCollection getMongoCollection() { - return mongo.getDB(mongoDbName).getCollection(mongoCollectionName); - } - - public DBCollection getUpdateLogMongoCollection() { - return mongo.getDB(mongoDbName).getCollection(SEAWEED_UPDATE_LOG); - } - - public StoreConfig getStoreConfig() { - return storeConfig; - } - - public Mongo getMongo() { - return mongo; - } - - public Object getAttribute(String key) { - return attributes.get(key); - } - - public Object setAttribute(String key, Object attr) { - return attributes.put(key, attr); - } - - public Collection getIndexes() throws DataStoreException { - try { - return core.getIndexes(); - } catch (Exception e) { - throw new DataStoreException("error get indexes", e); - } - } - - public BSONObject reIndex(BSONObject query) throws DataStoreException { - try { - // 如没有查询条件,则全部重建 - if (null == query || CollectionUtils.isEmpty(query.keySet())) { - if (indexing.compareAndSet(false, true)) { - String currentVersion = Core.VERSION_DATE_FORMAT.format(new Date()); // 操作发起的时间戳 - Iterator i = this.getMongoCollection().find().addOption(Bytes.QUERYOPTION_NOTIMEOUT) - .sort(new BasicDBObject("_id", 1)); - core.reIndex(null, i); - // 删除重建之前的索引 - query = new BasicBSONObject(); - BSONObject versionObj = new BasicBSONObject(); - versionObj.put("$lte", currentVersion); - query.put(SolrSchema.VERTION, versionObj); - core.deleteByQuery(query); - return query; - } else { - throw new DataStoreException(storeConfig.getName() + ": the re-index is in progress."); - } - } else { - doReindexByQuery(query); - return null; - } - } catch (Exception e) { - throw new DataStoreException("error re-index", e); - } finally { - indexing.compareAndSet(true, false); - } - } - - private interface Closure { - /** - * 返回false则跳出遍历 - * - * @param input - * @return - * @throws SolrException - * @throws DataStoreException - */ - public boolean execute(List input) throws SolrException, DataStoreException; - } - - /** - * 根据查询条件重建索引 - * - * @param query - * @throws DataStoreException - */ - private void doReindexByQuery(BSONObject query) throws DataStoreException { - String operationName = this.storeConfig.getName() + "_re-index_" + System.currentTimeMillis(); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(operationName + ": query=" + query); - } - - try { - iterateSolrQuery(operationName, query, this.storeConfig.getDataStoreConfig().getServerConfig().getReIndexBatchSize(), - new Closure() { - - public boolean execute(List objIdList) throws SolrException, DataStoreException { - DBObject seletor = new BasicDBObject(); - seletor.put("_id", new BasicDBObject("$in", objIdList)); - Iterator i = getMongoCollection().find(seletor); - core.reIndex(objIdList, i); - return true; - } - - }); - core.deleteByQuery(query); - } catch (SolrException e) { - throw new DataStoreException(operationName + ": ", e); - } - - } - - /** - * 根据query遍历。
- * 这个方法只适用于reIndex, update, remove等会重建索引的操作, 不能简单的用来遍历 - * - * @param operationName - * @param query - * @param batchSize - * @param closure - * @throws SolrException - * @throws DataStoreException - */ - private void iterateSolrQuery(String operationName, BSONObject query, int batchSize, Closure closure) throws SolrException, - DataStoreException { - if (null == query.get(SolrSchema.VERTION)) { - String currentVersion = Core.VERSION_DATE_FORMAT.format(new Date()); // 操作发起的时间戳 - BSONObject versionObj = new BasicBSONObject(); - versionObj.put("$lte", currentVersion); - query.put(SolrSchema.VERTION, versionObj); // 只遍历时间戳之前的数据 - } - - long startTime = System.currentTimeMillis(); - int currentPage = 0; // 当前遍历页面 - long c = 0; - long lastC = c; - int round = 0; - long roundIndexCount = 0; - QueryResponse response = null; - int lastNumFound = -1; - while (true) { - response = findInSolr(query, null, null, currentPage * batchSize, batchSize, true, true); - if (currentPage == 0) {// 每轮第一次迭代 - if (lastNumFound == -1) { - // 第一轮 - lastNumFound = response.getNumFound(); - } else if (response.getNumFound() > 0 && lastNumFound == response.getNumFound()) { - long sleep = 20000; - if (LOGGER.isInfoEnabled()) { - LOGGER.info("{}[{}]: wait solr commit, sleep {}s.", new Object[] { operationName, round, sleep / 1000 }); - } - try { - Thread.sleep(sleep); - } catch (InterruptedException e) { - throw new DataStoreException(operationName + ": ", e); - } - response = findInSolr(query, null, null, 0, 1, true, true); - if (response.getNumFound() > 0 && lastNumFound == response.getNumFound()) { - // TODO: 可能需要删除solr中有但mongodb中没有的索引数据 - // core.deleteByQuery(query); - break; - } - lastNumFound = response.getNumFound(); - continue; - } else { - lastNumFound = response.getNumFound(); - } - } - - int returnedSize = response.getNumReturned(); - List idList = response.getIdList(); - // 有可能为空 - if (null != idList && idList.size() > 0) { - // 执行重建 - if (!closure.execute(idList)) { - // 返回false则不再遍历 - break; - } - } - c += returnedSize; - roundIndexCount += returnedSize; - if (c - lastC >= batchSize && LOGGER.isInfoEnabled()) { - LOGGER.info("{}[{}]: {}/{}, {}/s", new Object[] { operationName, round, c, response.getNumFound(), - (int) (c / ((System.currentTimeMillis() - startTime) / 1000.0)) }); - lastC = c; - } - - // 因为重建后的索引已不在query条件内,所以query条件的整个集合大小随着遍历的往前进行,一直在变动 - // 而分页是顺序向前的,所以一次执行,不能保证全部遍历到,所以要多次执行 - if (currentPage == 0 && returnedSize >= lastNumFound) { - // 第一页数量就小于BATCH_SIZE,则不用往下遍历 - break; - } else if (returnedSize == 0 || roundIndexCount >= lastNumFound) { - // 本轮重建数量大于总数量,从头再来过 - currentPage = 0; - round++; - roundIndexCount = 0; - } else { - // 当前页数量跟BATCH_SIZE一致,向前翻页 - currentPage++; - } - } - if (LOGGER.isInfoEnabled()) { - LOGGER.info("{}[{}] finished, total time: {}s, count: {}/{}", new Object[] { operationName, round, - (System.currentTimeMillis() - startTime) / 1000, roundIndexCount, c }); - } - } - - public QueryResponse find(BSONObject query, BSONObject fields, BSONObject sort, BSONObject facet, int skip, int limit, - boolean bulk, boolean fromMaster) throws DataStoreException { - checkFields(fields); - try { - QueryResponse response = findInSolr(query, sort, facet, skip, limit, fromMaster, bulk); - List docList = find(response, fields); - int numReturned = docList.size() - 1; - if (numReturned > response.getNumFound()) { - response.setNumFound(numReturned); - docList.get(0).put("$numFound", numReturned); - } - if (numReturned != response.getNumReturned()) { - response.setNumReturned(numReturned); - } - response.setDocList(docList); - return response; - } catch (SolrException e) { - throw new DataStoreException("error execute find{query=" + query + ", fields=" + fields + ", sort=" + sort - + ", facet=" + facet + ", skip=" + skip + ", limit=" + limit + "}: ", e); - } - } - - private QueryResponse findInSolr(BSONObject query, BSONObject sort, BSONObject facet, int skip, int limit, - boolean fromMaster, boolean bulk) throws SolrException, DataStoreException { - QueryResponse response = null; - List idList = null; - if (null == facet || facet.keySet().size() == 0) { - idList = getObjectIdsInQuery(query); - } - if (null != idList && idList.size() > 0) { - response = new QueryResponse(); - response.setIdList(idList); - response.setNumReturned(idList.size()); - } else { - response = core.find(query, sort, facet, skip, limit, fromMaster, bulk); - } - return response; - } - - @SuppressWarnings("unchecked") - private List getObjectIdsInQuery(BSONObject query) throws DataStoreException { - List ret = null; - Object value = query.get("_id"); - if (null == value) { - ret = null; - } else if (value instanceof BSONObject) { - List ids = (List) ((BSONObject) value).get("$in"); - if (null != ids && ids.size() > 0) { - if (ids.get(0) instanceof ObjectId) { - ret = (List) ids; - } else { - ret = new ArrayList(ids.size()); - for (Object id : ids) { - ret.add(new ObjectId(String.valueOf(id))); - } - } - } - } else if (value instanceof ObjectId) { - ret = Arrays.asList((ObjectId) value); - } else if (value instanceof String) { - ret = Arrays.asList(new ObjectId((String) value)); - } - if (ret != null && containsNoneIdQuery(query)) { - throw new DataStoreException("id query with other query:" + query); - } - return ret; - } - - private boolean containsNoneIdQuery(BSONObject query) { - if (null == query) { - return false; - } - for (String key : query.keySet()) { - if (SPECIAL_KEYS.contains(key)) { - continue; - } - return true; - } - return false; - } - - public int count(BSONObject query, boolean fromMaster) throws DataStoreException { - try { - QueryResponse response = core.find(query, null, null, 0, 0, fromMaster, false); - List docList = find(response, null); - response.setDocList(docList); - return response.getNumFound(); - } catch (SolrException e) { - throw new DataStoreException("error execute count{query=" + query + "}: ", e); - } - } - - private List find(QueryResponse response, BSONObject fields) throws DataStoreException { - List idList = response.getIdList(); - List newDocList = new ArrayList(null == idList ? 1 : (idList.size() + 1)); - - DBObject meta = new BasicDBObject("$numFound", response.getNumFound()); - List facetFieldList = response.getFacetFieldList(); - if (null != facetFieldList && facetFieldList.size() > 0) { - List facetList = new ArrayList(facetFieldList.size()); - for (FacetField facetField : facetFieldList) { - DBObject facet = new BasicDBObject(); - facet.put("name", facetField.getName()); - facet.put("end", facetField.getEnd()); - facet.put("gap", facetField.getGap()); - facet.put("valueCount", facetField.getValueCount()); - List values = facetField.getValues(); - if (null != values && values.size() > 0) { - List vs = new ArrayList(values.size()); - for (Count c : values) { - DBObject v = new BasicDBObject(); - v.put("name", c.getName()); - v.put("count", c.getCount()); - vs.add(v); - } - facet.put("values", vs); - } - facetList.add(facet); - } - meta.put("$facet", facetList); - } - newDocList.add(meta); - - if (null != idList && idList.size() > 0) { - /** - * 此处是为了性能考虑 - * 当查询只需要_id的时候,直接返回从solr中检索出的idList即可 - * 但是由于存在solr中存在的_id在mongodb中不存在,所以把这段代码注释了,从而一定程度降低了只查询_id请求的性能,保证了数据一致性 - */ -// if (null != fields && fields.containsField("_id") && fields.keySet().size() == 1) { -// for (ObjectId id : idList) { -// newDocList.add(new BasicDBObject("_id", id)); -// } -// } else { - DBObject query = new BasicDBObject(); - if (idList.size() == 1) { - query.put("_id", idList.get(0)); - } else { - query.put("_id", new BasicDBObject("$in", idList)); - } - DBObject fieldsQuery = null; - if (null != fields) { - fieldsQuery = new BasicDBObject(); - fieldsQuery.putAll(fields); - } - - long start = System.currentTimeMillis(); - List docList = this.getMongoCollection().find(query, fieldsQuery).toArray(); - Collections.sort(docList, OBJECT_ID_COMPARATOR); - long time = System.currentTimeMillis() - start; - if (time > 100) { - if (PERFORMANCE_LOGGER.isWarnEnabled()) { - PERFORMANCE_LOGGER.warn(this.storeConfig.getName() + ": query mongodb, id size: " + idList.size() - + ", time: " + time); - } - } - if (null != docList && !docList.isEmpty()) { - for (Iterator i = idList.iterator(); i.hasNext();) { - int index = binarySearch(docList, i.next()); - if (index >= 0) { - newDocList.add(docList.get(index)); - } - } - } - - if (storeConfig.isTextFieldToGridFS()) { - newDocList = fileStore.find(newDocList, fields); - } -// } - } - return newDocList; - } - - private static int binarySearch(List list, ObjectId id) { - int low = 0; - int high = list.size() - 1; - - while (low <= high) { - int mid = (low + high) >>> 1; - ObjectId midVal = (ObjectId) list.get(mid).get("_id"); - int cmp = midVal.compareTo(id); - - if (cmp < 0) - low = mid + 1; - else if (cmp > 0) - high = mid - 1; - else - return mid; // key found - } - return -(low + 1); // key not found - } - - private boolean isOpSupported(String op) { - return "$inc".equals(op) || "$set".equals(op) || "$unset".equals(op); - } - - private void checkUpdateFields(BSONObject data) throws DataStoreException { - for (String op : data.keySet()) { - if (!isOpSupported(op)) { - throw new DataStoreException("modifier operation " + op + " is not supported!"); - } - BSONObject update = (BSONObject) data.get(op); - if (null != update) { - for (String field : update.keySet()) { - if (!storeConfig.containsField(field)) { - throw new DataStoreException("unknown field: " + field + " !"); - } - if (!storeConfig.isFieldValueLegal(field, update.get(field))) { - throw new DataStoreException("invalid value " + update.get(field) + " of field '" + field + "'!"); - } - } - } - } - } - - private void checkFields(BSONObject fields) throws DataStoreException { - if (null != fields) { - for (String field : fields.keySet()) { - if (!storeConfig.containsField(field)) { - throw new DataStoreException("unknown field: " + field + " !"); - } - } - } - } - - public void update(final BSONObject query, final BSONObject data,final String ip) throws DataStoreException { - checkUpdateFields(data); - String operationName = this.storeConfig.getName() + "_update_" + System.currentTimeMillis(); - - if (LOGGER.isInfoEnabled()) { - LOGGER.info(operationName + ": query=" + query); - } - - try { - iterateSolrQuery(operationName, query, 1000, new Closure() { - - public boolean execute(List objIdList) throws SolrException, DataStoreException { - final DBObject update = new BasicDBObject(); - final String ipAddress = ip; - update.putAll(data); - - // 更新大字段关联文件 - if (storeConfig.isTextFieldToGridFS()) { - fileStore.update(objIdList, update); - - // 把要更新的大字段数据,置为null,不需要把数据插入主表 - BSONObject realData = (BSONObject) update.get("$set"); - if (null != realData) { - Set removeField = new HashSet(); - for (String field : realData.keySet()) { - if (storeConfig.isBigField(field)) { - // realData.removeField(field); - removeField.add(field); - } - } - realData.keySet().removeAll(removeField); - } - } - - DBObject seletor = new BasicDBObject(); - if (objIdList.size() == 1) { - seletor.put("_id", objIdList.get(0)); - } else { - seletor.put("_id", new BasicDBObject("$in", objIdList)); - } - getMongoCollection().update(seletor, update, false, true, WriteConcern.NORMAL); - //修改小区状态为1时记录状态 - if(update.get("$set")!=null){ - BasicBSONObject updateData = new BasicBSONObject(); - updateData = (BasicBSONObject) update.get("$set"); - String status = updateData.getString("status"); - if(status!=null && "1".equals(status)){ - DBObject saveObj = new BasicDBObject(); - saveObj.put("ip", ipAddress); - saveObj.put("districtIp", objIdList.get(0).toString()); - - saveObj.put("type", "remove"); - saveObj.put("c_date", new Date()); - getUpdateLogMongoCollection().save(saveObj); - } +public class DataStoreImpl implements DataStore +{ + private static final Logger LOGGER; + private static final Logger PERFORMANCE_LOGGER; + private static final String SEAWEED_UPDATE_LOG = "seaweed_update_log"; + private static final Comparator OBJECT_ID_COMPARATOR; + private static final Set SPECIAL_KEYS; + private static final int MAX_REMOVE_ROWS = 10; + private final AtomicBoolean indexing; + private final Core core; + private final Map attributes; + private final StoreConfig storeConfig; + private Mongo mongo; + private String mongoDbName; + private String mongoCollectionName; + private final FileStore fileStore; + + public DataStoreImpl(final StoreConfig storeConfig, final Mongo mongo) throws ConfigurationException { + this.indexing = new AtomicBoolean(false); + final String[] split = MongoUtils.parseFullCollectionName(storeConfig.getName()); + if (null == split) { + throw new IllegalArgumentException("invalid store name: \"" + storeConfig.getName() + "\""); + } + this.storeConfig = storeConfig; + this.mongo = mongo; + this.mongoDbName = split[0]; + this.mongoCollectionName = split[1]; + this.core = new CoreImpl(storeConfig); + this.attributes = new HashMap(); + if (storeConfig.isTextFieldToGridFS()) { + this.fileStore = new FileStoreImpl(new GridFS(mongo.getDB(this.mongoDbName), this.mongoCollectionName), storeConfig.getFields()); + } + else { + this.fileStore = null; + } + } + + @Override + public void reload() throws ConfigurationException { + this.core.reload(); + } + + public DBCollection getMongoCollection() { + return this.mongo.getDB(this.mongoDbName).getCollection(this.mongoCollectionName); + } + + public DBCollection getUpdateLogMongoCollection() { + return this.mongo.getDB(this.mongoDbName).getCollection("seaweed_update_log"); + } + + @Override + public StoreConfig getStoreConfig() { + return this.storeConfig; + } + + @Override + public Mongo getMongo() { + return this.mongo; + } + + @Override + public Object getAttribute(final String key) { + return this.attributes.get(key); + } + + @Override + public Object setAttribute(final String key, final Object attr) { + return this.attributes.put(key, attr); + } + + @Override + public Collection getIndexes() throws DataStoreException { + try { + return this.core.getIndexes(); + } + catch (Exception e) { + throw new DataStoreException("error get indexes", e); + } + } + + @Override + public BSONObject reIndex(BSONObject query) throws DataStoreException { + try { + if (null != query && !CollectionUtils.isEmpty((Collection)query.keySet())) { + this.doReindexByQuery(query); + return null; + } + if (this.indexing.compareAndSet(false, true)) { + final String currentVersion = Core.VERSION_DATE_FORMAT.format(new Date()); + final Iterator i = (Iterator)this.getMongoCollection().find().addOption(16).sort((DBObject)new BasicDBObject("_id", (Object)1)); + this.core.reIndex(null, i); + query = (BSONObject)new BasicBSONObject(); + final BSONObject versionObj = (BSONObject)new BasicBSONObject(); + versionObj.put("$lte", (Object)currentVersion); + query.put("_version", (Object)versionObj); + this.core.deleteByQuery(query); + return query; + } + throw new DataStoreException(this.storeConfig.getName() + ": the re-index is in progress."); + } + catch (Exception e) { + throw new DataStoreException("error re-index", e); + } + finally { + this.indexing.compareAndSet(true, false); + } + } + + private void doReindexByQuery(final BSONObject query) throws DataStoreException { + final String operationName = this.storeConfig.getName() + "_re-index_" + System.currentTimeMillis(); + if (DataStoreImpl.LOGGER.isInfoEnabled()) { + DataStoreImpl.LOGGER.info(operationName + ": query=" + query); + } + try { + this.iterateSolrQuery(operationName, query, this.storeConfig.getDataStoreConfig().getServerConfig().getReIndexBatchSize(), new Closure() { + @Override + public boolean execute(final List objIdList) throws SolrException, DataStoreException { + final DBObject seletor = (DBObject)new BasicDBObject(); + seletor.put("_id", (Object)new BasicDBObject("$in", (Object)objIdList)); + final Iterator i = (Iterator)DataStoreImpl.this.getMongoCollection().find(seletor); + DataStoreImpl.this.core.reIndex(objIdList, i); + return true; + } + }); + this.core.deleteByQuery(query); + } + catch (SolrException e) { + throw new DataStoreException(operationName + ": ", e); + } + } + + private void iterateSolrQuery(final String operationName, final BSONObject query, final int batchSize, final Closure closure) throws SolrException, DataStoreException { + if (null == query.get("_version")) { + final String currentVersion = Core.VERSION_DATE_FORMAT.format(new Date()); + final BSONObject versionObj = new BasicBSONObject(); + versionObj.put("$lte", currentVersion); + query.put("_version", versionObj); + } + final long startTime = System.currentTimeMillis(); + int currentPage = 0; + long lastC; + long c = lastC = 0L; + int round = 0; + long roundIndexCount = 0L; + QueryResponse response = null; + int lastNumFound = -1; + while (true) { + response = this.findInSolr(query, null, null, currentPage * batchSize, batchSize, true, true); + if (currentPage == 0) { + if (lastNumFound == -1) { + lastNumFound = response.getNumFound(); + } + else if (response.getNumFound() > 0 && lastNumFound == response.getNumFound()) { + final long sleep = 20000L; + if (DataStoreImpl.LOGGER.isInfoEnabled()) { + DataStoreImpl.LOGGER.info("{}[{}]: wait solr commit, sleep {}s.", new Object[] { operationName, round, sleep / 1000L }); } - // 重建索引 - core.save(getMongoCollection().find(seletor).addOption(Bytes.QUERYOPTION_NOTIMEOUT)); - return true; - } - - }); - } catch (SolrException e) { - throw new DataStoreException(operationName + ": ", e); - } - } - - public void remove(final BSONObject query) throws DataStoreException { - - String operationName = this.storeConfig.getName() + "_remove_" + System.currentTimeMillis(); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(operationName + ": query=" + query); - } - int count = this.count(query, true); - if (count > MAX_REMOVE_ROWS && !query.containsField("$remove")) { - throw new DataStoreException("can not remove more than 1000 rows at single time, actual rows: " + count); - } - try { - List removeIdList = getObjectIdsInQuery(query); - if(removeIdList != null) { - doExecuteDelete(removeIdList); - }else { - iterateSolrQuery(operationName, query, 1000, new Closure() { - public boolean execute(List objIdList) throws SolrException, DataStoreException { - return doExecuteDelete(objIdList); - } - }); - } - } catch (SolrException e) { - throw new DataStoreException(operationName + ": ", e); - } - } - - private boolean doExecuteDelete(List objIdList) throws SolrException, DataStoreException { - DBObject seletor = new BasicDBObject(); - if (objIdList.size() == 1) { - seletor.put("_id", objIdList.get(0)); - } else { - seletor.put("_id", new BasicDBObject("$in", objIdList)); - } - getMongoCollection().remove(seletor, WriteConcern.NORMAL); - - // 删除大字段的关联文件 - if (storeConfig.isTextFieldToGridFS()) { - fileStore.remove(objIdList); - } - - core.deleteByQuery(seletor); - return true; - } - - public void save(List docList) throws DataStoreException { - if (null == docList || docList.isEmpty()) { - return; - } - if (storeConfig.isTextFieldToGridFS()) { - List insertList = new ArrayList(docList.size()); - List fileList = new ArrayList(docList.size()); - boolean hasBigField = false; - for (BSONObject bson : docList) { - DBObject fileObj = new BasicDBObject(); - for (String field : bson.keySet()) { - if (!storeConfig.containsField(field)) { - throw new DataStoreException("unknown field: " + field + " !"); - } - if (!storeConfig.isFieldValueLegal(field, bson.get(field))) { - throw new DataStoreException("invalid value: " + bson.get(field) + " of field '" + field + "'!"); - } - - // 若是大字段,将内容保存至fileObj后存入fileList - if (storeConfig.isBigField(field)) { - hasBigField = true; - - fileObj.put(field, bson.get(field)); - - // 主表中不保存该字段内容 - bson.put(field, null); - } - } - fileList.add(fileObj); - - DBObject dbObj = new BasicDBObject(); - dbObj.putAll(bson); - insertList.add(dbObj); - this.getMongoCollection().save(dbObj, WriteConcern.NORMAL); - } - - // 保存大字段 - if (hasBigField) - fileStore.save(fileList, insertList); - } else { - for (BSONObject bson : docList) { - for (String field : bson.keySet()) { - if (!storeConfig.containsField(field)) { - throw new DataStoreException("unknown field: " + field + " !"); - } - if (!storeConfig.isFieldValueLegal(field, bson.get(field))) { - throw new DataStoreException("invalid value: " + bson.get(field) + " of field '" + field + "'!"); - } - } - DBObject dbObj = new BasicDBObject(); - dbObj.putAll(bson); - this.getMongoCollection().save(dbObj, WriteConcern.NORMAL); - } - } - - try { - core.save(docList.iterator()); - } catch (SolrException e) { - throw new DataStoreException("error execute save{docList=" + docList + "}: ", e); - } - } - - public void ensureIndex(Collection indexes) throws DataStoreException { - if (null == indexes || indexes.isEmpty()) { - return; - } - try { - Iterator i = this.getMongoCollection().find().addOption(Bytes.QUERYOPTION_NOTIMEOUT); - core.ensureIndex(indexes, i); - } catch (SolrException e) { - throw new DataStoreException("error execute ensureIndex{indexes=" + indexes + "}:", e); - } - } - + try { + Thread.sleep(sleep); + } + catch (InterruptedException e) { + throw new DataStoreException(operationName + ": ", e); + } + response = this.findInSolr(query, null, null, 0, 1, true, true); + if (response.getNumFound() > 0 && lastNumFound == response.getNumFound()) { + break; + } + lastNumFound = response.getNumFound(); + continue; + } + else { + lastNumFound = response.getNumFound(); + } + } + final int returnedSize = response.getNumReturned(); + final List idList = response.getIdList(); + if (null != idList && idList.size() > 0 && !closure.execute(idList)) { + break; + } + c += returnedSize; + roundIndexCount += returnedSize; + if (c - lastC >= batchSize && DataStoreImpl.LOGGER.isInfoEnabled()) { + DataStoreImpl.LOGGER.info("{}[{}]: {}/{}, {}/s", new Object[] { operationName, round, c, response.getNumFound(), (int)(c / ((System.currentTimeMillis() - startTime) / 1000.0)) }); + lastC = c; + } + if (currentPage == 0 && returnedSize >= lastNumFound) { + break; + } + if (returnedSize == 0 || roundIndexCount >= lastNumFound) { + currentPage = 0; + ++round; + roundIndexCount = 0L; + } + else { + ++currentPage; + } + } + if (DataStoreImpl.LOGGER.isInfoEnabled()) { + DataStoreImpl.LOGGER.info("{}[{}] finished, total time: {}s, count: {}/{}", new Object[] { operationName, round, (System.currentTimeMillis() - startTime) / 1000L, roundIndexCount, c }); + } + } + + @Override + public QueryResponse find(final BSONObject query, final BSONObject fields, final BSONObject sort, final BSONObject facet, final int skip, final int limit, final boolean bulk, final boolean fromMaster) throws DataStoreException { + this.checkFields(fields); + try { + final QueryResponse response = this.findInSolr(query, sort, facet, skip, limit, fromMaster, bulk); + final List docList = this.find(response, fields); + final int numReturned = docList.size() - 1; + if (numReturned > response.getNumFound()) { + response.setNumFound(numReturned); + docList.get(0).put("$numFound", (Object)numReturned); + } + if (numReturned != response.getNumReturned()) { + response.setNumReturned(numReturned); + } + response.setDocList(docList); + return response; + } + catch (SolrException e) { + throw new DataStoreException("error execute find{query=" + query + ", fields=" + fields + ", sort=" + sort + ", facet=" + facet + ", skip=" + skip + ", limit=" + limit + "}: ", e); + } + } + + private QueryResponse findInSolr(final BSONObject query, final BSONObject sort, final BSONObject facet, final int skip, final int limit, final boolean fromMaster, final boolean bulk) throws SolrException, DataStoreException { + QueryResponse response = null; + List idList = null; + if (null == facet || facet.keySet().size() == 0) { + idList = this.getObjectIdsInQuery(query); + } + if (null != idList && idList.size() > 0) { + response = new QueryResponse(); + response.setIdList(idList); + response.setNumReturned(idList.size()); + } + else { + response = this.core.find(query, sort, facet, skip, limit, fromMaster, bulk); + } + return response; + } + + private List getObjectIdsInQuery(final BSONObject query) throws DataStoreException { + List ret = null; + final Object value = query.get("_id"); + if (null == value) { + ret = null; + } + else if (value instanceof BSONObject) { + final List ids = (List)((BSONObject)value).get("$in"); + if (null != ids && ids.size() > 0) { + if (ids.get(0) instanceof ObjectId) { + ret = (List)ids; + } + else { + ret = new ArrayList(ids.size()); + for (final Object id : ids) { + ret.add(new ObjectId(String.valueOf(id))); + } + } + } + } + else if (value instanceof ObjectId) { + ret = Arrays.asList((ObjectId)value); + } + else if (value instanceof String) { + ret = Arrays.asList(new ObjectId((String)value)); + } + if (ret != null && this.containsNoneIdQuery(query)) { + throw new DataStoreException("id query with other query:" + query); + } + return ret; + } + + private boolean containsNoneIdQuery(final BSONObject query) { + if (null == query) { + return false; + } + for (final String key : query.keySet()) { + if (DataStoreImpl.SPECIAL_KEYS.contains(key)) { + continue; + } + return true; + } + return false; + } + + @Override + public int count(final BSONObject query, final boolean fromMaster) throws DataStoreException { + try { + final QueryResponse response = this.core.find(query, null, null, 0, 0, fromMaster, false); + final List docList = this.find(response, null); + response.setDocList(docList); + return response.getNumFound(); + } + catch (SolrException e) { + throw new DataStoreException("error execute count{query=" + query + "}: ", e); + } + } + + private List find(final QueryResponse response, final BSONObject fields) throws DataStoreException { + final List idList = response.getIdList(); + List newDocList = new ArrayList((null == idList) ? 1 : (idList.size() + 1)); + final DBObject meta = new BasicDBObject("$numFound", response.getNumFound()); + final List facetFieldList = response.getFacetFieldList(); + if (null != facetFieldList && facetFieldList.size() > 0) { + final List facetList = new ArrayList(facetFieldList.size()); + for (final FacetField facetField : facetFieldList) { + final DBObject facet = new BasicDBObject(); + facet.put("name", facetField.getName()); + facet.put("end", facetField.getEnd()); + facet.put("gap", facetField.getGap()); + facet.put("valueCount", facetField.getValueCount()); + final List values = facetField.getValues(); + if (null != values && values.size() > 0) { + final List vs = new ArrayList(values.size()); + for (final FacetField.Count c : values) { + final DBObject v = new BasicDBObject(); + v.put("name", c.getName()); + v.put("count", c.getCount()); + vs.add(v); + } + facet.put("values", vs); + } + facetList.add(facet); + } + meta.put("$facet", facetList); + } + newDocList.add(meta); + if (null != idList && idList.size() > 0) { + final DBObject query = new BasicDBObject(); + if (idList.size() == 1) { + query.put("_id", idList.get(0)); + } + else { + query.put("_id", new BasicDBObject("$in", idList)); + } + DBObject fieldsQuery = null; + if (null != fields) { + fieldsQuery = new BasicDBObject(); + fieldsQuery.putAll(fields); + } + final long start = System.currentTimeMillis(); + final List docList = this.getMongoCollection().find(query, fieldsQuery).toArray(); + Collections.sort(docList, DataStoreImpl.OBJECT_ID_COMPARATOR); + final long time = System.currentTimeMillis() - start; + if (time > 100L && DataStoreImpl.PERFORMANCE_LOGGER.isWarnEnabled()) { + DataStoreImpl.PERFORMANCE_LOGGER.warn(this.storeConfig.getName() + ": query mongodb, id size: " + idList.size() + ", time: " + time); + } + if (null != docList && !docList.isEmpty()) { + final Iterator i = idList.iterator(); + while (i.hasNext()) { + final int index = binarySearch(docList, i.next()); + if (index >= 0) { + newDocList.add(docList.get(index)); + } + } + } + if (this.storeConfig.isTextFieldToGridFS()) { + newDocList = this.fileStore.find(newDocList, fields); + } + } + return newDocList; + } + + private static int binarySearch(final List list, final ObjectId id) { + int low = 0; + int high = list.size() - 1; + while (low <= high) { + final int mid = low + high >>> 1; + final ObjectId midVal = (ObjectId)list.get(mid).get("_id"); + final int cmp = midVal.compareTo(id); + if (cmp < 0) { + low = mid + 1; + } + else { + if (cmp <= 0) { + return mid; + } + high = mid - 1; + } + } + return -(low + 1); + } + + private boolean isOpSupported(final String op) { + return "$inc".equals(op) || "$set".equals(op) || "$unset".equals(op); + } + + private void checkUpdateFields(final BSONObject data) throws DataStoreException { + for (final String op : data.keySet()) { + if (!this.isOpSupported(op)) { + throw new DataStoreException("modifier operation " + op + " is not supported!"); + } + final BSONObject update = (BSONObject)data.get(op); + if (null == update) { + continue; + } + for (final String field : update.keySet()) { + if (!this.storeConfig.containsField(field)) { + throw new DataStoreException("unknown field: " + field + " !"); + } + if (!this.storeConfig.isFieldValueLegal(field, update.get(field))) { + throw new DataStoreException("invalid value " + update.get(field) + " of field '" + field + "'!"); + } + } + } + } + + private void checkFields(final BSONObject fields) throws DataStoreException { + if (null != fields) { + for (final String field : fields.keySet()) { + if (!this.storeConfig.containsField(field)) { + throw new DataStoreException("unknown field: " + field + " !"); + } + } + } + } + + @Override + public int update(final BSONObject query, final BSONObject data) throws DataStoreException { + final AtomicInteger count = new AtomicInteger(0); + this.checkUpdateFields(data); + final String operationName = this.storeConfig.getName() + "_update_" + System.currentTimeMillis(); + if (DataStoreImpl.LOGGER.isInfoEnabled()) { + DataStoreImpl.LOGGER.info(operationName + ": query=" + query); + } + try { + this.iterateSolrQuery(operationName, query, 1000, new Closure() { + @Override + public boolean execute(final List objIdList) throws SolrException, DataStoreException { + final DBObject update = (DBObject)new BasicDBObject(); + update.putAll(data); + if (DataStoreImpl.this.storeConfig.isTextFieldToGridFS()) { + DataStoreImpl.this.fileStore.update(objIdList, (BSONObject)update); + final BSONObject realData = (BSONObject)update.get("$set"); + if (null != realData) { + final Set removeField = new HashSet(); + for (final String field : realData.keySet()) { + if (DataStoreImpl.this.storeConfig.isBigField(field)) { + removeField.add(field); + } + } + realData.keySet().removeAll(removeField); + } + } + final DBObject seletor = (DBObject)new BasicDBObject(); + if (objIdList.size() == 1) { + seletor.put("_id", (Object)objIdList.get(0)); + } + else { + seletor.put("_id", (Object)new BasicDBObject("$in", (Object)objIdList)); + } + final WriteResult writeResult = DataStoreImpl.this.getMongoCollection().update(seletor, update, false, true, WriteConcern.ACKNOWLEDGED); + count.set(writeResult.getN()); + DataStoreImpl.this.core.save((Iterator)DataStoreImpl.this.getMongoCollection().find(seletor).addOption(16)); + return true; + } + }); + return count.get(); + } + catch (SolrException e) { + throw new DataStoreException(operationName + ": ", e); + } + } + + @Override + public void remove(final BSONObject query) throws DataStoreException { + final String operationName = this.storeConfig.getName() + "_remove_" + System.currentTimeMillis(); + if (DataStoreImpl.LOGGER.isInfoEnabled()) { + DataStoreImpl.LOGGER.info(operationName + ": query=" + query); + } + final int count = this.count(query, true); + if (count > 10 && !query.containsField("$remove")) { + throw new DataStoreException("can not remove more than 10 rows at single time, actual rows: " + count); + } + try { + final List removeIdList = this.getObjectIdsInQuery(query); + if (removeIdList != null) { + this.doExecuteDelete(removeIdList); + } + else { + this.iterateSolrQuery(operationName, query, 1000, new Closure() { + @Override + public boolean execute(final List objIdList) throws SolrException, DataStoreException { + return DataStoreImpl.this.doExecuteDelete(objIdList); + } + }); + } + } + catch (SolrException e) { + throw new DataStoreException(operationName + ": ", e); + } + } + + private boolean doExecuteDelete(final List objIdList) throws SolrException, DataStoreException { + final DBObject seletor = (DBObject)new BasicDBObject(); + if (objIdList.size() == 1) { + seletor.put("_id", (Object)objIdList.get(0)); + } + else { + seletor.put("_id", (Object)new BasicDBObject("$in", (Object)objIdList)); + } + this.getMongoCollection().remove(seletor, WriteConcern.NORMAL); + if (this.storeConfig.isTextFieldToGridFS()) { + this.fileStore.remove(objIdList); + } + this.core.deleteByQuery((BSONObject)seletor); + return true; + } + + @Override + public void save(final List docList) throws DataStoreException { + if (null == docList || docList.isEmpty()) { + return; + } + if (this.storeConfig.isTextFieldToGridFS()) { + final List insertList = new ArrayList(docList.size()); + final List fileList = new ArrayList(docList.size()); + boolean hasBigField = false; + for (final BSONObject bson : docList) { + final DBObject fileObj = (DBObject)new BasicDBObject(); + for (final String field : bson.keySet()) { + if (!this.storeConfig.containsField(field)) { + throw new DataStoreException("unknown field: " + field + " !"); + } + if (!this.storeConfig.isFieldValueLegal(field, bson.get(field))) { + throw new DataStoreException("invalid value: " + bson.get(field) + " of field '" + field + "'!"); + } + if (!this.storeConfig.isBigField(field)) { + continue; + } + hasBigField = true; + fileObj.put(field, bson.get(field)); + bson.put(field, (Object)null); + } + fileList.add(fileObj); + final DBObject dbObj = (DBObject)new BasicDBObject(); + dbObj.putAll(bson); + insertList.add(dbObj); + this.getMongoCollection().save(dbObj, WriteConcern.NORMAL); + } + if (hasBigField) { + this.fileStore.save(fileList, insertList); + } + } + else { + for (final BSONObject bson2 : docList) { + for (final String field2 : bson2.keySet()) { + if (!this.storeConfig.containsField(field2)) { + throw new DataStoreException("unknown field: " + field2 + " !"); + } + if (!this.storeConfig.isFieldValueLegal(field2, bson2.get(field2))) { + throw new DataStoreException("invalid value: " + bson2.get(field2) + " of field '" + field2 + "'!"); + } + } + final DBObject dbObj2 = (DBObject)new BasicDBObject(); + dbObj2.putAll(bson2); + this.getMongoCollection().save(dbObj2, WriteConcern.NORMAL); + } + } + try { + this.core.save(docList.iterator()); + } + catch (SolrException e) { + throw new DataStoreException("error execute save{docList=" + docList + "}: ", e); + } + } + + @Override + public void ensureIndex(final Collection indexes) throws DataStoreException { + if (null == indexes || indexes.isEmpty()) { + return; + } + try { + final Iterator i = (Iterator)this.getMongoCollection().find().addOption(16); + this.core.ensureIndex(indexes, i); + } + catch (SolrException e) { + throw new DataStoreException("error execute ensureIndex{indexes=" + indexes + "}:", e); + } + } + + static { + LOGGER = LoggerFactory.getLogger(DataStoreImpl.class); + PERFORMANCE_LOGGER = LoggerFactory.getPerformanceLogger(DataStoreImpl.class); + OBJECT_ID_COMPARATOR = new Comparator() { + @Override + public int compare(final DBObject o1, final DBObject o2) { + final ObjectId id1 = (ObjectId)o1.get("_id"); + final ObjectId id2 = (ObjectId)o2.get("_id"); + return id1.compareTo(id2); + } + }; + SPECIAL_KEYS = new HashSet(Arrays.asList("_id", "$bulk", "_version")); + } + + private interface Closure + { + boolean execute(final List p0) throws SolrException, DataStoreException; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreServer.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreServer.java index ccab8e57f..959b16cfc 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreServer.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreServer.java @@ -1,193 +1,178 @@ package com.fangjia.datastore; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import org.apache.mina.core.buffer.IoBuffer; -import org.apache.mina.core.service.IoHandlerAdapter; +import org.slf4j.LoggerFactory; import org.apache.mina.core.service.IoService; -import org.apache.mina.core.service.IoServiceListener; -import org.apache.mina.core.service.SimpleIoProcessorPool; -import org.apache.mina.core.session.IdleStatus; -import org.apache.mina.core.session.IoSession; -import org.apache.mina.filter.codec.ProtocolCodecException; -import org.apache.mina.filter.codec.ProtocolCodecFilter; -import org.apache.mina.filter.logging.LogLevel; -import org.apache.mina.filter.logging.LoggingFilter; -import org.apache.mina.transport.socket.SocketSessionConfig; -import org.apache.mina.transport.socket.nio.NioProcessor; -import org.apache.mina.transport.socket.nio.NioSession; -import org.apache.mina.transport.socket.nio.NioSocketAcceptor; +import java.util.Arrays; import org.bson.BSONObject; import org.bson.BasicBSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fangjia.datastore.config.DataStoreConfig; -import com.fangjia.datastore.handler.DeleteMessageHandler; -import com.fangjia.datastore.handler.GetMoreMessageHandler; -import com.fangjia.datastore.handler.InsertMessageHandler; -import com.fangjia.datastore.handler.KillCursorsMessageHandler; -import com.fangjia.datastore.handler.MessageHandler; -import com.fangjia.datastore.handler.QueryMessageHandler; +import com.fangjia.datastore.mongo.protocol.ReplyMessage; +import org.apache.mina.filter.codec.ProtocolCodecException; +import java.io.IOException; +import org.apache.mina.core.session.IoSession; +import org.apache.mina.transport.socket.SocketSessionConfig; +import java.net.SocketAddress; +import java.net.InetSocketAddress; +import org.apache.mina.core.session.IdleStatus; +import org.apache.mina.core.service.IoHandler; import com.fangjia.datastore.handler.UpdateMessageHandler; -import com.fangjia.datastore.mongo.codec.BSONIoBufferAllocator; +import com.fangjia.datastore.mongo.protocol.UpdateMessage; +import com.fangjia.datastore.handler.DeleteMessageHandler; +import com.fangjia.datastore.mongo.protocol.DeleteMessage; +import com.fangjia.datastore.handler.KillCursorsMessageHandler; +import com.fangjia.datastore.mongo.protocol.KillCursorsMessage; +import com.fangjia.datastore.handler.GetMoreMessageHandler; +import com.fangjia.datastore.mongo.protocol.GetMoreMessage; +import com.fangjia.datastore.handler.QueryMessageHandler; +import com.fangjia.datastore.mongo.protocol.QueryMessage; +import com.fangjia.datastore.handler.InsertMessageHandler; +import com.fangjia.datastore.mongo.protocol.InsertMessage; +import java.util.concurrent.ExecutorService; +import org.apache.mina.filter.logging.LogLevel; +import org.apache.mina.filter.logging.LoggingFilter; +import org.apache.mina.core.filterchain.IoFilter; +import org.apache.mina.filter.codec.ProtocolDecoder; +import org.apache.mina.filter.codec.ProtocolEncoder; +import org.apache.mina.filter.codec.ProtocolCodecFilter; import com.fangjia.datastore.mongo.codec.MongoProtocolDecoder; import com.fangjia.datastore.mongo.codec.MongoProtocolEncoder; -import com.fangjia.datastore.mongo.protocol.DeleteMessage; -import com.fangjia.datastore.mongo.protocol.GetMoreMessage; -import com.fangjia.datastore.mongo.protocol.InsertMessage; -import com.fangjia.datastore.mongo.protocol.KillCursorsMessage; +import org.apache.mina.core.service.IoProcessor; +import java.util.concurrent.Executor; +import org.apache.mina.core.service.SimpleIoProcessorPool; +import org.apache.mina.transport.socket.nio.NioProcessor; +import java.util.concurrent.Executors; +import org.apache.mina.core.buffer.IoBufferAllocator; +import org.apache.mina.core.buffer.IoBuffer; +import com.fangjia.datastore.mongo.codec.BSONIoBufferAllocator; +import com.fangjia.datastore.config.DataStoreConfig; +import java.util.HashMap; +import com.fangjia.datastore.handler.MessageHandler; +import java.util.Map; import com.fangjia.datastore.mongo.protocol.MongoMessage; -import com.fangjia.datastore.mongo.protocol.QueryMessage; -import com.fangjia.datastore.mongo.protocol.ReplyMessage; -import com.fangjia.datastore.mongo.protocol.UpdateMessage; - -public class DataStoreServer extends IoHandlerAdapter implements IoServiceListener { - - private final NioSocketAcceptor acceptor; - private final DataStoreContext context; - private static final Logger LOGGER = LoggerFactory.getLogger(DataStoreServer.class); - private static ThreadLocal CURRENT_MESSAGE = new ThreadLocal(); - private final Map, MessageHandler> handlerMap = new HashMap, MessageHandler>(); - - public DataStoreServer(String[] args) throws Exception { - if (LOGGER.isInfoEnabled()) { - LOGGER.info("starting datastore..."); - } - - DataStoreConfig config = new DataStoreConfig(System.getProperty("datastore.home", "d:\\goojiaconfig\\datastore")); - config.setReadOnly(Boolean.getBoolean("datastore.readOnly")); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(config.toString()); - } - context = new DataStoreContextImpl(config); - - IoBuffer.setAllocator(new BSONIoBufferAllocator()); - - final ExecutorService executorService = Executors.newCachedThreadPool(); - acceptor = new NioSocketAcceptor(executorService, new SimpleIoProcessorPool(NioProcessor.class, config - .getServerConfig().getMaxConnections())); - - acceptor.getFilterChain().addLast("codec", - new ProtocolCodecFilter(new MongoProtocolEncoder(), new MongoProtocolDecoder())); - acceptor.getFilterChain().addLast("performance", - new PerformanceLogFilter(config.getServerConfig().getPerformanceLogFilterMaxTime())); - if (!config.getServerConfig().isDisableLogging()) { - LoggingFilter loggingFilter = new LoggingFilter(DataStoreServer.class); - loggingFilter.setExceptionCaughtLogLevel(LogLevel.NONE); - acceptor.getFilterChain().addLast("logger", loggingFilter); - } - - Runtime.getRuntime().addShutdownHook(new Thread() { - - @Override - public void run() { - LOGGER.info("shutting down..."); - acceptor.unbind(); - executorService.shutdown(); - LOGGER.info("shut down"); - } - - }); - - // register message handlers - register(InsertMessage.class, new InsertMessageHandler(context)); - register(QueryMessage.class, new QueryMessageHandler(context)); - register(GetMoreMessage.class, new GetMoreMessageHandler(context)); - register(KillCursorsMessage.class, new KillCursorsMessageHandler(context)); - register(DeleteMessage.class, new DeleteMessageHandler(context)); - register(UpdateMessage.class, new UpdateMessageHandler(context)); - - acceptor.setHandler(this); - acceptor.addListener(this); - acceptor.setReuseAddress(true); - SocketSessionConfig sessionConfig = acceptor.getSessionConfig(); - sessionConfig.setTcpNoDelay(true); - sessionConfig.setKeepAlive(true); - sessionConfig.setReuseAddress(true); - sessionConfig.setReadBufferSize(config.getServerConfig().getMinaReadBufferSize()); - sessionConfig.setIdleTime(IdleStatus.BOTH_IDLE, 300); - acceptor.bind(new InetSocketAddress(config.getServerConfig().getPort())); - } - - private void register(Class messageClass, MessageHandler handler) { - this.handlerMap.put(messageClass, handler); - } - - /** - * @param args - * @throws Exception - * @throws - */ - public static void main(String[] args) throws Exception { - new DataStoreServer(args); - } - - @Override - public void exceptionCaught(IoSession session, Throwable cause) throws Exception { - if (cause instanceof IOException && null != cause.getMessage() && cause.getMessage().contains("Connection reset by peer")) { - if (LOGGER.isInfoEnabled()) { - LOGGER.info("connection closed on session: {}", session); - } - } else if (cause instanceof ProtocolCodecException) { - // ProtocolDecoderException or ProtocolEncoderException - MongoMessage message = CURRENT_MESSAGE.get(); - if (null != message) { - if (message instanceof QueryMessage) { - QueryMessage queryMessage = (QueryMessage) message; - ReplyMessage reply = new ReplyMessage(); - reply.setNumberReturned(1); - reply.setResponseFlag(8); - reply.getMessageHeader().setResponseTo(queryMessage.getMessageHeader().getRequestID()); - BSONObject msg = new BasicBSONObject("$err", cause.getMessage()); - reply.setDocuments(Arrays.asList(msg)); - session.write(reply); - } - } - } else if (LOGGER.isErrorEnabled()) { - LOGGER.error("exception caught on session:{" + session + "}", cause); - } - } - - @Override - public void messageReceived(IoSession session, Object message) throws Exception { - if (message instanceof MongoMessage) { - MongoMessage mongoMessage = (MongoMessage) message; - CURRENT_MESSAGE.set(mongoMessage); - MessageHandler handler = this.handlerMap.get(mongoMessage.getClass()); - if (null != handler) { - handler.messageReceived(session, mongoMessage); - } else { - throw new RuntimeException("unknown message received:" + message); - } - } else { - throw new RuntimeException("unknown message received:" + message); - } - } - - public void serviceActivated(IoService service) throws Exception { - if (LOGGER.isInfoEnabled()) { - LOGGER.info("datastore started"); - LOGGER.info("listening at " + this.context.getDataStoreConfig().getServerConfig().getPort()); - } - } - - public void serviceIdle(IoService service, IdleStatus idleStatus) throws Exception { - // empty - } - - public void serviceDeactivated(IoService service) throws Exception { - // empty - } - - public void sessionDestroyed(IoSession session) throws Exception { - // empty - } +import org.slf4j.Logger; +import org.apache.mina.transport.socket.nio.NioSocketAcceptor; +import org.apache.mina.core.service.IoServiceListener; +import org.apache.mina.core.service.IoHandlerAdapter; +public class DataStoreServer extends IoHandlerAdapter implements IoServiceListener +{ + private final NioSocketAcceptor acceptor; + private final DataStoreContext context; + private static final Logger LOGGER; + private static ThreadLocal CURRENT_MESSAGE; + private final Map, MessageHandler> handlerMap; + + public DataStoreServer(final String[] args) throws Exception { + this.handlerMap = new HashMap, MessageHandler>(); + if (DataStoreServer.LOGGER.isInfoEnabled()) { + DataStoreServer.LOGGER.info("starting datastore..."); + } + final DataStoreConfig config = new DataStoreConfig(System.getProperty("datastore.home", "D:\\goojiaconfig\\seaweedsolr6")); + config.setReadOnly(Boolean.getBoolean("datastore.readOnly")); + if (DataStoreServer.LOGGER.isInfoEnabled()) { + DataStoreServer.LOGGER.info(config.toString()); + } + this.context = new DataStoreContextImpl(config); + IoBuffer.setAllocator(new BSONIoBufferAllocator()); + final ExecutorService executorService = Executors.newCachedThreadPool(); + this.acceptor = new NioSocketAcceptor(executorService, new SimpleIoProcessorPool(NioProcessor.class, config.getServerConfig().getMaxConnections())); + this.acceptor.getFilterChain().addLast("codec", new ProtocolCodecFilter(new MongoProtocolEncoder(), new MongoProtocolDecoder())); + this.acceptor.getFilterChain().addLast("performance", new PerformanceLogFilter(config.getServerConfig().getPerformanceLogFilterMaxTime())); + if (!config.getServerConfig().isDisableLogging()) { + final LoggingFilter loggingFilter = new LoggingFilter(DataStoreServer.class); + loggingFilter.setExceptionCaughtLogLevel(LogLevel.NONE); + this.acceptor.getFilterChain().addLast("logger", loggingFilter); + } + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + DataStoreServer.LOGGER.info("shutting down..."); + DataStoreServer.this.acceptor.unbind(); + executorService.shutdown(); + DataStoreServer.LOGGER.info("shut down"); + } + }); + this.register(InsertMessage.class, new InsertMessageHandler(this.context)); + this.register(QueryMessage.class, new QueryMessageHandler(this.context)); + this.register(GetMoreMessage.class, new GetMoreMessageHandler(this.context)); + this.register(KillCursorsMessage.class, new KillCursorsMessageHandler(this.context)); + this.register(DeleteMessage.class, new DeleteMessageHandler(this.context)); + this.register(UpdateMessage.class, new UpdateMessageHandler(this.context)); + this.acceptor.setHandler(this); + this.acceptor.addListener(this); + this.acceptor.setReuseAddress(true); + final SocketSessionConfig sessionConfig = this.acceptor.getSessionConfig(); + sessionConfig.setTcpNoDelay(true); + sessionConfig.setKeepAlive(true); + sessionConfig.setReuseAddress(true); + sessionConfig.setReadBufferSize(config.getServerConfig().getMinaReadBufferSize()); + sessionConfig.setIdleTime(IdleStatus.BOTH_IDLE, 300); + this.acceptor.bind(new InetSocketAddress(config.getServerConfig().getPort())); + } + + private void register(final Class messageClass, final MessageHandler handler) { + this.handlerMap.put(messageClass, handler); + } + + public static void main(final String[] args) throws Exception { + new DataStoreServer(args); + } + + public void exceptionCaught(final IoSession session, final Throwable cause) throws Exception { + if (cause instanceof IOException && null != cause.getMessage() && cause.getMessage().contains("Connection reset by peer")) { + if (DataStoreServer.LOGGER.isInfoEnabled()) { + DataStoreServer.LOGGER.info("connection closed on session: {}", (Object)session); + } + } + else if (cause instanceof ProtocolCodecException) { + final MongoMessage message = DataStoreServer.CURRENT_MESSAGE.get(); + if (null != message && message instanceof QueryMessage) { + final QueryMessage queryMessage = (QueryMessage)message; + final ReplyMessage reply = new ReplyMessage(); + reply.setNumberReturned(1); + reply.setResponseFlag(8); + reply.getMessageHeader().setResponseTo(queryMessage.getMessageHeader().getRequestID()); + final BSONObject msg = (BSONObject)new BasicBSONObject("$err", cause.getMessage()); + reply.setDocuments(Arrays.asList(msg)); + session.write((Object)reply); + } + } + else if (DataStoreServer.LOGGER.isErrorEnabled()) { + DataStoreServer.LOGGER.error("exception caught on session:{" + session + "}", cause); + } + } + + public void messageReceived(final IoSession session, final Object message) throws Exception { + if (!(message instanceof MongoMessage)) { + throw new RuntimeException("unknown message received:" + message); + } + final MongoMessage mongoMessage = (MongoMessage)message; + DataStoreServer.CURRENT_MESSAGE.set(mongoMessage); + final MessageHandler handler = this.handlerMap.get(mongoMessage.getClass()); + if (null != handler) { + handler.messageReceived(session, mongoMessage); + return; + } + throw new RuntimeException("unknown message received:" + message); + } + + public void serviceActivated(final IoService service) throws Exception { + if (DataStoreServer.LOGGER.isInfoEnabled()) { + DataStoreServer.LOGGER.info("datastore started"); + DataStoreServer.LOGGER.info("listening at " + this.context.getDataStoreConfig().getServerConfig().getPort()); + } + } + + public void serviceIdle(final IoService service, final IdleStatus idleStatus) throws Exception { + } + + public void serviceDeactivated(final IoService service) throws Exception { + } + + public void sessionDestroyed(final IoSession session) throws Exception { + } + + static { + LOGGER = LoggerFactory.getLogger((Class)DataStoreServer.class); + DataStoreServer.CURRENT_MESSAGE = new ThreadLocal(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStore.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStore.java index 0ec78a5c7..daf576ffa 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStore.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStore.java @@ -1,64 +1,22 @@ package com.fangjia.datastore; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.util.List; -import java.util.Set; - import org.bson.BSONObject; +import java.util.Set; import org.bson.types.ObjectId; - import com.mongodb.DBObject; +import java.util.List; -public interface FileStore { - public final static String DEFAULT_CHARSET = "UTF-8"; - - /** - * - * @param fileList - * 仅包含大字段的文档列表 - * @param dbObjList - * 已经保存到数据库的主对象列表 - * @throws IllegalArgumentException - * 大字段数据不支持默认编码时抛出 - */ - void save(List fileList, List dbObjList) throws DataStoreException; - - /** - * 删除指定的大字段内容 - * - * @param objIdList - * 删除的主对象的id列表 - * @param removeFields - * 需要删除的大字段名集合 - */ - public void remove(List objIdList, Set removeFields); - - /** - * 删除所有大字段内容 - * - * @param objIdList - */ - public void remove(List objIdList); - - /** - * @param objIdList - * 待更新的主对象id列表 - * @param data - * 更新文档 - * @throws UnsupportedEncodingException - * 大字段数据不支持默认编码时抛出 - */ - public void update(List objIdList, BSONObject data) throws DataStoreException; - - /** - * @param docList - * 查询的结果文档 - * @param fields - * 查询字段对象 - * @return - * @throws IOException - * 读取大字段数据异常时、大字段数据不支持默认编码时抛出 - */ - public List find(List docList, BSONObject fields) throws DataStoreException; +public interface FileStore +{ + public static final String DEFAULT_CHARSET = "UTF-8"; + + void save(final List p0, final List p1) throws DataStoreException; + + void remove(final List p0, final Set p1); + + void remove(final List p0); + + void update(final List p0, final BSONObject p1) throws DataStoreException; + + List find(final List p0, final BSONObject p1) throws DataStoreException; } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStoreImpl.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStoreImpl.java index 4285d99b7..9cef6152f 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStoreImpl.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStoreImpl.java @@ -1,249 +1,229 @@ package com.fangjia.datastore; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.zip.GZIPOutputStream; - import org.apache.commons.io.IOUtils; +import com.mongodb.gridfs.GridFSDBFile; +import java.util.HashSet; +import com.mongodb.BasicDBObject; +import java.util.ArrayList; +import java.util.Set; +import java.io.OutputStream; +import java.util.zip.GZIPOutputStream; +import java.io.ByteArrayOutputStream; +import com.mongodb.gridfs.GridFSFile; +import java.util.Date; +import java.io.IOException; import org.apache.commons.lang.StringUtils; +import com.fangjia.datastore.config.FieldTypeEnum; import org.bson.BSONObject; import org.bson.types.ObjectId; - -import com.fangjia.datastore.config.Field; -import com.fangjia.datastore.config.FieldTypeEnum; -import com.mongodb.BasicDBObject; import com.mongodb.DBObject; +import java.util.List; +import java.util.Iterator; +import java.util.HashMap; +import java.util.Collection; +import com.fangjia.datastore.config.Field; +import java.util.Map; import com.mongodb.gridfs.GridFS; -import com.mongodb.gridfs.GridFSDBFile; -import com.mongodb.gridfs.GridFSFile; -public class FileStoreImpl implements FileStore { - - private GridFS gridFS; - private Map bigFieldSet; - - public FileStoreImpl(GridFS gridFS, Collection fields) { - this.gridFS = gridFS; - - // 把大字段的字段名放入set - bigFieldSet = new HashMap(); - for (Field field : fields) { - if (field.isBigField()) { - bigFieldSet.put(field.getName(), field); - } - } - } - - public void save(List fileList, List dbObjList) { - if (null == fileList || null == dbObjList || fileList.size() != dbObjList.size()) - throw new IllegalArgumentException("empty list or unequal length list is not allowed!"); - - Iterator dbObjIt = dbObjList.iterator(); - for (Iterator fileIt = fileList.iterator(); fileIt.hasNext();) { - // 取得已保存主对象的id - String id = ((ObjectId) dbObjIt.next().get("_id")).toString(); - - BSONObject file = fileIt.next(); - for (String fieldName : file.keySet()) { - byte[] bytes = null; - - Field field = bigFieldSet.get(fieldName); - if (null == field) { - continue; - } - FieldTypeEnum fieldType = field.getType(); - if (FieldTypeEnum.TEXT == fieldType) { - String value = (String) file.get(fieldName); - if (!StringUtils.isBlank(value)) { - try { - bytes = gzip(value); - } catch (IOException e) { - throw new IllegalArgumentException("unsupported encoding value!"); - } - } - } - - if (null != bytes && bytes.length > 0) { - GridFSFile mongofile = gridFS.createFile(bytes); - mongofile.put("uploadDate", new Date()); - mongofile.put("filename", id); - mongofile.put("contentType", fieldName); - mongofile.save(); - } - } - } - } - - private byte[] gzip(String string) throws IOException { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - GZIPOutputStream gzipOutputStream = new GZIPOutputStream(outputStream); - gzipOutputStream.write(string.getBytes(DEFAULT_CHARSET)); - return outputStream.toByteArray(); - } - - public void remove(List objIdList) { - remove(objIdList, null); - } - - public void remove(List objIdList, Set removeFields) { - // 删除表的所有大字段 - if (null == removeFields) - removeFields = bigFieldSet.keySet(); - - // 若到删除的字段中没有大字段,直接返回 - if (removeFields.size() == 0) - return; - - List idList = new ArrayList(objIdList.size()); - for (ObjectId objId : objIdList) { - idList.add(objId.toString()); - } - - // 删除大字段关联的GridFS - DBObject fileSeletor = new BasicDBObject(); - fileSeletor.put("filename", new BasicDBObject("$in", idList)); - fileSeletor.put("contentType", new BasicDBObject("$in", removeFields)); - gridFS.remove(fileSeletor); - } - - public void update(List objIdList, BSONObject data) throws DataStoreException { - // 待删除的字段 - BSONObject unset = (BSONObject) data.get("$unset"); - if (null != unset) { - // 待删除的大字段set - Set removeSet = new HashSet(); - for (String field : unset.keySet()) { - if (bigFieldSet.containsKey(field)) - removeSet.add(field); - } - - if (removeSet.size() > 0) - remove(objIdList, removeSet); - } - - // 更新数据 - BSONObject set = (BSONObject) data.get("$set"); - if (null == set) - return; - - Map fileMap = new HashMap(); - for (String field : set.keySet()) { - if (bigFieldSet.containsKey(field)) { - fileMap.put(field, set.get(field)); - } - } - - // 更新数据中没有大字段 - if (fileMap.size() == 0) - return; - - // 删除大字段关联的GridFS - remove(objIdList, fileMap.keySet()); - - // 将新的内容存入GridFS - for (ObjectId objId : objIdList) { - String id = objId.toString(); - for (String fieldName : fileMap.keySet()) { - byte[] bytes = null; - - Field field = bigFieldSet.get(fieldName); - if (null == field) { - continue; - } - FieldTypeEnum fieldType = field.getType(); - if (FieldTypeEnum.TEXT == fieldType) { - String value = (String) fileMap.get(fieldName); - if (!StringUtils.isBlank(value)) { - try { - bytes = gzip(value); - } catch (IOException e) { - throw new IllegalArgumentException("unsupported encoding value!"); - } - } - } - - if (null != bytes && bytes.length > 0) { - GridFSFile mongofile = gridFS.createFile(bytes); - mongofile.put("uploadDate", new Date()); - mongofile.put("filename", id); - mongofile.put("contentType", fieldName); - mongofile.save(); - } - } - } - - } - - public List find(List docList, BSONObject fields) throws DataStoreException { - // 判断查询的字段中有没有大字段 - Set queryBigField = new HashSet(); - if (null == fields) { - queryBigField = bigFieldSet.keySet(); - } else { - for (String field : fields.keySet()) { - if (bigFieldSet.containsKey(field)) { - queryBigField.add(field); - } - } - } - - // 没有大字段,则返回原结果文档 - if (queryBigField.size() == 0) - return docList; - - Map docMap = new HashMap(docList.size()); - List idList = new ArrayList(docList.size()); - for (DBObject dbObj : docList) { - ObjectId objId = (ObjectId) dbObj.get("_id"); - if (null != objId) { - String id = objId.toString(); - idList.add(id); - docMap.put(id, dbObj); - } - } - - DBObject fileSeletor = new BasicDBObject(); - fileSeletor.put("filename", new BasicDBObject("$in", idList)); - fileSeletor.put("contentType", new BasicDBObject("$in", queryBigField)); - List fileList = gridFS.find(fileSeletor); - - for (GridFSDBFile file : fileList) { - // 对应主记录id - String id = file.getFilename(); - // 对应主记录大字段名 - String fieldName = file.getContentType(); - - DBObject obj = docMap.get(id); - if (obj.keySet().contains(fieldName)) { - - try { - ByteArrayOutputStream data = new ByteArrayOutputStream(); - GZIPOutputStream out = new GZIPOutputStream(data); - IOUtils.copy(file.getInputStream(), out); - - Field field = bigFieldSet.get(fieldName); - if (null == field) { - continue; - } - FieldTypeEnum fieldType = field.getType(); - if (FieldTypeEnum.TEXT == fieldType) { - obj.put(fieldName, new String(data.toByteArray(), DEFAULT_CHARSET)); - } - } catch (IOException e) { - throw new DataStoreException("error extract text field[" + fieldName + "]:", e); - } - } - } - - return docList; - } +public class FileStoreImpl implements FileStore +{ + private GridFS gridFS; + private Map bigFieldSet; + + public FileStoreImpl(final GridFS gridFS, final Collection fields) { + this.gridFS = gridFS; + this.bigFieldSet = new HashMap(); + for (final Field field : fields) { + if (field.isBigField()) { + this.bigFieldSet.put(field.getName(), field); + } + } + } + + @Override + public void save(final List fileList, final List dbObjList) { + if (null == fileList || null == dbObjList || fileList.size() != dbObjList.size()) { + throw new IllegalArgumentException("empty list or unequal length list is not allowed!"); + } + final Iterator dbObjIt = dbObjList.iterator(); + final Iterator fileIt = fileList.iterator(); + while (fileIt.hasNext()) { + final String id = ((ObjectId)dbObjIt.next().get("_id")).toString(); + final BSONObject file = (BSONObject)fileIt.next(); + for (final String fieldName : file.keySet()) { + byte[] bytes = null; + final Field field = this.bigFieldSet.get(fieldName); + if (null == field) { + continue; + } + final FieldTypeEnum fieldType = field.getType(); + if (FieldTypeEnum.TEXT == fieldType) { + final String value = (String)file.get(fieldName); + if (!StringUtils.isBlank(value)) { + try { + bytes = this.gzip(value); + } + catch (IOException e) { + throw new IllegalArgumentException("unsupported encoding value!"); + } + } + } + if (null == bytes || bytes.length <= 0) { + continue; + } + final GridFSFile mongofile = (GridFSFile)this.gridFS.createFile(bytes); + mongofile.put("uploadDate", (Object)new Date()); + mongofile.put("filename", (Object)id); + mongofile.put("contentType", (Object)fieldName); + mongofile.save(); + } + } + } + + private byte[] gzip(final String string) throws IOException { + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + final GZIPOutputStream gzipOutputStream = new GZIPOutputStream(outputStream); + gzipOutputStream.write(string.getBytes("UTF-8")); + return outputStream.toByteArray(); + } + + @Override + public void remove(final List objIdList) { + this.remove(objIdList, null); + } + + @Override + public void remove(final List objIdList, Set removeFields) { + if (null == removeFields) { + removeFields = this.bigFieldSet.keySet(); + } + if (removeFields.size() == 0) { + return; + } + final List idList = new ArrayList(objIdList.size()); + for (final ObjectId objId : objIdList) { + idList.add(objId.toString()); + } + final DBObject fileSeletor = (DBObject)new BasicDBObject(); + fileSeletor.put("filename", (Object)new BasicDBObject("$in", (Object)idList)); + fileSeletor.put("contentType", (Object)new BasicDBObject("$in", (Object)removeFields)); + this.gridFS.remove(fileSeletor); + } + + @Override + public void update(final List objIdList, final BSONObject data) throws DataStoreException { + final BSONObject unset = (BSONObject)data.get("$unset"); + if (null != unset) { + final Set removeSet = new HashSet(); + for (final String field : unset.keySet()) { + if (this.bigFieldSet.containsKey(field)) { + removeSet.add(field); + } + } + if (removeSet.size() > 0) { + this.remove(objIdList, removeSet); + } + } + final BSONObject set = (BSONObject)data.get("$set"); + if (null == set) { + return; + } + final Map fileMap = new HashMap(); + for (final String field2 : set.keySet()) { + if (this.bigFieldSet.containsKey(field2)) { + fileMap.put(field2, set.get(field2)); + } + } + if (fileMap.size() == 0) { + return; + } + this.remove(objIdList, fileMap.keySet()); + for (final ObjectId objId : objIdList) { + final String id = objId.toString(); + for (final String fieldName : fileMap.keySet()) { + byte[] bytes = null; + final Field field3 = this.bigFieldSet.get(fieldName); + if (null == field3) { + continue; + } + final FieldTypeEnum fieldType = field3.getType(); + if (FieldTypeEnum.TEXT == fieldType) { + final String value = (String)fileMap.get(fieldName); + if (!StringUtils.isBlank(value)) { + try { + bytes = this.gzip(value); + } + catch (IOException e) { + throw new IllegalArgumentException("unsupported encoding value!"); + } + } + } + if (null == bytes || bytes.length <= 0) { + continue; + } + final GridFSFile mongofile = (GridFSFile)this.gridFS.createFile(bytes); + mongofile.put("uploadDate", (Object)new Date()); + mongofile.put("filename", (Object)id); + mongofile.put("contentType", (Object)fieldName); + mongofile.save(); + } + } + } + + @Override + public List find(final List docList, final BSONObject fields) throws DataStoreException { + Set queryBigField = new HashSet(); + if (null == fields) { + queryBigField = this.bigFieldSet.keySet(); + } + else { + for (final String field : fields.keySet()) { + if (this.bigFieldSet.containsKey(field)) { + queryBigField.add(field); + } + } + } + if (queryBigField.size() == 0) { + return docList; + } + final Map docMap = new HashMap(docList.size()); + final List idList = new ArrayList(docList.size()); + for (final DBObject dbObj : docList) { + final ObjectId objId = (ObjectId)dbObj.get("_id"); + if (null != objId) { + final String id = objId.toString(); + idList.add(id); + docMap.put(id, dbObj); + } + } + final DBObject fileSeletor = (DBObject)new BasicDBObject(); + fileSeletor.put("filename", (Object)new BasicDBObject("$in", (Object)idList)); + fileSeletor.put("contentType", (Object)new BasicDBObject("$in", (Object)queryBigField)); + final List fileList = (List)this.gridFS.find(fileSeletor); + for (final GridFSDBFile file : fileList) { + final String id2 = file.getFilename(); + final String fieldName = file.getContentType(); + final DBObject obj = docMap.get(id2); + if (obj.keySet().contains(fieldName)) { + try { + final ByteArrayOutputStream data = new ByteArrayOutputStream(); + final GZIPOutputStream out = new GZIPOutputStream(data); + IOUtils.copy(file.getInputStream(), (OutputStream)out); + final Field field2 = this.bigFieldSet.get(fieldName); + if (null == field2) { + continue; + } + final FieldTypeEnum fieldType = field2.getType(); + if (FieldTypeEnum.TEXT != fieldType) { + continue; + } + obj.put(fieldName, (Object)new String(data.toByteArray(), "UTF-8")); + } + catch (IOException e) { + throw new DataStoreException("error extract text field[" + fieldName + "]:", e); + } + } + } + return docList; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Mongo.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Mongo.java index 1955bee2c..bb424447a 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Mongo.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Mongo.java @@ -1,46 +1,46 @@ package com.fangjia.datastore; -import java.util.List; - -import com.fangjia.datastore.config.MongoConfig; -import com.mongodb.DB; -import com.mongodb.MongoException; import com.mongodb.WriteConcern; +import com.mongodb.MongoException; +import java.util.List; +import com.mongodb.DB; +import com.fangjia.datastore.config.MongoConfig; -public class Mongo { - - private final MongoConfig mongoConfig; - - public Mongo(MongoConfig mongoConfig) { - this.mongoConfig = mongoConfig; - } - - private com.mongodb.Mongo getMongo() { - try { - return mongoConfig.getMongo(); - } catch (ConfigurationException e) { - throw new RuntimeException(e); - } - } - - public DB getDB(String dbname) { - return getMongo().getDB(dbname); - } - - public List getDatabaseNames() throws MongoException { - return getMongo().getDatabaseNames(); - } - - public void setWriteConcern(WriteConcern concern) { - getMongo().setWriteConcern(concern); - } - - public WriteConcern getWriteConcern() { - return getMongo().getWriteConcern(); - } - - public String toString() { - return getMongo().toString(); - } - +public class Mongo +{ + private final MongoConfig mongoConfig; + + public Mongo(final MongoConfig mongoConfig) { + this.mongoConfig = mongoConfig; + } + + private com.mongodb.Mongo getMongo() { + try { + return this.mongoConfig.getMongo(); + } + catch (ConfigurationException e) { + throw new RuntimeException(e); + } + } + + public DB getDB(final String dbname) { + return this.getMongo().getDB(dbname); + } + + public List getDatabaseNames() throws MongoException { + return (List)this.getMongo().getDatabaseNames(); + } + + public void setWriteConcern(final WriteConcern concern) { + this.getMongo().setWriteConcern(concern); + } + + public WriteConcern getWriteConcern() { + return this.getMongo().getWriteConcern(); + } + + @Override + public String toString() { + return this.getMongo().toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/PerformanceLogFilter.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/PerformanceLogFilter.java index 26d62b527..7ae1f812f 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/PerformanceLogFilter.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/PerformanceLogFilter.java @@ -1,114 +1,119 @@ package com.fangjia.datastore; -import org.apache.mina.core.filterchain.IoFilterAdapter; -import org.apache.mina.core.session.IdleStatus; -import org.apache.mina.core.session.IoSession; -import org.apache.mina.core.write.WriteRequest; -import org.slf4j.Logger; - -import com.fangjia.datastore.mongo.protocol.MongoMessage; import com.fangjia.datastore.utils.LoggerFactory; +import org.apache.mina.core.session.IdleStatus; +import org.apache.mina.core.write.WriteRequest; +import org.apache.mina.core.filterchain.IoFilter; +import com.fangjia.datastore.mongo.protocol.MongoMessage; +import org.apache.mina.core.session.IoSession; +import org.slf4j.Logger; +import org.apache.mina.core.filterchain.IoFilterAdapter; -public class PerformanceLogFilter extends IoFilterAdapter { - private final long maxTime; - - private static final Logger LOGGER = LoggerFactory.getPerformanceLogger(PerformanceLogFilter.class); - - public PerformanceLogFilter(long maxTime) { - this.maxTime = maxTime; - } - - private void logPerformace(long time, String methodName, IoSession session, Object message) { - Object[] params; - if (message instanceof MongoMessage) { - MongoMessage mongoMessage = (MongoMessage) message; - params = new Object[] { time, methodName, session, mongoMessage.toPrettyString() }; - } else { - params = new Object[] { time, methodName, session, message }; - } - if (time < maxTime * 3) { - LOGGER.warn("time={}, methodName={}, session={}, message={}", params); - } else { - LOGGER.error("time={}, methodName={}, session={}, message={}", params); - } - } - - @Override - public void messageReceived(NextFilter nextFilter, IoSession session, Object message) throws Exception { - long time = System.currentTimeMillis(); - try { - nextFilter.messageReceived(session, message); - } finally { - time = System.currentTimeMillis() - time; - if (time >= maxTime) { - logPerformace(time, "messageReceived", session, message); - } - } - } - - @Override - public void messageSent(NextFilter nextFilter, IoSession session, WriteRequest writeRequest) throws Exception { - long time = System.currentTimeMillis(); - try { - nextFilter.messageSent(session, writeRequest); - } finally { - time = System.currentTimeMillis() - time; - if (time >= maxTime) { - logPerformace(time, "messageSent", session, writeRequest); - } - } - } - - @Override - public void sessionCreated(NextFilter nextFilter, IoSession session) throws Exception { - long time = System.currentTimeMillis(); - try { - nextFilter.sessionCreated(session); - } finally { - time = System.currentTimeMillis() - time; - if (time >= maxTime) { - logPerformace(time, "sessionCreated", session, null); - } - } - } - - @Override - public void sessionOpened(NextFilter nextFilter, IoSession session) throws Exception { - long time = System.currentTimeMillis(); - try { - nextFilter.sessionOpened(session); - } finally { - time = System.currentTimeMillis() - time; - if (time >= maxTime) { - logPerformace(time, "sessionOpened", session, null); - } - } - } - - @Override - public void sessionIdle(NextFilter nextFilter, IoSession session, IdleStatus status) throws Exception { - long time = System.currentTimeMillis(); - try { - nextFilter.sessionIdle(session, status); - } finally { - time = System.currentTimeMillis() - time; - if (time >= maxTime) { - logPerformace(time, "sessionIdle", session, status); - } - } - } - - @Override - public void sessionClosed(NextFilter nextFilter, IoSession session) throws Exception { - long time = System.currentTimeMillis(); - try { - nextFilter.sessionClosed(session); - } finally { - time = System.currentTimeMillis() - time; - if (time >= maxTime) { - logPerformace(time, "sessionClosed", session, null); - } - } - } - +public class PerformanceLogFilter extends IoFilterAdapter +{ + private final long maxTime; + private static final Logger LOGGER; + + public PerformanceLogFilter(final long maxTime) { + this.maxTime = maxTime; + } + + private void logPerformace(final long time, final String methodName, final IoSession session, final Object message) { + Object[] params; + if (message instanceof MongoMessage) { + final MongoMessage mongoMessage = (MongoMessage)message; + params = new Object[] { time, methodName, session, mongoMessage.toPrettyString() }; + } + else { + params = new Object[] { time, methodName, session, message }; + } + if (time < this.maxTime * 3L) { + PerformanceLogFilter.LOGGER.warn("time={}, methodName={}, session={}, message={}", params); + } + else { + PerformanceLogFilter.LOGGER.error("time={}, methodName={}, session={}, message={}", params); + } + } + + public void messageReceived(final NextFilter nextFilter, final IoSession session, final Object message) throws Exception { + long time = System.currentTimeMillis(); + try { + nextFilter.messageReceived(session, message); + } + finally { + time = System.currentTimeMillis() - time; + if (time >= this.maxTime) { + this.logPerformace(time, "messageReceived", session, message); + } + } + } + + public void messageSent(final NextFilter nextFilter, final IoSession session, final WriteRequest writeRequest) throws Exception { + long time = System.currentTimeMillis(); + try { + nextFilter.messageSent(session, writeRequest); + } + finally { + time = System.currentTimeMillis() - time; + if (time >= this.maxTime) { + this.logPerformace(time, "messageSent", session, writeRequest); + } + } + } + + public void sessionCreated(final NextFilter nextFilter, final IoSession session) throws Exception { + long time = System.currentTimeMillis(); + try { + nextFilter.sessionCreated(session); + } + finally { + time = System.currentTimeMillis() - time; + if (time >= this.maxTime) { + this.logPerformace(time, "sessionCreated", session, null); + } + } + } + + public void sessionOpened(final NextFilter nextFilter, final IoSession session) throws Exception { + long time = System.currentTimeMillis(); + try { + nextFilter.sessionOpened(session); + } + finally { + time = System.currentTimeMillis() - time; + if (time >= this.maxTime) { + this.logPerformace(time, "sessionOpened", session, null); + } + } + } + + public void sessionIdle(final NextFilter nextFilter, final IoSession session, final IdleStatus status) throws Exception { + long time = System.currentTimeMillis(); + try { + nextFilter.sessionIdle(session, status); + } + finally { + time = System.currentTimeMillis() - time; + if (time >= this.maxTime) { + this.logPerformace(time, "sessionIdle", session, status); + } + } + } + + public void sessionClosed(final NextFilter nextFilter, final IoSession session) throws Exception { + long time = System.currentTimeMillis(); + try { + nextFilter.sessionClosed(session); + } + finally { + time = System.currentTimeMillis() - time; + if (time >= this.maxTime) { + this.logPerformace(time, "sessionClosed", session, null); + } + } + } + + static { + LOGGER = LoggerFactory.getPerformanceLogger(PerformanceLogFilter.class); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/QueryResponse.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/QueryResponse.java index 224844bef..f5392909b 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/QueryResponse.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/QueryResponse.java @@ -1,72 +1,80 @@ package com.fangjia.datastore; import java.util.Collections; +import org.apache.solr.client.solrj.response.FacetField; +import com.mongodb.DBObject; +import org.bson.types.ObjectId; import java.util.List; -import org.apache.solr.client.solrj.response.FacetField; -import org.bson.types.ObjectId; - -import com.mongodb.DBObject; - -public class QueryResponse { - private List idList = Collections.emptyList(); - private List docList = Collections.emptyList(); - private List facetFieldList = null; - private int numFound; - private int numReturned; - private int numToSkip; - public final static QueryResponse EMPTY_RESPONSE = new QueryResponse(); - - public List getIdList() { - return idList; - } - - public void setIdList(List idList) { - this.idList = idList; - } - - public boolean hasMore() { - return this.numFound > numReturned + numToSkip; - } - - public int getNumFound() { - return numFound; - } - - public void setNumFound(int numFound) { - this.numFound = numFound; - } - - public int getNumReturned() { - return numReturned; - } - - public void setNumReturned(int numReturned) { - this.numReturned = numReturned; - } - - public int getNumToSkip() { - return numToSkip; - } - - public void setNumToSkip(int numToSkip) { - this.numToSkip = numToSkip; - } - - public List getFacetFieldList() { - return facetFieldList; - } - - public void setFacetFieldList(List facetFieldList) { - this.facetFieldList = facetFieldList; - } - - public List getDocList() { - return docList; - } - - public void setDocList(List docList) { - this.docList = docList; - } - +public class QueryResponse +{ + private List idList; + private List docList; + private List facetFieldList; + private int numFound; + private int numReturned; + private int numToSkip; + public static final QueryResponse EMPTY_RESPONSE; + + public QueryResponse() { + this.idList = Collections.emptyList(); + this.docList = Collections.emptyList(); + this.facetFieldList = null; + } + + public List getIdList() { + return this.idList; + } + + public void setIdList(final List idList) { + this.idList = idList; + } + + public boolean hasMore() { + return this.numFound > this.numReturned + this.numToSkip; + } + + public int getNumFound() { + return this.numFound; + } + + public void setNumFound(final int numFound) { + this.numFound = numFound; + } + + public int getNumReturned() { + return this.numReturned; + } + + public void setNumReturned(final int numReturned) { + this.numReturned = numReturned; + } + + public int getNumToSkip() { + return this.numToSkip; + } + + public void setNumToSkip(final int numToSkip) { + this.numToSkip = numToSkip; + } + + public List getFacetFieldList() { + return this.facetFieldList; + } + + public void setFacetFieldList(final List facetFieldList) { + this.facetFieldList = facetFieldList; + } + + public List getDocList() { + return this.docList; + } + + public void setDocList(final List docList) { + this.docList = docList; + } + + static { + EMPTY_RESPONSE = new QueryResponse(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecuteException.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecuteException.java index f2e1546b4..590bb13d9 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecuteException.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecuteException.java @@ -1,22 +1,21 @@ package com.fangjia.datastore; -public class TriggerExecuteException extends Exception { - - private static final long serialVersionUID = -1165411196743370433L; - - public TriggerExecuteException() { - } - - public TriggerExecuteException(String message) { - super(message); - } - - public TriggerExecuteException(Throwable cause) { - super(cause); - } - - public TriggerExecuteException(String message, Throwable cause) { - super(message, cause); - } - +public class TriggerExecuteException extends Exception +{ + private static final long serialVersionUID = -1165411196743370433L; + + public TriggerExecuteException() { + } + + public TriggerExecuteException(final String message) { + super(message); + } + + public TriggerExecuteException(final Throwable cause) { + super(cause); + } + + public TriggerExecuteException(final String message, final Throwable cause) { + super(message, cause); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecutor.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecutor.java index b998e420e..ce37f7653 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecutor.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecutor.java @@ -1,165 +1,161 @@ package com.fangjia.datastore; -import java.io.IOException; +import org.slf4j.LoggerFactory; +import org.apache.http.HttpRequest; import java.io.InputStream; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.List; - -import org.apache.http.HttpHost; import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.params.AllClientPNames; -import org.apache.http.conn.scheme.PlainSocketFactory; -import org.apache.http.conn.scheme.Scheme; -import org.apache.http.conn.scheme.SchemeRegistry; +import com.mongodb.DBObject; +import org.bson.BasicBSONDecoder; +import org.apache.http.protocol.HttpContext; +import org.apache.http.HttpHost; +import org.apache.http.HttpEntity; import org.apache.http.entity.ByteArrayEntity; +import org.sothis.core.util.GZIPUtils; +import org.bson.BasicBSONEncoder; +import com.mongodb.BasicDBObject; +import org.apache.http.client.methods.HttpPost; +import java.net.URL; +import java.net.URISyntaxException; +import java.io.IOException; +import org.bson.BSONObject; +import java.util.List; +import com.fangjia.datastore.config.Trigger; +import org.apache.http.params.HttpParams; +import org.apache.http.conn.ClientConnectionManager; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; import org.apache.http.params.BasicHttpParams; -import org.apache.http.params.CoreProtocolPNames; -import org.apache.http.params.HttpParams; -import org.apache.http.protocol.HttpContext; -import org.bson.BSONEncoder; -import org.bson.BSONObject; -import org.bson.BasicBSONDecoder; +import org.apache.http.conn.scheme.SchemeSocketFactory; +import org.apache.http.conn.scheme.Scheme; +import org.apache.http.conn.scheme.PlainSocketFactory; +import org.apache.http.conn.scheme.SchemeRegistry; +import org.apache.http.client.HttpClient; import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sothis.core.util.GZIPUtils; -import com.fangjia.datastore.config.Trigger; -import com.mongodb.BasicDBObject; -import com.mongodb.DBObject; - -public class TriggerExecutor { - - private final static int MAX_TOTAL = 200; - private final static int MAX_FOR_ROUTE = 200; - private final static String DEFAULT_CHARSET = "UTF-8"; - private final static int DEFAULT_PORT = 80; - private final static int MAX_RETRY_COUNT = 10; - - private final static Logger LOGGER = LoggerFactory.getLogger(TriggerExecutor.class); - private final static TriggerExecutor TRIGGER_EXECUTOR = new TriggerExecutor(); - - private final HttpClient httpClient; - - private TriggerExecutor() { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("init trigger executor"); - } - SchemeRegistry schemeRegistry = new SchemeRegistry(); - schemeRegistry.register(new Scheme("http", DEFAULT_PORT, new PlainSocketFactory())); - - HttpParams httpParams = new BasicHttpParams(); - httpParams.setParameter(CoreProtocolPNames.HTTP_CONTENT_CHARSET, DEFAULT_CHARSET); - httpParams.setParameter(AllClientPNames.SO_TIMEOUT, 300000); - httpParams.setParameter(AllClientPNames.CONNECTION_TIMEOUT, 60000); - - ThreadSafeClientConnManager connManager = new ThreadSafeClientConnManager(schemeRegistry); - connManager.setMaxTotal(MAX_TOTAL); - connManager.setDefaultMaxPerRoute(MAX_FOR_ROUTE); - - httpClient = new DefaultHttpClient(connManager, httpParams); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("trigger executor initialized[{}]", toString()); - } - } - - public static TriggerExecutor getInstance() { - return TRIGGER_EXECUTOR; - } - - public List execute(Trigger trigger, List docList) throws TriggerExecuteException { - try { - List ret = doExecute(trigger, docList); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("execute trigger: " + trigger + ", doc size:" + docList.size() + ", return size:" + ret.size()); - } - return ret; - } catch (IOException e) { - throw new TriggerExecuteException("error execute trigger: " + trigger, e); - } catch (URISyntaxException e) { - throw new TriggerExecuteException("error execute trigger: " + trigger, e); - } - } - - private List doExecute(Trigger trigger, List docList) throws IOException, URISyntaxException, - TriggerExecuteException { - URL url = new URL(trigger.getInterface()); - HttpPost httpPost = new HttpPost(url.toURI()); - DBObject object = new BasicDBObject(); - object.put("doc_list_size", docList.size()); - object.put("doc_list", docList); - httpPost.setEntity(new ByteArrayEntity(GZIPUtils.makeAsByteArray(new BSONEncoder().encode(object)))); - HttpHost host = new HttpHost(url.getHost(), url.getPort() == -1 ? DEFAULT_PORT : url.getPort()); - HttpResponse httpResponse = executeWithRetry(host, httpPost, null, trigger); - InputStream inputStream = httpResponse.getEntity().getContent(); - try { - if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { - BSONObject returnObject = new BasicBSONDecoder().readObject(GZIPUtils.wrap(inputStream)); - if (null != returnObject) { - List ret = (List) returnObject.get("doc_list"); - if (ret == null) { - throw new TriggerExecuteException("trigger" + trigger + " return an null doc_list[" + ret + "]"); - } - return ret; - } else { - throw new TriggerExecuteException("trigger" + trigger + " returns nothing at all !"); - } - } else { - throw new TriggerExecuteException("error execute trigger:" + trigger + ", get http code:" - + httpResponse.getStatusLine().getStatusCode()); - } - } finally { - if (null != inputStream) { - inputStream.close(); - } - } - } - - private HttpResponse executeWithRetry(HttpHost host, HttpPost httpPost, HttpContext httpContext, Trigger trigger) - throws IOException, TriggerExecuteException { - int retryCounter = 0; - HttpResponse httpResponse = null; - while (true) { - try { - httpResponse = httpClient.execute(host, httpPost, (HttpContext) null); - if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { - return httpResponse; - } else { - retryCounter++; - sleepAndRetry(new IllegalStateException("invalid http status code: " - + httpResponse.getStatusLine().getStatusCode()), retryCounter, trigger); - } - } catch (Exception e) { - retryCounter++; - sleepAndRetry(e, retryCounter, trigger); - } finally { - if (null != httpResponse && httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { - httpResponse.getEntity().getContent().close(); - } - } - } - } - - private void sleepAndRetry(Exception e, int retryCounter, Trigger trigger) throws TriggerExecuteException { - if (retryCounter > MAX_RETRY_COUNT) { - throw new TriggerExecuteException( - "error execute trigger[" + trigger + "] after " + MAX_RETRY_COUNT + " times retry!", e); - } - if (LOGGER.isErrorEnabled()) { - LOGGER.error("error execute trigger[" + trigger + "]:(" + e.getMessage() + "), retry for the " + retryCounter - + "th time after " + (retryCounter * 30) + " seconds..."); - } - try { - Thread.sleep(retryCounter * 30000); - } catch (InterruptedException e1) { - if (LOGGER.isWarnEnabled()) { - LOGGER.warn("", e); - } - } - } +public class TriggerExecutor +{ + private static final int MAX_TOTAL = 200; + private static final int MAX_FOR_ROUTE = 200; + private static final String DEFAULT_CHARSET = "UTF-8"; + private static final int DEFAULT_PORT = 80; + private static final int MAX_RETRY_COUNT = 10; + private static final Logger LOGGER; + private static final TriggerExecutor TRIGGER_EXECUTOR; + private final HttpClient httpClient; + + private TriggerExecutor() { + if (TriggerExecutor.LOGGER.isDebugEnabled()) { + TriggerExecutor.LOGGER.debug("init trigger executor"); + } + final SchemeRegistry schemeRegistry = new SchemeRegistry(); + schemeRegistry.register(new Scheme("http", 80, (SchemeSocketFactory)new PlainSocketFactory())); + final HttpParams httpParams = (HttpParams)new BasicHttpParams(); + httpParams.setParameter("http.protocol.content-charset", (Object)"UTF-8"); + httpParams.setParameter("http.socket.timeout", (Object)300000); + httpParams.setParameter("http.connection.timeout", (Object)60000); + final ThreadSafeClientConnManager connManager = new ThreadSafeClientConnManager(schemeRegistry); + connManager.setMaxTotal(200); + connManager.setDefaultMaxPerRoute(200); + this.httpClient = (HttpClient)new DefaultHttpClient((ClientConnectionManager)connManager, httpParams); + if (TriggerExecutor.LOGGER.isDebugEnabled()) { + TriggerExecutor.LOGGER.debug("trigger executor initialized[{}]", (Object)this.toString()); + } + } + + public static TriggerExecutor getInstance() { + return TriggerExecutor.TRIGGER_EXECUTOR; + } + + public List execute(final Trigger trigger, final List docList) throws TriggerExecuteException { + try { + final List ret = this.doExecute(trigger, docList); + if (TriggerExecutor.LOGGER.isDebugEnabled()) { + TriggerExecutor.LOGGER.debug("execute trigger: " + trigger + ", doc size:" + docList.size() + ", return size:" + ret.size()); + } + return ret; + } + catch (IOException e) { + throw new TriggerExecuteException("error execute trigger: " + trigger, e); + } + catch (URISyntaxException e2) { + throw new TriggerExecuteException("error execute trigger: " + trigger, e2); + } + } + + private List doExecute(final Trigger trigger, final List docList) throws IOException, URISyntaxException, TriggerExecuteException { + final URL url = new URL(trigger.getInterface()); + final HttpPost httpPost = new HttpPost(url.toURI()); + final DBObject object = (DBObject)new BasicDBObject(); + object.put("doc_list_size", (Object)docList.size()); + object.put("doc_list", (Object)docList); + httpPost.setEntity((HttpEntity)new ByteArrayEntity(GZIPUtils.makeAsByteArray(new BasicBSONEncoder().encode((BSONObject)object)))); + final HttpHost host = new HttpHost(url.getHost(), (url.getPort() == -1) ? 80 : url.getPort()); + final HttpResponse httpResponse = this.executeWithRetry(host, httpPost, null, trigger); + final InputStream inputStream = httpResponse.getEntity().getContent(); + try { + if (httpResponse.getStatusLine().getStatusCode() != 200) { + throw new TriggerExecuteException("error execute trigger:" + trigger + ", get http code:" + httpResponse.getStatusLine().getStatusCode()); + } + final BSONObject returnObject = new BasicBSONDecoder().readObject(GZIPUtils.wrap(inputStream)); + if (null == returnObject) { + throw new TriggerExecuteException("trigger" + trigger + " returns nothing at all !"); + } + final List ret = (List)returnObject.get("doc_list"); + if (ret == null) { + throw new TriggerExecuteException("trigger" + trigger + " return an null doc_list[" + ret + "]"); + } + return ret; + } + finally { + if (null != inputStream) { + inputStream.close(); + } + } + } + + private HttpResponse executeWithRetry(final HttpHost host, final HttpPost httpPost, final HttpContext httpContext, final Trigger trigger) throws IOException, TriggerExecuteException { + int retryCounter = 0; + HttpResponse httpResponse = null; + while (true) { + try { + httpResponse = this.httpClient.execute(host, (HttpRequest)httpPost, (HttpContext)null); + if (httpResponse.getStatusLine().getStatusCode() == 200) { + return httpResponse; + } + ++retryCounter; + this.sleepAndRetry(new IllegalStateException("invalid http status code: " + httpResponse.getStatusLine().getStatusCode()), retryCounter, trigger); + } + catch (Exception e) { + ++retryCounter; + this.sleepAndRetry(e, retryCounter, trigger); + } + finally { + if (null != httpResponse && httpResponse.getStatusLine().getStatusCode() != 200) { + httpResponse.getEntity().getContent().close(); + } + } + } + } + + private void sleepAndRetry(final Exception e, final int retryCounter, final Trigger trigger) throws TriggerExecuteException { + if (retryCounter > 10) { + throw new TriggerExecuteException("error execute trigger[" + trigger + "] after " + 10 + " times retry!", e); + } + if (TriggerExecutor.LOGGER.isErrorEnabled()) { + TriggerExecutor.LOGGER.error("error execute trigger[" + trigger + "]:(" + e.getMessage() + "), retry for the " + retryCounter + "th time after " + retryCounter * 30 + " seconds..."); + } + try { + Thread.sleep(retryCounter * 30000); + } + catch (InterruptedException e2) { + if (TriggerExecutor.LOGGER.isWarnEnabled()) { + TriggerExecutor.LOGGER.warn("", (Throwable)e); + } + } + } + + static { + LOGGER = LoggerFactory.getLogger((Class)TriggerExecutor.class); + TRIGGER_EXECUTOR = new TriggerExecutor(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/DataStoreConfig.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/DataStoreConfig.java index b70a6f73a..f532a2655 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/DataStoreConfig.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/DataStoreConfig.java @@ -1,170 +1,172 @@ package com.fangjia.datastore.config; -import java.io.File; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.dom4j.Document; import org.dom4j.DocumentException; +import java.io.File; +import java.util.Iterator; +import java.util.List; import org.dom4j.Element; import org.dom4j.Node; +import org.dom4j.Document; +import java.util.Collections; +import java.util.Set; +import com.fangjia.datastore.ConfigurationException; +import java.util.HashMap; +import java.util.Map; import org.dom4j.io.SAXReader; -import com.fangjia.datastore.ConfigurationException; - -public class DataStoreConfig implements Reloadable { - private final SAXReader reader = new SAXReader(); - private final String serverConfigFile; - private final String storeConfigFile; - private final ServerConfig serverConfig; - private final Map storeConfigs; - private final Map solrConfigs; - private final Map mongoConfigs; - private boolean readOnly = false; - - public DataStoreConfig(String datastoreHome) throws ConfigurationException { - this.serverConfigFile = datastoreHome + "/conf/server.xml"; - this.storeConfigFile = datastoreHome + "/conf/store.xml"; - this.serverConfig = new ServerConfig(); - this.storeConfigs = new HashMap(); - this.solrConfigs = new HashMap(); - this.mongoConfigs = new HashMap(); - load(); - } - - public ServerConfig getServerConfig() { - return serverConfig; - } - - public SolrConfig getSolrConfigById(String id) { - return solrConfigs.get(id); - } - - public MongoConfig getMongoConfigById(String id) { - return this.mongoConfigs.get(id); - } - - public StoreConfig getStoreConfig(String storeName) { - return this.storeConfigs.get(storeName); - } - - public Set getStoreNames() { - return Collections.unmodifiableSet(this.storeConfigs.keySet()); - } - - public void reload() throws ConfigurationException { - load(); - } - - private synchronized void load() throws ConfigurationException { - Document document = read(this.serverConfigFile); - serverConfig.reload(document.selectSingleNode("//datastore//server")); - initSolrConfigs(document); - initMongoConfigs(document); - document = read(this.storeConfigFile); - initStoreConfigs(document, solrConfigs, mongoConfigs); - } - - synchronized StoreConfig getOrCreateStoreConfig(String storeName) { - StoreConfig ret = this.storeConfigs.get(storeName); - if (null == ret) { - ret = new StoreConfig(this, storeName); - storeConfigs.put(storeName, ret); - } - return ret; - } - - private synchronized SolrConfig getOrCreateSolrConfig(String solrId) { - SolrConfig ret = this.solrConfigs.get(solrId); - if (null == ret) { - ret = new SolrConfig(this, solrId); - solrConfigs.put(solrId, ret); - } - return ret; - } - - private synchronized MongoConfig getOrCreateMongoConfig(String mongoId) throws ConfigurationException { - MongoConfig ret = this.mongoConfigs.get(mongoId); - if (null == ret) { - ret = new MongoConfig(mongoId); - mongoConfigs.put(mongoId, ret); - } - return ret; - } - - private void initSolrConfigs(Document document) throws ConfigurationException { - List solrNodes = document.selectNodes("//datastore//solr"); - if (null == solrNodes || solrNodes.isEmpty()) { - throw new ConfigurationException("no solr is configured in " + serverConfigFile + " !"); - } - Map newSolrConfigs = new HashMap(); - for (Node solrNode : solrNodes) { - String id = ((Element) solrNode).attributeValue("id"); - SolrConfig config = getOrCreateSolrConfig(id); - config.reload(solrNode); - newSolrConfigs.put(id, config); - } - this.solrConfigs.clear(); - this.solrConfigs.putAll(newSolrConfigs); - } - - private void initMongoConfigs(Document document) throws ConfigurationException { - List mongoNodes = document.selectNodes("//datastore//mongo"); - if (null == mongoNodes || mongoNodes.isEmpty()) { - throw new ConfigurationException("no mongo is configured in " + serverConfigFile + " !"); - } - Map newMongoConfigs = new HashMap(); - for (Node mongoNode : mongoNodes) { - String id = ((Element) mongoNode).attributeValue("id"); - MongoConfig mongoConfig = getOrCreateMongoConfig(id); - mongoConfig.reload(mongoNode); - newMongoConfigs.put(id, mongoConfig); - } - this.mongoConfigs.clear(); - this.mongoConfigs.putAll(newMongoConfigs); - } - - private void initStoreConfigs(Document document, Map solrConfigMap, - Map mongoConfigMap) throws ConfigurationException { - List storeNodes = document.selectNodes("//datastore//store"); - if (null == storeNodes || storeNodes.isEmpty()) { - throw new ConfigurationException("no store is configured in " + storeConfigFile + " !"); - } - Map newStoreConfigs = new HashMap(); - for (Node storeNode : storeNodes) { - String name = ((Element) storeNode).attributeValue("name"); - final StoreConfig storeConfig = getOrCreateStoreConfig(name); - storeConfig.reload(storeNode); - newStoreConfigs.put(name, storeConfig); - } - this.storeConfigs.clear(); - this.storeConfigs.putAll(newStoreConfigs); - } - - private Document read(String configFile) throws ConfigurationException { - try { - return reader.read(new File(configFile)); - } catch (DocumentException e) { - throw new ConfigurationException("error read datastore config file: " + configFile, e); - } - } - - @Override - public String toString() { - StringBuilder ret = new StringBuilder(); - ret.append("server config file:").append(this.serverConfigFile).append(", store config file: ").append( - this.storeConfigFile).append(", readOnly:").append(this.readOnly); - return ret.toString(); - } - - public boolean isReadOnly() { - return readOnly; - } - - public void setReadOnly(boolean readOnly) { - this.readOnly = readOnly; - } +public class DataStoreConfig implements Reloadable +{ + private final SAXReader reader; + private final String serverConfigFile; + private final String storeConfigFile; + private final ServerConfig serverConfig; + private final Map storeConfigs; + private final Map solrConfigs; + private final Map mongoConfigs; + private boolean readOnly; + + public DataStoreConfig(final String datastoreHome) throws ConfigurationException { + this.reader = new SAXReader(); + this.readOnly = false; + this.serverConfigFile = datastoreHome + "/conf/server.xml"; + this.storeConfigFile = datastoreHome + "/conf/store.xml"; + this.serverConfig = new ServerConfig(); + this.storeConfigs = new HashMap(); + this.solrConfigs = new HashMap(); + this.mongoConfigs = new HashMap(); + this.load(); + } + + public ServerConfig getServerConfig() { + return this.serverConfig; + } + + public SolrConfig getSolrConfigById(final String id) { + return this.solrConfigs.get(id); + } + + public MongoConfig getMongoConfigById(final String id) { + return this.mongoConfigs.get(id); + } + + public StoreConfig getStoreConfig(final String storeName) { + return this.storeConfigs.get(storeName); + } + + public Set getStoreNames() { + return Collections.unmodifiableSet((Set)this.storeConfigs.keySet()); + } + + @Override + public void reload() throws ConfigurationException { + this.load(); + } + + private synchronized void load() throws ConfigurationException { + Document document = this.read(this.serverConfigFile); + this.serverConfig.reload(document.selectSingleNode("//datastore//server")); + this.initSolrConfigs(document); + this.initMongoConfigs(document); + document = this.read(this.storeConfigFile); + this.initStoreConfigs(document, this.solrConfigs, this.mongoConfigs); + } + + synchronized StoreConfig getOrCreateStoreConfig(final String storeName) { + StoreConfig ret = this.storeConfigs.get(storeName); + if (null == ret) { + ret = new StoreConfig(this, storeName); + this.storeConfigs.put(storeName, ret); + } + return ret; + } + + private synchronized SolrConfig getOrCreateSolrConfig(final String solrId) { + SolrConfig ret = this.solrConfigs.get(solrId); + if (null == ret) { + ret = new SolrConfig(this, solrId); + this.solrConfigs.put(solrId, ret); + } + return ret; + } + + private synchronized MongoConfig getOrCreateMongoConfig(final String mongoId) throws ConfigurationException { + MongoConfig ret = this.mongoConfigs.get(mongoId); + if (null == ret) { + ret = new MongoConfig(mongoId); + this.mongoConfigs.put(mongoId, ret); + } + return ret; + } + + private void initSolrConfigs(final Document document) throws ConfigurationException { + final List solrNodes = (List)document.selectNodes("//datastore//solr"); + if (null == solrNodes || solrNodes.isEmpty()) { + throw new ConfigurationException("no solr is configured in " + this.serverConfigFile + " !"); + } + final Map newSolrConfigs = new HashMap(); + for (final Node solrNode : solrNodes) { + final String id = ((Element)solrNode).attributeValue("id"); + final SolrConfig config = this.getOrCreateSolrConfig(id); + config.reload(solrNode); + newSolrConfigs.put(id, config); + } + this.solrConfigs.clear(); + this.solrConfigs.putAll(newSolrConfigs); + } + + private void initMongoConfigs(final Document document) throws ConfigurationException { + final List mongoNodes = (List)document.selectNodes("//datastore//mongo"); + if (null == mongoNodes || mongoNodes.isEmpty()) { + throw new ConfigurationException("no mongo is configured in " + this.serverConfigFile + " !"); + } + final Map newMongoConfigs = new HashMap(); + for (final Node mongoNode : mongoNodes) { + final String id = ((Element)mongoNode).attributeValue("id"); + final MongoConfig mongoConfig = this.getOrCreateMongoConfig(id); + mongoConfig.reload(mongoNode); + newMongoConfigs.put(id, mongoConfig); + } + this.mongoConfigs.clear(); + this.mongoConfigs.putAll(newMongoConfigs); + } + + private void initStoreConfigs(final Document document, final Map solrConfigMap, final Map mongoConfigMap) throws ConfigurationException { + final List storeNodes = (List)document.selectNodes("//datastore//store"); + if (null == storeNodes || storeNodes.isEmpty()) { + throw new ConfigurationException("no store is configured in " + this.storeConfigFile + " !"); + } + final Map newStoreConfigs = new HashMap(); + for (final Node storeNode : storeNodes) { + final String name = ((Element)storeNode).attributeValue("name"); + final StoreConfig storeConfig = this.getOrCreateStoreConfig(name); + storeConfig.reload(storeNode); + newStoreConfigs.put(name, storeConfig); + } + this.storeConfigs.clear(); + this.storeConfigs.putAll(newStoreConfigs); + } + + private Document read(final String configFile) throws ConfigurationException { + try { + return this.reader.read(new File(configFile)); + } + catch (DocumentException e) { + throw new ConfigurationException("error read datastore config file: " + configFile, (Throwable)e); + } + } + + @Override + public String toString() { + final StringBuilder ret = new StringBuilder(); + ret.append("server config file:").append(this.serverConfigFile).append(", store config file: ").append(this.storeConfigFile).append(", readOnly:").append(this.readOnly); + return ret.toString(); + } + + public boolean isReadOnly() { + return this.readOnly; + } + + public void setReadOnly(final boolean readOnly) { + this.readOnly = readOnly; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Field.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Field.java index 00c58cb2a..c1fad3456 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Field.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Field.java @@ -1,127 +1,118 @@ package com.fangjia.datastore.config; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; - -import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; import org.bson.types.ObjectId; +import java.util.regex.Pattern; +import java.util.Date; +import java.util.Map; +import java.util.List; +import org.apache.commons.lang.StringUtils; -public class Field { - private String name; - private FieldTypeEnum type; - private boolean required; - - Field() { - } - - public String getName() { - return name; - } - - public FieldTypeEnum getType() { - return type; - } - - void setName(String name) { - if (StringUtils.isEmpty(name)) { - throw new IllegalArgumentException("invalid field name:" + name); - } - this.name = name; - } - - void setType(FieldTypeEnum type) { - if (null == type) { - throw new IllegalArgumentException("invalid type of field[" + name + "]:" + type); - } - this.type = type; - } - - public boolean isRequired() { - return required; - } - - void setRequired(boolean required) { - this.required = required; - } - - public boolean isBigField() { - return this.type == FieldTypeEnum.TEXT; - } - - /** - * 验证一个值是否匹配其对应的类型 - * - * @param value - * @return 匹配则返回true,不匹配返回false - */ - public boolean validate(Object value) { - if (this.required && value == null) { - return false; - } - if (value == null) { - return true; - } - Class valueClass = value.getClass(); - switch (this.type) { - case BOOLEAN: - return valueClass == boolean.class || valueClass == Boolean.class; - case BYTE: - return valueClass == byte.class || valueClass == Byte.class; - case INTEGER: - return valueClass == int.class || valueClass == Integer.class; - case LONG: - return valueClass == long.class || valueClass == Long.class; - case DOUBLE: - return valueClass == double.class || valueClass == Double.class; - case STRING: - case TEXT: - return valueClass == String.class; - case LIST: - return value instanceof List; - case MAP: - return value instanceof Map; - case DATETIME: - return value instanceof Date; - case BINARY: - return valueClass == byte[].class; - case REGEXP: - return valueClass == Pattern.class; - case OBJECTID: - return valueClass == ObjectId.class; - default: - throw new RuntimeException("un cased type:" + this); - } - } - - @Override - public int hashCode() { - return new HashCodeBuilder().append(this.name).append(this.type).append(this.required).toHashCode(); - } - - @Override - public boolean equals(Object obj) { - if (obj == this) { - return true; - } - if (null == obj) { - return false; - } - if (!(obj instanceof Field)) { - return false; - } - Field other = (Field) obj; - return new EqualsBuilder().append(this.name, other.name).append(this.type, other.type) - .append(this.required, other.required).isEquals(); - } - - @Override - public String toString() { - return new StringBuilder().append("{field:").append(name).append(", type:").append(type).append(", required:") - .append(required).append("}").toString(); - } - +public class Field +{ + private String name; + private FieldTypeEnum type; + private boolean required; + + Field() { + } + + public String getName() { + return this.name; + } + + public FieldTypeEnum getType() { + return this.type; + } + + void setName(final String name) { + if (StringUtils.isEmpty(name)) { + throw new IllegalArgumentException("invalid field name:" + name); + } + this.name = name; + } + + void setType(final FieldTypeEnum type) { + if (null == type) { + throw new IllegalArgumentException("invalid type of field[" + this.name + "]:" + type); + } + this.type = type; + } + + public boolean isRequired() { + return this.required; + } + + void setRequired(final boolean required) { + this.required = required; + } + + public boolean isBigField() { + return this.type == FieldTypeEnum.TEXT; + } + + public boolean validate(final Object value) { + if (this.required && value == null) { + return false; + } + if (value == null) { + return true; + } + final Class valueClass = value.getClass(); + switch (this.type) { + case BOOLEAN: + return valueClass == Boolean.TYPE || valueClass == Boolean.class; + case BYTE: + return valueClass == Byte.TYPE || valueClass == Byte.class; + case INTEGER: + return valueClass == Integer.TYPE || valueClass == Integer.class; + case LONG: + return valueClass == Long.TYPE || valueClass == Long.class; + case DOUBLE: + return valueClass == Double.TYPE || valueClass == Double.class; + case STRING: + case TEXT: + return valueClass == String.class; + case LIST: + return value instanceof List; + case MAP: + return value instanceof Map; + case DATETIME: + return value instanceof Date; + case BINARY: + return valueClass == byte[].class; + case REGEXP: + return valueClass == Pattern.class; + case OBJECTID: + return valueClass == ObjectId.class; + default: + throw new RuntimeException("un cased type:" + this); + } + } + + @Override + public int hashCode() { + return new HashCodeBuilder().append((Object)this.name).append((Object)this.type).append(this.required).toHashCode(); + } + + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (null == obj) { + return false; + } + if (!(obj instanceof Field)) { + return false; + } + final Field other = (Field)obj; + return new EqualsBuilder().append((Object)this.name, (Object)other.name).append((Object)this.type, (Object)other.type).append(this.required, other.required).isEquals(); + } + + @Override + public String toString() { + return "{field:" + this.name + ", type:" + this.type + ", required:" + this.required + "}"; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/FieldTypeEnum.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/FieldTypeEnum.java index a75f63b43..ffd4b0bb2 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/FieldTypeEnum.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/FieldTypeEnum.java @@ -1,22 +1,31 @@ package com.fangjia.datastore.config; -public enum FieldTypeEnum { - BOOLEAN, BYTE, INTEGER, LONG, DOUBLE, STRING, TEXT, LIST, MAP, DATETIME, BINARY, REGEXP, OBJECTID; - - private FieldTypeEnum() { - } - - public String getType() { - return this.name().toLowerCase(); - } - - public static FieldTypeEnum parse(String type) { - for (FieldTypeEnum typeEnum : FieldTypeEnum.values()) { - if (typeEnum.name().toLowerCase().equals(type)) { - return typeEnum; - } - } - return null; - } - +public enum FieldTypeEnum +{ + BOOLEAN, + BYTE, + INTEGER, + LONG, + DOUBLE, + STRING, + TEXT, + LIST, + MAP, + DATETIME, + BINARY, + REGEXP, + OBJECTID; + + public String getType() { + return this.name().toLowerCase(); + } + + public static FieldTypeEnum parse(final String type) { + for (final FieldTypeEnum typeEnum : values()) { + if (typeEnum.name().toLowerCase().equals(type)) { + return typeEnum; + } + } + return null; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/MongoConfig.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/MongoConfig.java index a3d352e24..98aee8f1d 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/MongoConfig.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/MongoConfig.java @@ -2,87 +2,98 @@ package com.fangjia.datastore.config; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; -import org.dom4j.Node; - -import com.fangjia.datastore.ConfigurationException; import com.fangjia.datastore.utils.XmlUtils; +import org.dom4j.Node; +import com.mongodb.MongoURI; +import org.apache.commons.lang.StringUtils; +import com.fangjia.datastore.ConfigurationException; import com.mongodb.Mongo; -public final class MongoConfig { - - private String host; - private int port; - private final String id; - private Mongo mongo; - - MongoConfig(String id) throws ConfigurationException { - this.id = id; - } - - private synchronized void init() throws ConfigurationException { - if (null == mongo) { - try { - mongo = new Mongo(host, port); - } catch (Exception e) { - throw new ConfigurationException("error init mongo[" + host + ":" + port, e); - } - } - } - - public Mongo getMongo() throws ConfigurationException { - if (null == mongo) { - init(); - } - return mongo; - } - - synchronized void reload(Node mongoNode) throws ConfigurationException { - String host = XmlUtils.getNodeValue(mongoNode, "host", "datastore.mongo.db.fangjia.com"); - int port = XmlUtils.getNodeValueAsInt(mongoNode, "port", 27017); - if (host.equals(this.host) && this.port == port) { - return; - } - this.host = host; - this.port = port; - if (null != mongo) { - try { - init(); - } catch (Exception e) { - throw new ConfigurationException("error init mongo[" + host + ":" + port, e); - } - } - } - - @Override - public int hashCode() { - return new HashCodeBuilder().append(this.host).append(this.port).append(this.id).toHashCode(); - } - - @Override - public boolean equals(Object obj) { - if (obj == this) { - return true; - } - if (null == obj) { - return false; - } - if (!(obj instanceof MongoConfig)) { - return false; - } - MongoConfig other = (MongoConfig) obj; - return new EqualsBuilder().append(this.id, other.id).append(this.host, other.host).append(this.port, other.port) - .isEquals(); - } - - @Override - public String toString() { - StringBuilder ret = new StringBuilder(); - ret.append('{'); - ret.append("id:").append(this.id); - ret.append(", host:").append(this.host); - ret.append(", port:").append(this.port); - ret.append('}'); - return ret.toString(); - } - +public final class MongoConfig +{ + private String host; + private String uri; + private int port; + private final String id; + private Mongo mongo; + + MongoConfig(final String id) throws ConfigurationException { + this.id = id; + } + + private synchronized void init() throws ConfigurationException { + if (null == this.mongo) { + try { + MongoURI mongoURI; + if (StringUtils.isNotBlank(this.uri)) { + mongoURI = new MongoURI(this.uri); + } + else { + mongoURI = new MongoURI("mongodb://" + this.host + ":" + this.port); + } + this.mongo = new Mongo(mongoURI); + } + catch (Exception e) { + throw new ConfigurationException("error init mongo[" + this.host + ":" + this.port, e); + } + } + } + + public Mongo getMongo() throws ConfigurationException { + if (null == this.mongo) { + this.init(); + } + return this.mongo; + } + + synchronized void reload(final Node mongoNode) throws ConfigurationException { + final String host = XmlUtils.getNodeValue(mongoNode, "host", "datastore.mongo.db.fangjia.com"); + final int port = XmlUtils.getNodeValueAsInt(mongoNode, "port", 27017); + final String uri = XmlUtils.getNodeValue(mongoNode, "uri", ""); + if (host.equals(this.host) && this.port == port) { + return; + } + this.host = host; + this.port = port; + this.uri = uri; + if (null != this.mongo) { + try { + this.init(); + } + catch (Exception e) { + throw new ConfigurationException("error init mongo[" + host + ":" + port, e); + } + } + } + + @Override + public int hashCode() { + return new HashCodeBuilder().append((Object)this.host).append(this.port).append((Object)this.id).toHashCode(); + } + + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (null == obj) { + return false; + } + if (!(obj instanceof MongoConfig)) { + return false; + } + final MongoConfig other = (MongoConfig)obj; + return new EqualsBuilder().append((Object)this.id, (Object)other.id).append((Object)this.host, (Object)other.host).append(this.port, other.port).isEquals(); + } + + @Override + public String toString() { + final StringBuilder ret = new StringBuilder(); + ret.append('{'); + ret.append("id:").append(this.id); + ret.append(", host:").append(this.host); + ret.append(", port:").append(this.port); + ret.append('}'); + return ret.toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Reloadable.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Reloadable.java index 07bd3e51b..ba98d1688 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Reloadable.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Reloadable.java @@ -2,6 +2,7 @@ package com.fangjia.datastore.config; import com.fangjia.datastore.ConfigurationException; -public interface Reloadable { - void reload() throws ConfigurationException; +public interface Reloadable +{ + void reload() throws ConfigurationException; } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/ServerConfig.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/ServerConfig.java index 9159563dc..3bd447036 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/ServerConfig.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/ServerConfig.java @@ -1,74 +1,72 @@ package com.fangjia.datastore.config; +import com.fangjia.datastore.utils.XmlUtils; +import com.fangjia.datastore.ConfigurationException; import org.dom4j.Node; -import com.fangjia.datastore.ConfigurationException; -import com.fangjia.datastore.utils.XmlUtils; - -public final class ServerConfig { - private int port; - private boolean disableLogging; - private long performanceLogFilterMaxTime; - private int minaReadBufferSize; - private int cursorExpirationMinutes; - private int defaultCursorLimit; - private int maxConnections; - private int triggerBatchSize; - private int reIndexBatchSize; - - ServerConfig() { - super(); - } - - void reload(Node server) throws ConfigurationException { - if (null == server) { - throw new ConfigurationException("no server is configured!"); - } - this.port = XmlUtils.getNodeValueAsInt(server, "port", 27017); - this.disableLogging = XmlUtils.getNodeValueAsBoolean(server, "disableLogging", false); - this.performanceLogFilterMaxTime = XmlUtils.getNodeValueAsLong(server, "performanceLogFilterMaxTime", 1000); - this.minaReadBufferSize = XmlUtils.getNodeValueAsInt(server, "minaReadBufferSize", 20480); - this.cursorExpirationMinutes = XmlUtils.getNodeValueAsInt(server, "cursorExpirationMinutes", 5); - this.defaultCursorLimit = XmlUtils.getNodeValueAsInt(server, "defaultCursorLimit", 20); - this.maxConnections = XmlUtils.getNodeValueAsInt(server, "maxConnections", 10); - this.triggerBatchSize = XmlUtils.getNodeValueAsInt(server, "triggerBatchSize", 200); - this.reIndexBatchSize = XmlUtils.getNodeValueAsInt(server, "reIndexBatchSize", 200); - } - - public int getPort() { - return port; - } - - public boolean isDisableLogging() { - return disableLogging; - } - - public long getPerformanceLogFilterMaxTime() { - return performanceLogFilterMaxTime; - } - - public int getMinaReadBufferSize() { - return minaReadBufferSize; - } - - public int getCursorExpirationMinutes() { - return cursorExpirationMinutes; - } - - public int getDefaultCursorLimit() { - return defaultCursorLimit; - } - - public int getMaxConnections() { - return maxConnections; - } - - public int getTriggerBatchSize() { - return triggerBatchSize; - } - - public int getReIndexBatchSize() { - return reIndexBatchSize; - } - +public final class ServerConfig +{ + private int port; + private boolean disableLogging; + private long performanceLogFilterMaxTime; + private int minaReadBufferSize; + private int cursorExpirationMinutes; + private int defaultCursorLimit; + private int maxConnections; + private int triggerBatchSize; + private int reIndexBatchSize; + + ServerConfig() { + } + + void reload(final Node server) throws ConfigurationException { + if (null == server) { + throw new ConfigurationException("no server is configured!"); + } + this.port = XmlUtils.getNodeValueAsInt(server, "port", 27017); + this.disableLogging = XmlUtils.getNodeValueAsBoolean(server, "disableLogging", false); + this.performanceLogFilterMaxTime = XmlUtils.getNodeValueAsLong(server, "performanceLogFilterMaxTime", 1000L); + this.minaReadBufferSize = XmlUtils.getNodeValueAsInt(server, "minaReadBufferSize", 20480); + this.cursorExpirationMinutes = XmlUtils.getNodeValueAsInt(server, "cursorExpirationMinutes", 5); + this.defaultCursorLimit = XmlUtils.getNodeValueAsInt(server, "defaultCursorLimit", 20); + this.maxConnections = XmlUtils.getNodeValueAsInt(server, "maxConnections", 10); + this.triggerBatchSize = XmlUtils.getNodeValueAsInt(server, "triggerBatchSize", 200); + this.reIndexBatchSize = XmlUtils.getNodeValueAsInt(server, "reIndexBatchSize", 200); + } + + public int getPort() { + return this.port; + } + + public boolean isDisableLogging() { + return this.disableLogging; + } + + public long getPerformanceLogFilterMaxTime() { + return this.performanceLogFilterMaxTime; + } + + public int getMinaReadBufferSize() { + return this.minaReadBufferSize; + } + + public int getCursorExpirationMinutes() { + return this.cursorExpirationMinutes; + } + + public int getDefaultCursorLimit() { + return this.defaultCursorLimit; + } + + public int getMaxConnections() { + return this.maxConnections; + } + + public int getTriggerBatchSize() { + return this.triggerBatchSize; + } + + public int getReIndexBatchSize() { + return this.reIndexBatchSize; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/SolrConfig.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/SolrConfig.java index 283d169e0..f08c37efd 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/SolrConfig.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/SolrConfig.java @@ -1,132 +1,132 @@ package com.fangjia.datastore.config; +import com.fangjia.datastore.ConfigurationException; import org.apache.commons.lang.StringUtils; +import com.fangjia.datastore.utils.XmlUtils; import org.dom4j.Node; -import com.fangjia.datastore.ConfigurationException; -import com.fangjia.datastore.utils.XmlUtils; - -public class SolrConfig { - private final String id; - private String host; - private String home; - private final DataStoreConfig dataStoreConfig; - private String shards; - private String slaves; - private SolrConfig[] shardConfigs; - private SolrConfig[] slaveConfigs; - private String filter; - - SolrConfig(DataStoreConfig dataStoreConfig, String id) { - this.dataStoreConfig = dataStoreConfig; - this.id = id; - } - - synchronized void reload(Node solrNode) throws ConfigurationException { - this.host = removeLastSlash(XmlUtils.getNodeValue(solrNode, "host", null)); - this.shards = XmlUtils.getNodeValue(solrNode, "shards", null); - this.slaves = XmlUtils.getNodeValue(solrNode, "slaves", null); - this.home = XmlUtils.getNodeValue(solrNode, "home", null); - if (StringUtils.isNotEmpty(shards) && StringUtils.isEmpty(home)) { - throw new ConfigurationException("[" + id + "] invalid home attribute:" + home); - } - if (StringUtils.isEmpty(shards) && StringUtils.isEmpty(slaves) && StringUtils.isEmpty(host)) { - throw new ConfigurationException("[" + id + "] invalid host attribute:" + host); - } - this.filter = removeLastSlash(XmlUtils.getNodeValue(solrNode, "filter", null)); - } - - private SolrConfig[] getConfigs(String ids) { - SolrConfig[] ret; - if (StringUtils.isBlank(ids)) { - ret = new SolrConfig[0]; - } else { - String[] idSplit = ids.split(","); - ret = new SolrConfig[idSplit.length]; - for (int i = 0; i < idSplit.length; i++) { - ret[i] = dataStoreConfig.getSolrConfigById(idSplit[i]); - } - } - return ret; - } - - public SolrConfig[] getShardsConfigs() { - if (null == shardConfigs) { - shardConfigs = this.getConfigs(shards); - } - return shardConfigs; - } - - public SolrConfig[] getSlaveConfigs() { - if (null == slaveConfigs) { - slaveConfigs = this.getConfigs(slaves); - } - return slaveConfigs; - } - - private String removeLastSlash(String s) { - if (StringUtils.isEmpty(s)) { - return s; - } - if (s.charAt(s.length() - 1) == '/') { - return s.substring(0, s.length() - 1); - } - return s; - } - - public String getShardsParam(String coreName) { - SolrConfig[] shardConfigs = getShardsConfigs(); - if (shardConfigs.length == 0) { - return ""; - } - StringBuilder ret = new StringBuilder(); - for (int i = 0; i < shardConfigs.length; i++) { - ret.append(shardConfigs[i].host).append('/').append(coreName); - if (i < shardConfigs.length - 1) { - ret.append(','); - } - } - return ret.toString(); - } - - public String getId() { - return id; - } - - public String getHost() { - return host; - } - - public DataStoreConfig getDataStoreConfig() { - return dataStoreConfig; - } - - public String getShards() { - return shards; - } - - public String getFilter() { - return filter; - } - - public String getHome() { - return home; - } - - @Override - public String toString() { - StringBuilder ret = new StringBuilder(); - ret.append('{'); - ret.append("id: ").append(this.id); - ret.append(", host: ").append(this.host); - ret.append(", shards: ").append(this.shards); - ret.append(", slaves: ").append(this.slaves); - ret.append('}'); - return ret.toString(); - } - - public String getSlaves() { - return slaves; - } - +public class SolrConfig +{ + private final String id; + private String host; + private String home; + private final DataStoreConfig dataStoreConfig; + private String shards; + private String slaves; + private SolrConfig[] shardConfigs; + private SolrConfig[] slaveConfigs; + private String filter; + + SolrConfig(final DataStoreConfig dataStoreConfig, final String id) { + this.dataStoreConfig = dataStoreConfig; + this.id = id; + } + + synchronized void reload(final Node solrNode) throws ConfigurationException { + this.host = this.removeLastSlash(XmlUtils.getNodeValue(solrNode, "host", null)); + this.shards = XmlUtils.getNodeValue(solrNode, "shards", null); + this.slaves = XmlUtils.getNodeValue(solrNode, "slaves", null); + this.home = XmlUtils.getNodeValue(solrNode, "home", null); + if (StringUtils.isNotEmpty(this.shards) && StringUtils.isEmpty(this.home)) { + throw new ConfigurationException("[" + this.id + "] invalid home attribute:" + this.home); + } + if (StringUtils.isEmpty(this.shards) && StringUtils.isEmpty(this.slaves) && StringUtils.isEmpty(this.host)) { + throw new ConfigurationException("[" + this.id + "] invalid host attribute:" + this.host); + } + this.filter = this.removeLastSlash(XmlUtils.getNodeValue(solrNode, "filter", null)); + } + + private SolrConfig[] getConfigs(final String ids) { + SolrConfig[] ret; + if (StringUtils.isBlank(ids)) { + ret = new SolrConfig[0]; + } + else { + final String[] idSplit = ids.split(","); + ret = new SolrConfig[idSplit.length]; + for (int i = 0; i < idSplit.length; ++i) { + ret[i] = this.dataStoreConfig.getSolrConfigById(idSplit[i]); + } + } + return ret; + } + + public SolrConfig[] getShardsConfigs() { + if (null == this.shardConfigs) { + this.shardConfigs = this.getConfigs(this.shards); + } + return this.shardConfigs; + } + + public SolrConfig[] getSlaveConfigs() { + if (null == this.slaveConfigs) { + this.slaveConfigs = this.getConfigs(this.slaves); + } + return this.slaveConfigs; + } + + private String removeLastSlash(final String s) { + if (StringUtils.isEmpty(s)) { + return s; + } + if (s.charAt(s.length() - 1) == '/') { + return s.substring(0, s.length() - 1); + } + return s; + } + + public String getShardsParam(final String coreName) { + final SolrConfig[] shardConfigs = this.getShardsConfigs(); + if (shardConfigs.length == 0) { + return ""; + } + final StringBuilder ret = new StringBuilder(); + for (int i = 0; i < shardConfigs.length; ++i) { + ret.append(shardConfigs[i].host).append('/').append(coreName); + if (i < shardConfigs.length - 1) { + ret.append(','); + } + } + return ret.toString(); + } + + public String getId() { + return this.id; + } + + public String getHost() { + return this.host; + } + + public DataStoreConfig getDataStoreConfig() { + return this.dataStoreConfig; + } + + public String getShards() { + return this.shards; + } + + public String getFilter() { + return this.filter; + } + + public String getHome() { + return this.home; + } + + @Override + public String toString() { + final StringBuilder ret = new StringBuilder(); + ret.append('{'); + ret.append("id: ").append(this.id); + ret.append(", host: ").append(this.host); + ret.append(", shards: ").append(this.shards); + ret.append(", slaves: ").append(this.slaves); + ret.append('}'); + return ret.toString(); + } + + public String getSlaves() { + return this.slaves; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/StoreConfig.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/StoreConfig.java index fbeed7114..2d289057a 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/StoreConfig.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/StoreConfig.java @@ -1,229 +1,208 @@ package com.fangjia.datastore.config; -import java.util.Collection; import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.commons.lang.StringUtils; +import java.util.Collection; import org.bson.types.ObjectId; +import java.util.Iterator; +import java.util.List; +import com.fangjia.datastore.utils.XmlUtils; +import com.fangjia.datastore.ConfigurationException; +import org.apache.commons.lang.StringUtils; import org.dom4j.Element; import org.dom4j.Node; +import java.util.HashMap; +import java.util.Map; -import com.fangjia.datastore.ConfigurationException; -import com.fangjia.datastore.utils.XmlUtils; - -public final class StoreConfig { - - private final String name; - private int solrMaxTotalConnections; - private int solrMaxConnectionsPerHost; - private SolrConfig[] masterSolrConfigs; - private SolrConfig masterSolrConfig; - private SolrConfig slaveSolrConfig; - private MongoConfig mongoConfig; - private boolean textFieldToGridFS; - private final Trigger trigger; - private final Map fields; - private final DataStoreConfig dataStoreConfig; - - StoreConfig(DataStoreConfig dataStoreConfig, String name) { - this.dataStoreConfig = dataStoreConfig; - fields = new HashMap(); - trigger = new Trigger(); - this.name = name; - } - - synchronized void reload(Node storeNode) throws ConfigurationException { - // init solr - String solrMastersAttr = ((Element) storeNode).attributeValue("solrMasters"); - if (StringUtils.isEmpty(solrMastersAttr)) { - throw new ConfigurationException("[" + name + "] invalid solrMasters attribute:" + solrMastersAttr); - } - String[] solrMasterIds = solrMastersAttr.split(","); - if (solrMasterIds.length > 2) { - throw new ConfigurationException("[" + name + "] too many solr server configured:" + solrMastersAttr); - } - masterSolrConfigs = new SolrConfig[solrMasterIds.length]; - for (int i = 0; i < solrMasterIds.length; i++) { - masterSolrConfigs[i] = dataStoreConfig.getSolrConfigById(solrMasterIds[i]); - if (null == masterSolrConfigs[i]) { - throw new ConfigurationException("[" + name + "] unknown solr id: " + solrMasterIds[i]); - } - } - - String solrSlaveAttr = System.getProperty("datastore." + name + ".solrSlave", ((Element) storeNode) - .attributeValue("solrSlave")); - if (StringUtils.isEmpty(solrSlaveAttr)) { - throw new ConfigurationException("[" + name + "] invalid solrSlave attribute:" + solrSlaveAttr); - } - slaveSolrConfig = dataStoreConfig.getSolrConfigById(solrSlaveAttr); - if (null == slaveSolrConfig) { - throw new ConfigurationException("[" + name + "] unknown solr id: " + solrSlaveAttr); - } - - String solrMasterAttr = System.getProperty("datastore." + name + ".solrMaster", ((Element) storeNode) - .attributeValue("solrMaster")); - if (StringUtils.isEmpty(solrMasterAttr)) { - throw new ConfigurationException("[" + name + "] invalid solrMaster attribute:" + solrMasterAttr); - } - masterSolrConfig = dataStoreConfig.getSolrConfigById(solrMasterAttr); - if (null == masterSolrConfig) { - throw new ConfigurationException("[" + name + "] unknown solr id: " + solrMasterAttr); - } - - if (slaveSolrConfig.getShardsConfigs().length != masterSolrConfig.getShardsConfigs().length) { - throw new ConfigurationException("[" + name + "] master and slave shard count must equals !"); - } - - // solr connections defaults - this.solrMaxConnectionsPerHost = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxConnectionsPerHost", 300); - if (slaveSolrConfig.getShardsConfigs().length > 0) { - this.solrMaxTotalConnections = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxTotalConnections", slaveSolrConfig - .getShardsConfigs().length - * solrMaxConnectionsPerHost); - } else { - this.solrMaxTotalConnections = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxTotalConnections", - solrMaxConnectionsPerHost); - } - - // init mongo - String mongoId = ((Element) storeNode).attributeValue("mongo"); - MongoConfig mongoConfig = dataStoreConfig.getMongoConfigById(mongoId); - if (null == mongoConfig) { - throw new ConfigurationException("[" + name + "] unknown mongo id: " + mongoId); - } - this.mongoConfig = mongoConfig; - - // init other config - this.textFieldToGridFS = XmlUtils.getNodeValueAsBoolean(storeNode, "textFieldToGridFS", false); - - // init fields - Map newFields = new HashMap(); - List fieldNodes = storeNode.selectNodes("schema//field"); - for (Node fieldNode : fieldNodes) { - Element element = (Element) fieldNode; - String fieldName = element.attributeValue("name"); - Field field = getOrCreateField(fieldName); - field.setName(fieldName); - field.setRequired(Boolean.parseBoolean(element.attributeValue("required"))); - field.setType(FieldTypeEnum.parse(element.attributeValue("type"))); - newFields.put(fieldName, field); - } - this.fields.clear(); - this.fields.putAll(newFields); - - // init triggers - Element triggerNode = (Element) storeNode.selectSingleNode("trigger"); - if(triggerNode != null) { - trigger.setInterface(triggerNode.attributeValue("interface")); - } - - } - - public Trigger getTrigger() { - return this.trigger; - } - - public Field getField(String fieldName) { - return this.fields.get(fieldName); - } - - public boolean containsField(String fieldName) { - return this.fields.containsKey(fieldName); - } - - public boolean isFieldValueLegal(String fieldName, Object value) { - if ("_id".equals(fieldName)) { - return value instanceof ObjectId; - } - if (fields.containsKey(fieldName)) { - Field field = fields.get(fieldName); - return field.validate(value); - } - return false; - } - - public boolean isBigField(String fieldName) { - return fields.get(fieldName).isBigField(); - } - - public Collection getFields() { - return Collections.unmodifiableCollection(this.fields.values()); - } - - synchronized Field getOrCreateField(String fieldName) { - if (StringUtils.isEmpty(fieldName)) { - throw new IllegalArgumentException("fieldName can not be null or empty!"); - } - Field ret = this.fields.get(fieldName); - if (null == ret) { - ret = new Field(); - ret.setName(fieldName); - this.fields.put(fieldName, ret); - } - return ret; - } - - public String getName() { - return name; - } - - public MongoConfig getMongoConfig() { - return mongoConfig; - } - - public DataStoreConfig getDataStoreConfig() { - return dataStoreConfig; - } - - public boolean isTextFieldToGridFS() { - return textFieldToGridFS; - } - - public SolrConfig getSlaveSolrConfig() { - return slaveSolrConfig; - } - - public SolrConfig[] getMasterSolrConfigs() { - return masterSolrConfigs; - } - - public SolrConfig getMasterSolrConfig() { - return masterSolrConfig; - } - - public int getSolrMaxTotalConnections() { - return solrMaxTotalConnections; - } - - public int getSolrMaxConnectionsPerHost() { - return solrMaxConnectionsPerHost; - } - - @Override - public String toString() { - StringBuilder ret = new StringBuilder(); - ret.append("\n{"); - ret.append("\n\tname: ").append(name); - ret.append(",\n\tmasterSolrConfig: ").append(masterSolrConfig); - ret.append(",\n\tslaveSolrConfig: ").append(this.slaveSolrConfig); - ret.append(",\n\tmongoConfig: ").append(mongoConfig); - ret.append(",\n\tsolrMaxTotalConnections: ").append(solrMaxTotalConnections); - ret.append(",\n\tsolrMaxConnectionsPerHost: ").append(this.solrMaxConnectionsPerHost); - ret.append(",\n\ttextFieldToGridFS: ").append(this.textFieldToGridFS); - ret.append(",\n\ttrigger: ").append(trigger); - ret.append(",\n\tmasterSolrConfigs: [\n"); - for (int i = 0; i < this.masterSolrConfigs.length; i++) { - ret.append('\t').append('\t').append(this.masterSolrConfigs[i]); - if (i < this.masterSolrConfigs.length - 1) { - ret.append(','); - } - ret.append('\n'); - } - ret.append("\t]\n}"); - return ret.toString(); - } - +public final class StoreConfig +{ + private final String name; + private int solrMaxTotalConnections; + private int solrMaxConnectionsPerHost; + private SolrConfig[] masterSolrConfigs; + private SolrConfig masterSolrConfig; + private SolrConfig slaveSolrConfig; + private MongoConfig mongoConfig; + private boolean textFieldToGridFS; + private final Trigger trigger; + private final Map fields; + private final DataStoreConfig dataStoreConfig; + + StoreConfig(final DataStoreConfig dataStoreConfig, final String name) { + this.dataStoreConfig = dataStoreConfig; + this.fields = new HashMap(); + this.trigger = new Trigger(); + this.name = name; + } + + synchronized void reload(final Node storeNode) throws ConfigurationException { + final String solrMastersAttr = ((Element)storeNode).attributeValue("solrMasters"); + if (StringUtils.isEmpty(solrMastersAttr)) { + throw new ConfigurationException("[" + this.name + "] invalid solrMasters attribute:" + solrMastersAttr); + } + final String[] solrMasterIds = solrMastersAttr.split(","); + if (solrMasterIds.length > 2) { + throw new ConfigurationException("[" + this.name + "] too many solr server configured:" + solrMastersAttr); + } + this.masterSolrConfigs = new SolrConfig[solrMasterIds.length]; + for (int i = 0; i < solrMasterIds.length; ++i) { + this.masterSolrConfigs[i] = this.dataStoreConfig.getSolrConfigById(solrMasterIds[i]); + if (null == this.masterSolrConfigs[i]) { + throw new ConfigurationException("[" + this.name + "] unknown solr id: " + solrMasterIds[i]); + } + } + final String solrSlaveAttr = System.getProperty("datastore." + this.name + ".solrSlave", ((Element)storeNode).attributeValue("solrSlave")); + if (StringUtils.isEmpty(solrSlaveAttr)) { + throw new ConfigurationException("[" + this.name + "] invalid solrSlave attribute:" + solrSlaveAttr); + } + this.slaveSolrConfig = this.dataStoreConfig.getSolrConfigById(solrSlaveAttr); + if (null == this.slaveSolrConfig) { + throw new ConfigurationException("[" + this.name + "] unknown solr id: " + solrSlaveAttr); + } + final String solrMasterAttr = System.getProperty("datastore." + this.name + ".solrMaster", ((Element)storeNode).attributeValue("solrMaster")); + if (StringUtils.isEmpty(solrMasterAttr)) { + throw new ConfigurationException("[" + this.name + "] invalid solrMaster attribute:" + solrMasterAttr); + } + this.masterSolrConfig = this.dataStoreConfig.getSolrConfigById(solrMasterAttr); + if (null == this.masterSolrConfig) { + throw new ConfigurationException("[" + this.name + "] unknown solr id: " + solrMasterAttr); + } + if (this.slaveSolrConfig.getShardsConfigs().length != this.masterSolrConfig.getShardsConfigs().length) { + throw new ConfigurationException("[" + this.name + "] master and slave shard count must equals !"); + } + this.solrMaxConnectionsPerHost = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxConnectionsPerHost", 300); + if (this.slaveSolrConfig.getShardsConfigs().length > 0) { + this.solrMaxTotalConnections = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxTotalConnections", this.slaveSolrConfig.getShardsConfigs().length * this.solrMaxConnectionsPerHost); + } + else { + this.solrMaxTotalConnections = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxTotalConnections", this.solrMaxConnectionsPerHost); + } + final String mongoId = ((Element)storeNode).attributeValue("mongo"); + final MongoConfig mongoConfig = this.dataStoreConfig.getMongoConfigById(mongoId); + if (null == mongoConfig) { + throw new ConfigurationException("[" + this.name + "] unknown mongo id: " + mongoId); + } + this.mongoConfig = mongoConfig; + this.textFieldToGridFS = XmlUtils.getNodeValueAsBoolean(storeNode, "textFieldToGridFS", false); + final Map newFields = new HashMap(); + final List fieldNodes = (List)storeNode.selectNodes("schema//field"); + for (final Node fieldNode : fieldNodes) { + final Element element = (Element)fieldNode; + final String fieldName = element.attributeValue("name"); + final Field field = this.getOrCreateField(fieldName); + field.setName(fieldName); + field.setRequired(Boolean.parseBoolean(element.attributeValue("required"))); + field.setType(FieldTypeEnum.parse(element.attributeValue("type"))); + newFields.put(fieldName, field); + } + this.fields.clear(); + this.fields.putAll(newFields); + final Element triggerNode = (Element)storeNode.selectSingleNode("trigger"); + if (triggerNode != null) { + this.trigger.setInterface(triggerNode.attributeValue("interface")); + } + } + + public Trigger getTrigger() { + return this.trigger; + } + + public Field getField(final String fieldName) { + return this.fields.get(fieldName); + } + + public boolean containsField(final String fieldName) { + return this.fields.containsKey(fieldName); + } + + public boolean isFieldValueLegal(final String fieldName, final Object value) { + if ("_id".equals(fieldName)) { + return value instanceof ObjectId; + } + if (this.fields.containsKey(fieldName)) { + final Field field = this.fields.get(fieldName); + return field.validate(value); + } + return false; + } + + public boolean isBigField(final String fieldName) { + return this.fields.get(fieldName).isBigField(); + } + + public Collection getFields() { + return Collections.unmodifiableCollection((Collection)this.fields.values()); + } + + synchronized Field getOrCreateField(final String fieldName) { + if (StringUtils.isEmpty(fieldName)) { + throw new IllegalArgumentException("fieldName can not be null or empty!"); + } + Field ret = this.fields.get(fieldName); + if (null == ret) { + ret = new Field(); + ret.setName(fieldName); + this.fields.put(fieldName, ret); + } + return ret; + } + + public String getName() { + return this.name; + } + + public MongoConfig getMongoConfig() { + return this.mongoConfig; + } + + public DataStoreConfig getDataStoreConfig() { + return this.dataStoreConfig; + } + + public boolean isTextFieldToGridFS() { + return this.textFieldToGridFS; + } + + public SolrConfig getSlaveSolrConfig() { + return this.slaveSolrConfig; + } + + public SolrConfig[] getMasterSolrConfigs() { + return this.masterSolrConfigs; + } + + public SolrConfig getMasterSolrConfig() { + return this.masterSolrConfig; + } + + public int getSolrMaxTotalConnections() { + return this.solrMaxTotalConnections; + } + + public int getSolrMaxConnectionsPerHost() { + return this.solrMaxConnectionsPerHost; + } + + @Override + public String toString() { + final StringBuilder ret = new StringBuilder(); + ret.append("\n{"); + ret.append("\n\tname: ").append(this.name); + ret.append(",\n\tmasterSolrConfig: ").append(this.masterSolrConfig); + ret.append(",\n\tslaveSolrConfig: ").append(this.slaveSolrConfig); + ret.append(",\n\tmongoConfig: ").append(this.mongoConfig); + ret.append(",\n\tsolrMaxTotalConnections: ").append(this.solrMaxTotalConnections); + ret.append(",\n\tsolrMaxConnectionsPerHost: ").append(this.solrMaxConnectionsPerHost); + ret.append(",\n\ttextFieldToGridFS: ").append(this.textFieldToGridFS); + ret.append(",\n\ttrigger: ").append(this.trigger); + ret.append(",\n\tmasterSolrConfigs: [\n"); + for (int i = 0; i < this.masterSolrConfigs.length; ++i) { + ret.append('\t').append('\t').append(this.masterSolrConfigs[i]); + if (i < this.masterSolrConfigs.length - 1) { + ret.append(','); + } + ret.append('\n'); + } + ret.append("\t]\n}"); + return ret.toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Trigger.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Trigger.java index 8608fffe0..08ec5471d 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Trigger.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Trigger.java @@ -1,49 +1,49 @@ package com.fangjia.datastore.config; -import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.commons.lang.StringUtils; -public final class Trigger { - private String interface_; - - Trigger() { - } - - public String getInterface() { - return interface_; - } - - void setInterface(String interface_) { - if (StringUtils.isEmpty(interface_) || !interface_.startsWith("http://")) { - throw new IllegalArgumentException("invalid trigger interface: " + interface_); - } - this.interface_ = interface_; - } - - @Override - public int hashCode() { - return new HashCodeBuilder().append(this.interface_).toHashCode(); - } - - @Override - public boolean equals(Object obj) { - if (null == obj) { - return false; - } - if (this == obj) { - return true; - } - if (!(obj instanceof Trigger)) { - return false; - } - Trigger other = (Trigger) obj; - return new EqualsBuilder().append(this.interface_, other.interface_).isEquals(); - } - - @Override - public String toString() { - return new StringBuilder().append("{interface=").append(interface_).append("}").toString(); - } - +public final class Trigger +{ + private String interface_; + + Trigger() { + } + + public String getInterface() { + return this.interface_; + } + + void setInterface(final String interface_) { + if (StringUtils.isEmpty(interface_) || !interface_.startsWith("http://")) { + throw new IllegalArgumentException("invalid trigger interface: " + interface_); + } + this.interface_ = interface_; + } + + @Override + public int hashCode() { + return new HashCodeBuilder().append((Object)this.interface_).toHashCode(); + } + + @Override + public boolean equals(final Object obj) { + if (null == obj) { + return false; + } + if (this == obj) { + return true; + } + if (!(obj instanceof Trigger)) { + return false; + } + final Trigger other = (Trigger)obj; + return new EqualsBuilder().append((Object)this.interface_, (Object)other.interface_).isEquals(); + } + + @Override + public String toString() { + return "{interface=" + this.interface_ + "}"; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/DeleteMessageHandler.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/DeleteMessageHandler.java index 5ed531fa7..2a853a2bc 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/DeleteMessageHandler.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/DeleteMessageHandler.java @@ -1,19 +1,20 @@ package com.fangjia.datastore.handler; +import com.fangjia.datastore.mongo.protocol.MongoMessage; import com.fangjia.datastore.DataStore; import com.fangjia.datastore.DataStoreContext; import com.fangjia.datastore.mongo.protocol.DeleteMessage; -public class DeleteMessageHandler extends MessageHandler { - - public DeleteMessageHandler(DataStoreContext dataStoreContext) { - super(dataStoreContext); - } - - public void messageReceived(DeleteMessage message,String ip) throws Exception { - assertNotReadOnly(); - DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName()); - dataStore.remove(message.getSelector()); - } - +public class DeleteMessageHandler extends MessageHandler +{ + public DeleteMessageHandler(final DataStoreContext dataStoreContext) { + super(dataStoreContext); + } + + @Override + public void messageReceived(final DeleteMessage message) throws Exception { + this.assertNotReadOnly(); + final DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName()); + dataStore.remove(message.getSelector()); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/GetMoreMessageHandler.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/GetMoreMessageHandler.java index 4baba97ee..a9c78e1ba 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/GetMoreMessageHandler.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/GetMoreMessageHandler.java @@ -1,54 +1,59 @@ package com.fangjia.datastore.handler; -import java.util.List; - -import org.bson.BSONObject; - +import com.fangjia.datastore.mongo.protocol.MongoMessage; +import com.mongodb.DBObject; +import com.fangjia.datastore.QueryResponse; +import com.fangjia.datastore.mongo.protocol.QueryMessage; import com.fangjia.datastore.Cursor; import com.fangjia.datastore.DataStore; -import com.fangjia.datastore.DataStoreContext; -import com.fangjia.datastore.QueryResponse; -import com.fangjia.datastore.mongo.protocol.GetMoreMessage; -import com.fangjia.datastore.mongo.protocol.QueryMessage; import com.fangjia.datastore.mongo.protocol.ReplyMessage; +import org.bson.BSONObject; +import java.util.List; +import com.fangjia.datastore.DataStoreContext; +import com.fangjia.datastore.mongo.protocol.GetMoreMessage; -public class GetMoreMessageHandler extends MessageHandler { - - public GetMoreMessageHandler(DataStoreContext dataStoreContext) { - super(dataStoreContext); - } - - public void messageReceived(GetMoreMessage message) throws Exception { - String fullCollectionName = message.getFullCollectionName(); - if(fullCollectionName.contains("$cmd")){ - ReplyMessage reply = prepareReplyMessage(message.getMessageHeader().getRequestID(), (List) null); - reply.setStartingFrom(0); - this.writeMessage(reply); - this.dataStoreContext.removeCursor(message.getCursorID()); - return; - } - DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName()); - Cursor cursor = this.dataStoreContext.getCursor(message.getCursorID()); - if (null == cursor) { - ReplyMessage reply = prepareReplyMessage(message.getMessageHeader().getRequestID(), (List) null); - reply.setStartingFrom(0); - this.writeMessage(reply); - } else { - QueryMessage queryMessage = cursor.getQueryMessage(); - QueryResponse response = dataStore.find(queryMessage.getRealQuery(), queryMessage.getReturnFieldSelector(), - queryMessage.getOrderByQuery(), queryMessage.getFacetQuery(), cursor.nextSkip(), - message.getNumberToReturn() - 1, queryMessage.isBulkQuery(), queryMessage.isFromMaster()); - ReplyMessage reply = prepareReplyMessage(message.getMessageHeader().getRequestID(), response.getDocList()); - reply.setStartingFrom(cursor.getSkip()); - if (response.hasMore()) { - // cursor.setTotalCount(response.getNumFound()); - // cursor.setLimit((int) response.getNumReturned()); - // this.dataStoreContext.addCursor(cursor); - // reply.setCursorID(cursor.getId()); - } else { - this.dataStoreContext.removeCursor(cursor.getId()); - } - this.writeMessage(reply); - } - } +public class GetMoreMessageHandler extends MessageHandler +{ + public GetMoreMessageHandler(final DataStoreContext dataStoreContext) { + super(dataStoreContext); + } + + @Override + public void messageReceived(final GetMoreMessage message) throws Exception { + final String fullCollectionName = message.getFullCollectionName(); + if (fullCollectionName.contains("$cmd")) { + final ReplyMessage reply = this.prepareReplyMessage(message.getMessageHeader().getRequestID(), (List)null); + reply.setStartingFrom(0); + this.writeMessage(reply); + this.dataStoreContext.removeCursor(message.getCursorID()); + return; + } + final DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName()); + final Cursor cursor = this.dataStoreContext.getCursor(message.getCursorID()); + if (null == cursor) { + final ReplyMessage reply2 = this.prepareReplyMessage(message.getMessageHeader().getRequestID(), (List)null); + reply2.setStartingFrom(0); + this.writeMessage(reply2); + } + else { + final QueryMessage queryMessage = cursor.getQueryMessage(); + final QueryResponse response = dataStore.find(queryMessage.getRealQuery(), queryMessage.getReturnFieldSelector(), queryMessage.getOrderByQuery(), queryMessage.getFacetQuery(), cursor.nextSkip(), this.useCursor() ? message.getNumberToReturn() : (message.getNumberToReturn() - 1), queryMessage.isBulkQuery(), queryMessage.isFromMaster()); + final List doc = response.getDocList(); + if (this.useCursor()) { + doc.remove(0); + } + final ReplyMessage reply3 = this.prepareReplyMessage(message.getMessageHeader().getRequestID(), (List)doc); + reply3.setStartingFrom(cursor.getSkip()); + if (response.hasMore() && this.useCursor()) { + cursor.setTotalCount(response.getNumFound()); + cursor.setLimit(response.getNumReturned()); + this.dataStoreContext.addCursor(cursor); + reply3.setCursorID(cursor.getId()); + } + else { + this.dataStoreContext.removeCursor(cursor.getId()); + } + this.writeMessage(reply3); + } + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/InsertMessageHandler.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/InsertMessageHandler.java index 27abe5e4f..54d6c41d9 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/InsertMessageHandler.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/InsertMessageHandler.java @@ -1,115 +1,117 @@ package com.fangjia.datastore.handler; -import java.util.ArrayList; -import java.util.List; - +import com.fangjia.datastore.mongo.protocol.MongoMessage; +import java.util.Iterator; import org.apache.commons.lang.StringUtils; -import org.bson.BSONObject; - +import java.util.ArrayList; +import com.fangjia.datastore.solr.Index; +import java.util.Collection; import com.fangjia.datastore.DataStore; +import org.bson.BSONObject; +import java.util.List; import com.fangjia.datastore.DataStoreContext; import com.fangjia.datastore.mongo.protocol.InsertMessage; -import com.fangjia.datastore.solr.Index; -import com.fangjia.datastore.solr.Index.IndexTypeEnum; -public class InsertMessageHandler extends MessageHandler { - - public InsertMessageHandler(DataStoreContext dataStoreContext) { - super(dataStoreContext); - } - - public void messageReceived(InsertMessage message,String ip) throws Exception { - if (message.getFullCollectionName().endsWith("system.indexes")) { - ensureIndex(message); - } else { - insert(message); - } - } - - /** - * db.foo.insert() - * - * @param message - * @throws Exception - */ - private void insert(InsertMessage message) throws Exception { - assertNotReadOnly(); - DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName()); - dataStore.save(message.getDocuments()); - } - - /** - * db.foo.ensureIndex() - * - * @param message - * @throws Exception - */ - private void ensureIndex(InsertMessage message) throws Exception { - assertNotReadOnly(); - DataStore dataStore = this.dataStoreContext.getDataStore(String.valueOf(message.getDocuments().get(0).get("ns"))); - List indexList = addToIndexList(message.getDocuments()); - dataStore.ensureIndex(indexList); - } - - private List addToIndexList(List docList) { - if (null == docList || docList.isEmpty()) { - return null; - } - List indexList = new ArrayList(); - for (BSONObject bson : docList) { - BSONObject keys = (BSONObject) bson.get("key"); - if (null == keys) { - continue; - } - for (String key : keys.keySet()) { - Object value = keys.get(key); - if (value instanceof String) { - Index index = new Index(key, IndexTypeEnum.parse((String) value)); - indexList.add(index); - } else if (value instanceof BSONObject) { - // TODO: - Index index = new Index(); - index.setName(key); - for (String k : ((BSONObject) value).keySet()) { - Object v = ((BSONObject) value).get(k); - if (StringUtils.equalsIgnoreCase(k, "type")) { - if (v instanceof String) { - index.setType(IndexTypeEnum.parse((String) v)); - } - } else if (StringUtils.equalsIgnoreCase(k, "stored")) { - if (v instanceof Boolean) { - index.setStored((Boolean) v); - } - } else if (StringUtils.equalsIgnoreCase(k, "indexed")) { - if (v instanceof Boolean) { - index.setIndexed((Boolean) v); - } - - } else if (StringUtils.equalsIgnoreCase(k, "multiValued")) { - if (v instanceof Boolean) { - index.setMultiValued((Boolean) v); - } - } else if (StringUtils.equalsIgnoreCase(k, "dynamic")) { - if (v instanceof Boolean) { - index.setDynamic((Boolean) v); - } - } else if (StringUtils.equalsIgnoreCase(k, "comments")) { - if (v instanceof String) { - index.setComments((String) v); - } - } - } - if (index.getType() == null) { - throw new IllegalArgumentException("illegal schema field: type"); - } - indexList.add(index); - } else { - if (logger.isErrorEnabled()) { - logger.error("invalid key value of ensureIndex:{}", value); - } - } - } - } - return indexList; - } +public class InsertMessageHandler extends MessageHandler +{ + public InsertMessageHandler(final DataStoreContext dataStoreContext) { + super(dataStoreContext); + } + + @Override + public void messageReceived(final InsertMessage message) throws Exception { + if (message.getFullCollectionName().endsWith("system.indexes")) { + this.ensureIndex(message); + } + else { + this.insert(message); + } + } + + private void insert(final InsertMessage message) throws Exception { + this.assertNotReadOnly(); + final DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName()); + dataStore.save(message.getDocuments()); + } + + private void ensureIndex(final InsertMessage message) throws Exception { + this.assertNotReadOnly(); + final DataStore dataStore = this.dataStoreContext.getDataStore(String.valueOf(message.getDocuments().get(0).get("ns"))); + final List indexList = this.addToIndexList(message.getDocuments()); + dataStore.ensureIndex(indexList); + } + + private List addToIndexList(final List docList) { + if (null == docList || docList.isEmpty()) { + return null; + } + final List indexList = new ArrayList(); + for (final BSONObject bson : docList) { + final BSONObject keys = (BSONObject)bson.get("key"); + if (null == keys) { + continue; + } + for (final String key : keys.keySet()) { + final Object value = keys.get(key); + if (value instanceof String) { + final Index index = new Index(key, Index.IndexTypeEnum.parse((String)value)); + indexList.add(index); + } + else if (value instanceof BSONObject) { + final Index index = new Index(); + index.setName(key); + for (final String k : ((BSONObject)value).keySet()) { + final Object v = ((BSONObject)value).get(k); + if (StringUtils.equalsIgnoreCase(k, "type")) { + if (!(v instanceof String)) { + continue; + } + index.setType(Index.IndexTypeEnum.parse((String)v)); + } + else if (StringUtils.equalsIgnoreCase(k, "stored")) { + if (!(v instanceof Boolean)) { + continue; + } + index.setStored((Boolean) v); + } + else if (StringUtils.equalsIgnoreCase(k, "indexed")) { + if (!(v instanceof Boolean)) { + continue; + } + index.setIndexed((Boolean)v); + } + else if (StringUtils.equalsIgnoreCase(k, "multiValued")) { + if (!(v instanceof Boolean)) { + continue; + } + index.setMultiValued((Boolean)v); + } + else if (StringUtils.equalsIgnoreCase(k, "dynamic")) { + if (!(v instanceof Boolean)) { + continue; + } + index.setDynamic((Boolean)v); + } + else { + if (!StringUtils.equalsIgnoreCase(k, "comments") || !(v instanceof String)) { + continue; + } + index.setComments((String)v); + } + } + if (index.getType() == null) { + throw new IllegalArgumentException("illegal schema field: type"); + } + indexList.add(index); + } + else { + if (!this.logger.isErrorEnabled()) { + continue; + } + this.logger.error("invalid key value of ensureIndex:{}", value); + } + } + } + return indexList; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/KillCursorsMessageHandler.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/KillCursorsMessageHandler.java index a5104a111..8591eebc3 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/KillCursorsMessageHandler.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/KillCursorsMessageHandler.java @@ -1,23 +1,24 @@ package com.fangjia.datastore.handler; +import com.fangjia.datastore.mongo.protocol.MongoMessage; +import java.util.Iterator; import java.util.List; - import com.fangjia.datastore.DataStoreContext; import com.fangjia.datastore.mongo.protocol.KillCursorsMessage; -public class KillCursorsMessageHandler extends MessageHandler { - - public KillCursorsMessageHandler(DataStoreContext dataStoreContext) { - super(dataStoreContext); - } - - public void messageReceived(KillCursorsMessage message,String ip) { - List cursorIds = message.getCursorIDs(); - if (cursorIds != null) { - for (Long cursorId : cursorIds) { - this.dataStoreContext.removeCursor(cursorId); - } - } - } - +public class KillCursorsMessageHandler extends MessageHandler +{ + public KillCursorsMessageHandler(final DataStoreContext dataStoreContext) { + super(dataStoreContext); + } + + @Override + public void messageReceived(final KillCursorsMessage message) { + final List cursorIds = message.getCursorIDs(); + if (cursorIds != null) { + for (final Long cursorId : cursorIds) { + this.dataStoreContext.removeCursor(cursorId); + } + } + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/MessageHandler.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/MessageHandler.java index a2d5cc6e0..9f4a4e995 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/MessageHandler.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/MessageHandler.java @@ -1,184 +1,180 @@ package com.fangjia.datastore.handler; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.mina.core.session.IoSession; -import org.bson.BSONObject; -import org.bson.BasicBSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fangjia.datastore.DataStoreContext; import com.fangjia.datastore.DataStoreException; -import com.fangjia.datastore.mongo.protocol.MongoMessage; +import java.util.Arrays; +import org.bson.BasicBSONObject; +import java.util.ArrayList; +import org.bson.BSONObject; +import java.util.List; import com.fangjia.datastore.mongo.protocol.ReplyMessage; +import java.net.InetSocketAddress; +import org.apache.mina.core.session.IoSession; +import org.slf4j.LoggerFactory; +import org.slf4j.Logger; +import com.fangjia.datastore.DataStoreContext; +import java.util.Map; +import com.fangjia.datastore.mongo.protocol.MongoMessage; -public abstract class MessageHandler { - private final static String LAST_ERROR_KEY = "com.goojia.datastore.LAST_ERROR_KEY"; - private final static String PREV_ERROR_KEY = "com.goojia.datastore.PREV_ERROR_KEY"; - - private final static String SESSION_KEY = "com.goojia.datastore.SESSION_KEY"; - private final static String ERROR_OCCURRED_KEY = "com.goojia.datastore.ERROR_OCCURRED_KEY"; - private final static ThreadLocal> THREAD_LOCAL = new ThreadLocal>() { - @Override - protected Map initialValue() { - return new HashMap(); - } - }; - - protected final DataStoreContext dataStoreContext; - - protected final Logger logger = LoggerFactory.getLogger(this.getClass()); - - public MessageHandler(DataStoreContext dataStoreContext) { - this.dataStoreContext = dataStoreContext; - } - - /** - * 处理特定的消息 - * - * @param message - */ - public abstract void messageReceived(T message,String ip) throws Exception; - - @SuppressWarnings("unchecked") - public void messageReceived(IoSession session, MongoMessage message) throws Exception { - System.out.println(session.getRemoteAddress()); - try { - setSession(session); - messageReceived((T) message,session.getRemoteAddress().toString()); - if (!isErrorOccured()) { - this.clearLastError(); - } - } catch (Throwable e) { - String hostAddress = ""; - if(session.getRemoteAddress() instanceof InetSocketAddress) { - hostAddress = ((InetSocketAddress)session.getRemoteAddress()).getAddress().getHostAddress(); - } - this.setLastError(buildExceptionMessage(e)); - logger.error("error processing host:" + hostAddress + ",message: " + message, e); - } finally { - clearErrorOccured(); - } - } - - private void clearErrorOccured() { - THREAD_LOCAL.get().remove(ERROR_OCCURRED_KEY); - } - - private void setErrorOccured() { - THREAD_LOCAL.get().put(ERROR_OCCURRED_KEY, true); - } - - private boolean isErrorOccured() { - Boolean b = (Boolean) THREAD_LOCAL.get().get(ERROR_OCCURRED_KEY); - return null == b ? false : b; - } - - protected IoSession getSession() { - return (IoSession) THREAD_LOCAL.get().get(SESSION_KEY); - } - - private void setSession(IoSession session) { - THREAD_LOCAL.get().put(SESSION_KEY, session); - } - - protected void setLastError(Object error) { - if (null != error) { - IoSession session = getSession(); - Object lastError = session.getAttribute(LAST_ERROR_KEY); - if (null != lastError) { - session.setAttribute(PREV_ERROR_KEY, lastError); - } - setErrorOccured(); - session.setAttribute(LAST_ERROR_KEY, String.valueOf(error)); - } - } - - private void clearLastError() { - getSession().removeAttribute(LAST_ERROR_KEY); - } - - protected Object getLastError() { - return getSession().getAttribute(LAST_ERROR_KEY); - } - - protected Object getPrevError() { - return getSession().getAttribute(PREV_ERROR_KEY); - } - - protected void writeMessage(ReplyMessage message) { - getSession().write(message); - } - - protected void writeAsReplyMessage(int responseTo, List docs) { - writeMessage(prepareReplyMessage(responseTo, docs)); - } - - @SuppressWarnings("unchecked") - protected ReplyMessage prepareReplyMessage(int responseTo, List docs) { - ReplyMessage reply = new ReplyMessage(); - reply.setNumberReturned(null == docs ? 0 : docs.size()); - reply.setResponseFlag(8); - reply.getMessageHeader().setResponseTo(responseTo); - if (null != docs && docs.size() > 0) { - reply.setDocuments((List) docs); - } - return reply; - } - - protected void writeAsReplyMessage(int responseTo, BSONObject... docs) { - writeMessage(prepareReplyMessage(responseTo, docs)); - } - - protected ReplyMessage prepareReplyMessage(int responseTo, BSONObject... docs) { - if (null != docs && docs.length > 0) { - List docList = new ArrayList(docs.length); - for (BSONObject doc : docs) { - docList.add(doc); - } - return prepareReplyMessage(responseTo, docList); - } else { - return prepareReplyMessage(responseTo, (List) null); - } - } - - protected void writeAsErrorMessage(int responseTo, Object errorMessage) { - writeMessage(prepareErrorMessage(responseTo, errorMessage)); - } - - protected ReplyMessage prepareErrorMessage(int responseTo, Object errorMessage) { - ReplyMessage reply = new ReplyMessage(); - reply.setNumberReturned(1); - reply.setResponseFlag(8); - reply.getMessageHeader().setResponseTo(responseTo); - BSONObject msg = new BasicBSONObject("$err", errorMessage == null ? null : errorMessage.toString()); - msg.put("err", errorMessage == null ? null : errorMessage.toString()); - msg.put("errmsg", errorMessage == null ? null : errorMessage.toString()); - msg.put("ok", 0.0); - reply.setDocuments(Arrays.asList(msg)); - return reply; - } - - protected String buildExceptionMessage(Throwable e) { - StringBuilder ret = new StringBuilder(); - ret.append(e.getClass().getName()).append(": ").append(e.getMessage()); - Throwable cause = e; - while ((cause = cause.getCause()) != null) { - ret.append(", caused by ").append(cause.getClass().getName()).append(": ").append(cause.getMessage()); - } - return ret.toString(); - } - - protected void assertNotReadOnly() throws DataStoreException { - if (this.dataStoreContext.getDataStoreConfig().isReadOnly()) { - throw new DataStoreException("read only server !"); - } - } +public abstract class MessageHandler +{ + private static final String LAST_ERROR_KEY = "com.goojia.datastore.LAST_ERROR_KEY"; + private static final String PREV_ERROR_KEY = "com.goojia.datastore.PREV_ERROR_KEY"; + private static final String SESSION_KEY = "com.goojia.datastore.SESSION_KEY"; + private static final String ERROR_OCCURRED_KEY = "com.goojia.datastore.ERROR_OCCURRED_KEY"; + private static final ThreadLocal> THREAD_LOCAL; + protected final DataStoreContext dataStoreContext; + protected final Logger logger = LoggerFactory.getLogger(this.getClass()); + + public MessageHandler(final DataStoreContext dataStoreContext) { + this.dataStoreContext = dataStoreContext; + } + public abstract void messageReceived(T message) throws Exception; + + public void messageReceived(final IoSession session, final MongoMessage message) throws Exception { + try { + this.setSession(session); + this.messageReceived((T)message); + if (!this.isErrorOccured()) { + this.clearLastError(); + } + } + catch (Throwable e) { + String hostAddress = ""; + if (session.getRemoteAddress() instanceof InetSocketAddress) { + hostAddress = ((InetSocketAddress)session.getRemoteAddress()).getAddress().getHostAddress(); + } + this.setLastError(this.buildExceptionMessage(e)); + this.logger.error("error processing host:" + hostAddress + ",message: " + message, e); + } + finally { + this.clearErrorOccured(); + } + } + + private void clearErrorOccured() { + MessageHandler.THREAD_LOCAL.get().remove("com.goojia.datastore.ERROR_OCCURRED_KEY"); + } + + private void setErrorOccured() { + MessageHandler.THREAD_LOCAL.get().put("com.goojia.datastore.ERROR_OCCURRED_KEY", true); + } + + private boolean isErrorOccured() { + final Boolean b = (Boolean) MessageHandler.THREAD_LOCAL.get().get("com.goojia.datastore.ERROR_OCCURRED_KEY"); + return null != b && b; + } + + protected IoSession getSession() { + return (IoSession)MessageHandler.THREAD_LOCAL.get().get("com.goojia.datastore.SESSION_KEY"); + } + + private void setSession(final IoSession session) { + MessageHandler.THREAD_LOCAL.get().put("com.goojia.datastore.SESSION_KEY", session); + } + + protected void setLastError(final Object error) { + if (null != error) { + final IoSession session = this.getSession(); + final Object lastError = session.getAttribute((Object)"com.goojia.datastore.LAST_ERROR_KEY"); + if (null != lastError) { + session.setAttribute((Object)"com.goojia.datastore.PREV_ERROR_KEY", lastError); + } + this.setErrorOccured(); + session.setAttribute((Object)"com.goojia.datastore.LAST_ERROR_KEY", (Object)String.valueOf(error)); + } + } + + private void clearLastError() { + this.getSession().removeAttribute((Object)"com.goojia.datastore.LAST_ERROR_KEY"); + } + + protected Object getLastError() { + return this.getSession().getAttribute((Object)"com.goojia.datastore.LAST_ERROR_KEY"); + } + + protected Object getPrevError() { + return this.getSession().getAttribute((Object)"com.goojia.datastore.PREV_ERROR_KEY"); + } + + protected void writeMessage(final ReplyMessage message) { + this.getSession().write((Object)message); + } + + protected void writeAsReplyMessage(final int responseTo, final List docs) { + this.writeMessage(this.prepareReplyMessage(responseTo, docs)); + } + + protected ReplyMessage prepareReplyMessage(final int responseTo, final List docs) { + final ReplyMessage reply = new ReplyMessage(); + reply.setNumberReturned((null == docs) ? 0 : docs.size()); + reply.setResponseFlag(8); + reply.getMessageHeader().setResponseTo(responseTo); + if (null != docs && docs.size() > 0) { + reply.setDocuments((List)docs); + } + return reply; + } + + protected void writeAsReplyMessage(final int responseTo, final BSONObject... docs) { + this.writeMessage(this.prepareReplyMessage(responseTo, docs)); + } + + protected ReplyMessage prepareReplyMessage(final int responseTo, final BSONObject... docs) { + if (null != docs && docs.length > 0) { + final List docList = new ArrayList(docs.length); + for (final BSONObject doc : docs) { + docList.add(doc); + } + return this.prepareReplyMessage(responseTo, docList); + } + return this.prepareReplyMessage(responseTo, (List)null); + } + + protected void writeAsErrorMessage(final int responseTo, final Object errorMessage) { + this.writeMessage(this.prepareErrorMessage(responseTo, errorMessage)); + } + + protected ReplyMessage prepareErrorMessage(final int responseTo, final Object errorMessage) { + final ReplyMessage reply = new ReplyMessage(); + reply.setNumberReturned(1); + reply.setResponseFlag(8); + reply.getMessageHeader().setResponseTo(responseTo); + final BSONObject msg = (BSONObject)new BasicBSONObject("$err", (Object)((errorMessage == null) ? null : errorMessage.toString())); + msg.put("err", (Object)((errorMessage == null) ? null : errorMessage.toString())); + msg.put("errmsg", (Object)((errorMessage == null) ? null : errorMessage.toString())); + msg.put("ok", (Object)0.0); + reply.setDocuments(Arrays.asList(msg)); + return reply; + } + + protected String buildExceptionMessage(final Throwable e) { + final StringBuilder ret = new StringBuilder(); + ret.append(e.getClass().getName()).append(": ").append(e.getMessage()); + Throwable cause = e; + while ((cause = cause.getCause()) != null) { + ret.append(", caused by ").append(cause.getClass().getName()).append(": ").append(cause.getMessage()); + } + return ret.toString(); + } + + protected void assertNotReadOnly() throws DataStoreException { + if (this.dataStoreContext.getDataStoreConfig().isReadOnly()) { + throw new DataStoreException("read only server !"); + } + } + + protected boolean useCursor() { + return (Boolean) this.getSession().getAttribute((Object)"useCursor", false); + } + + static { + THREAD_LOCAL = new ThreadLocal>() { + @Override + protected Map initialValue() { + return new HashMap(); + } + }; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/QueryMessageHandler.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/QueryMessageHandler.java index 8ae4e6ebb..fd2444c5b 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/QueryMessageHandler.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/QueryMessageHandler.java @@ -1,490 +1,501 @@ package com.fangjia.datastore.handler; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collection; +import com.fangjia.datastore.mongo.protocol.MongoMessage; +import org.apache.commons.lang.StringUtils; import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import com.mongodb.BasicDBList; -import org.apache.mina.core.session.IoSession; -import org.bson.BSONObject; -import org.bson.BasicBSONObject; - -import com.fangjia.datastore.ConfigurationException; -import com.fangjia.datastore.DataStore; -import com.fangjia.datastore.DataStoreContext; -import com.fangjia.datastore.DataStoreException; -import com.fangjia.datastore.QueryResponse; -import com.fangjia.datastore.config.Field; import com.fangjia.datastore.config.StoreConfig; -import com.fangjia.datastore.mongo.protocol.QueryMessage; +import com.fangjia.datastore.config.Field; import com.fangjia.datastore.mongo.protocol.ReplyMessage; +import com.fangjia.datastore.QueryResponse; +import com.fangjia.datastore.Cursor; +import java.util.Collection; import com.fangjia.datastore.solr.Index; -import com.fangjia.datastore.utils.MongoUtils; +import java.util.Iterator; +import java.util.Set; import com.mongodb.BasicDBObject; import com.mongodb.DBObject; +import java.util.ArrayList; +import com.fangjia.datastore.ConfigurationException; +import com.mongodb.BasicDBList; +import java.util.List; +import com.fangjia.datastore.DataStore; +import org.bson.types.BasicBSONList; +import org.bson.BasicBSONObject; +import org.bson.BSONObject; +import com.fangjia.datastore.DataStoreException; +import org.apache.mina.core.session.IoSession; +import java.net.InetSocketAddress; +import com.fangjia.datastore.utils.MongoUtils; +import com.fangjia.datastore.DataStoreContext; +import com.fangjia.datastore.mongo.protocol.QueryMessage; -public class QueryMessageHandler extends MessageHandler { - - public QueryMessageHandler(DataStoreContext dataStoreContext) { - super(dataStoreContext); - } - - public void messageReceived(QueryMessage message,String ip) throws Exception { - Object error = null; - try { - String[] nameSplit = MongoUtils.parseFullCollectionName(message.getFullCollectionName()); - if (null == nameSplit) { - error = "invalid full collection name: " + message.getFullCollectionName(); - } else { - doMessageReceived(message, nameSplit[0], nameSplit[1]); - } - } catch (Throwable e) { - IoSession session = getSession(); - String hostAddress = ""; - if(session != null && session.getRemoteAddress() instanceof InetSocketAddress) { - hostAddress = ((InetSocketAddress)session.getRemoteAddress()).getAddress().getHostAddress(); - } - - error = buildExceptionMessage(e); - this.writeAsErrorMessage(message.getMessageHeader().getRequestID(), error); - if (logger.isErrorEnabled()) { - logger.error("error process host:" + hostAddress + ",message: " + message, e); - } - } finally { - this.setLastError(error); - } - } - - private void doMessageReceived(QueryMessage message, String dbName, String collectionName) throws DataStoreException { - if (collectionName.equals("$cmd")) { - onCommand(message, dbName); - return; - } - if (collectionName.equals("system.namespaces")) { - showCollections(message, dbName, collectionName); - } else if (collectionName.equals("system.indexes")) { - getIndexes(message, dbName, collectionName); - } else { - find(message, dbName, collectionName); - } - } - - private void onCommand(QueryMessage message, String dbName) throws DataStoreException { - if (dbName.equals("admin")) { - onAdminCommand(message); - return; - } - BSONObject query = message.getQuery(); - if (query.containsField("getlasterror")) { - getLastError(message, dbName); - } else if (query.containsField("getpreverror")) { - getPrevError(message, dbName); - } else if (query.containsField("count")) { - count(message, dbName); - } else if (query.containsField("getSchema")) { - getSchema(message, dbName); - } else if (query.containsField("reIndex")) { - reIndex(message, dbName); - } else if (query.containsField("reload")) { - reload(message, dbName); - } else if(query.containsField("dbStats")){ - dbStats(message, dbName); - } else if(query.containsField("listCollections")){ - listCollections(message, dbName); - } else if (query.containsField("isMaster") || query.containsField("ismaster")) { - isMaster(message); - } else if(query.containsField("buildinfo") || query.containsField("buildInfo")){ - buildinfo(message); - } else if(query.containsField("serverStatus")){ - serverStatus(message); - } else if(query.containsField("connectionStatus")){ - connectionStatus(message); - } else if (query.containsField("find")) { - find(message, dbName, query.get("find").toString() ); - } else { - delegateCommand(message, dbName); - } - } - - private void onAdminCommand(QueryMessage message) throws DataStoreException { - BSONObject query = message.getQuery(); - if (query.containsField("whatsmyuri")) { - whatsMyUri(message); - } else if (query.containsField("listDatabases")) { - listDatabases(message); - } else if (query.containsField("nonce")) { - nonce(message); - } else if (query.containsField("isMaster") || query.containsField("ismaster")) { - isMaster(message); - } else if(query.containsField("buildinfo") || query.containsField("buildInfo")){ - buildinfo(message); - } else if (query.containsField("replSetGetStatus")) { - replSetGetStatus(message); - } else if(query.containsField("getlasterror")){ - getLastError(message, "admin"); - } else if(query.containsField("serverStatus")){ - serverStatus(message); - } else if(query.containsField("connectionStatus")){ - connectionStatus(message); - } else if (query.containsField("getLog")) { - getLog(message); - } else { - delegateAdminCommand(message); - } - } - - private void replSetGetStatus(QueryMessage message) { - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 0); - doc.put("errmsg", "not running with --replSet"); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - private void delegateCommand(QueryMessage message, String dbName) throws DataStoreException { - throw new DataStoreException("unsupported command:" + message.getQuery()); - } - - private void getLog(QueryMessage message){ - - BasicDBList names = new BasicDBList(); - names.add("global"); - names.add("startupWarnings"); - - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1); - doc.put("names", names); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - - } - - private void dbStats(QueryMessage message, String dbName) throws DataStoreException { - - BSONObject doc = new BasicBSONObject(); - doc.put("db", dbName); - doc.put("collections", this.dataStoreContext.getDataStoreConfig().getStoreNames()); - doc.put("ok", 1.0); - InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress(); - doc.put("primary", localAddress.getAddress().getHostAddress()); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - - - private void delegateAdminCommand(QueryMessage message) throws DataStoreException { - throw new DataStoreException("unsupported command:" + message.getQuery()); - } - - private void reload(QueryMessage message, String dbName) throws DataStoreException { - assertNotReadOnly(); - try { - this.dataStoreContext.reload(); - this.dataStoreContext.getDataStore(dbName + "." + message.getQuery().get("reload")).reload(); - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } catch (ConfigurationException e) { - throw new DataStoreException("error reload config: ", e); - } - } - - /** - * show collections - * - * @param message - */ - private void showCollections(QueryMessage message, String dbName, String collectionName) throws DataStoreException { - Set storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames(); - List docList = new ArrayList(storeNames.size()); - for (String sn : storeNames) { - if (sn.startsWith(dbName + ".")) { - docList.add(new BasicDBObject("name", sn)); - } - } - writeAsReplyMessage(message.getMessageHeader().getRequestID(), docList); - } - - /** - * db.foo.getIndexes() - * - * @param message - * @param dbName - * @param collectionName - * @throwstion - */ - private void getIndexes(QueryMessage message, String dbName, String collectionName) throws DataStoreException { - String ns = (String) message.getQuery().get("ns"); - Collection indexes = this.dataStoreContext.getDataStore(ns).getIndexes(); - List docList = new ArrayList(indexes.size()); - for (Index i : indexes) { - docList.add(i.toBSONObject()); - } - writeAsReplyMessage(message.getMessageHeader().getRequestID(), docList); - } - - /** - * db.foo.find() - * - * @param message - * @param dbName - * @param collectionName - * @throws DataStoreException - */ - private void find(QueryMessage message, String dbName, String collectionName) throws DataStoreException { - - String fullCollectionName; - if(message.getFullCollectionName().contains("$cmd")){ - fullCollectionName = dbName + "." + collectionName; - }else{ - fullCollectionName = message.getFullCollectionName(); - } - DataStore dataStore = this.dataStoreContext.getDataStore(fullCollectionName); - QueryResponse response = dataStore.find(message.getRealQuery(), message.getReturnFieldSelector(), message - .getOrderByQuery(), message.getFacetQuery(), message.getNumberToSkip(), message.getNumberToReturn() - 1, message - .isBulkQuery(), message.isFromMaster()); - ReplyMessage reply = prepareReplyMessage(message.getMessageHeader().getRequestID(), response.getDocList()); - reply.setStartingFrom(message.getNumberToSkip()); - // if (response.hasMore()) { - // Cursor cursor = new Cursor((int) message.getNumberToSkip(), (int) - // response.getNumReturned(), message, - // response.getNumFound()); - // this.dataStoreContext.addCursor(cursor); - // reply.setCursorID(cursor.getId()); - // } - this.writeMessage(reply); - } - - /** - * db.getLastError() - * - * @param message - * @param dbName - */ - private void getLastError(QueryMessage message, String dbName) { - Object lastError = this.getLastError(); - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - doc.put("n", 1); - doc.put("$err", lastError == null ? null : lastError.toString()); - doc.put("err", lastError == null ? null : lastError.toString()); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - /** - * db.getPrevError() - * - * @param message - * @param dbName - */ - private void getPrevError(QueryMessage message, String dbName) { - Object prevError = this.getPrevError(); - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - doc.put("n", 0); - doc.put("$err", prevError == null ? null : prevError.toString()); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - /** - * db.foo.count() - * - * @param message - * @param dbName - * @throws DataStoreException - */ - private void count(QueryMessage message, String dbName) throws DataStoreException { - BSONObject query = message.getQuery(); - int count = this.dataStoreContext.getDataStore(dbName + "." + query.get("count")).count(message.getRealQuery(), - message.isFromMaster()); - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - doc.put("n", (double) count); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - /** - * db.foo.rumCommnad('getSchema') - * - * @param message - * @param dbName - */ - private void getSchema(QueryMessage message, String dbName) throws DataStoreException { - BSONObject query = message.getQuery(); - String collectionName = (String) query.get("getSchema"); - StoreConfig storeConfig = this.dataStoreContext.getDataStore(dbName + "." + collectionName).getStoreConfig(); - Collection fields = storeConfig.getFields(); - List docList = new ArrayList(fields.size()); - for (Field field : fields) { - BasicDBObject f = new BasicDBObject(); - f.put("name", field.getName()); - f.put("type", field.getType().getType()); - docList.add(f); - } - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - doc.put("schema", docList); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - /** - *

- * 不带参数重建:db.foo.reIndex() 或 db.foo.runCommand('reIndex') - *

- * - *

- * 带参数重建:db.foo.runCommand('reIndex',{query:{cat:'sell',city:'上海'}}) - *

- * - * @param message - * @param dbName - * @throws Exception - */ - private void reIndex(QueryMessage message, String dbName) throws DataStoreException { - assertNotReadOnly(); - BSONObject query = message.getQuery(); - String collectionName = (String) query.get("reIndex"); - DataStore dataStore = this.dataStoreContext.getDataStore(dbName + "." + collectionName); - // 新增查询条件 add by yukai 2011/07/14 - BSONObject doc = new BasicBSONObject(); - Object realQuery = query.get("query"); - if (null == realQuery) - doc.put("version", dataStore.reIndex(null)); - else if (realQuery instanceof BSONObject) - dataStore.reIndex((BSONObject) realQuery); - else - throw new DataStoreException("invalid re-index query: " + realQuery); - doc.put("ok", 1.0); - doc.put("n", 0D); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - /** - * db.runCommand('whatsmyuri') - * - * @param message - */ - private void whatsMyUri(QueryMessage message) { - BSONObject doc = new BasicBSONObject(); - InetSocketAddress address = (InetSocketAddress) getSession().getRemoteAddress(); - doc.put("you", address.getAddress().getHostAddress() + ":" + address.getPort()); - doc.put("ok", 1.0); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - /** - * show dbs - * - * @param message - */ - private void listDatabases(QueryMessage message) { - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - Set storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames(); - Set dbNames = new HashSet(); - for (String sn : storeNames) { - int i = sn.indexOf('.'); - if (i > 0) { - String name = sn.substring(0, i); - dbNames.add(name); - } - } - List docs = new ArrayList(dbNames.size()); - for (String dn : dbNames) { - docs.add(new BasicDBObject("name", dn)); - } - doc.put("databases", docs); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - /** - * show dbs - * - * @param message - */ - private void listCollections(QueryMessage message, String dbName) { - System.out.println(message.getMessageHeader().getRequestID()); - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - BSONObject cursor = new BasicDBObject(); - doc.put("cursor", cursor); - - cursor.put("id", (long)message.getMessageHeader().getRequestID()); - cursor.put("ns", dbName + ".$cmd.listCollections"); - BasicDBList firstBatch = new BasicDBList(); - cursor.put("firstBatch", firstBatch); - Set storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames(); - for (String sn : storeNames) { - if (sn.startsWith(dbName + ".")) { - BSONObject collection = new BasicDBObject(); - collection.put("name", sn.replace(dbName + ".", "")); - collection.put("options", new BasicDBObject()); - firstBatch.add(collection); - } - } - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - - - - /** - * db.runCommand('nonce') - * - * @param message - */ - private void nonce(QueryMessage message) { - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - doc.put("errmsg", "no such cmd"); - doc.put("bad cmd", new BasicBSONObject("nonce", true)); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - private void isMaster(QueryMessage message) { - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - doc.put("ismaster", true); - doc.put("secondary", false); - InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress(); - doc.put("primary", localAddress.getAddress().getHostAddress()); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - private void buildinfo(QueryMessage message) { - BasicDBList versionArray = new BasicDBList(); - versionArray.add(3); - versionArray.add(2); - versionArray.add(9); - versionArray.add(0); - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - doc.put("bits", 64.0); - doc.put("debug", false); - doc.put("version" ,"3.2.9"); - doc.put("versionArray" ,versionArray); - doc.put("javascriptEngine", "V8"); - InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress(); - doc.put("primary", localAddress.getAddress().getHostAddress()); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - private void serverStatus(QueryMessage message){ - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress(); - doc.put("primary", localAddress.getAddress().getHostAddress()); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - private void connectionStatus(QueryMessage message){ - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress(); - doc.put("primary", localAddress.getAddress().getHostAddress()); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - +public class QueryMessageHandler extends MessageHandler +{ + public QueryMessageHandler(final DataStoreContext dataStoreContext) { + super(dataStoreContext); + } + + @Override + public void messageReceived(final QueryMessage message) throws Exception { + Object error = null; + try { + final String[] nameSplit = MongoUtils.parseFullCollectionName(message.getFullCollectionName()); + if (null == nameSplit) { + error = "invalid full collection name: " + message.getFullCollectionName(); + } + else { + this.doMessageReceived(message, nameSplit[0], nameSplit[1]); + } + } + catch (Throwable e) { + final IoSession session = this.getSession(); + String hostAddress = ""; + if (session != null && session.getRemoteAddress() instanceof InetSocketAddress) { + hostAddress = ((InetSocketAddress)session.getRemoteAddress()).getAddress().getHostAddress(); + } + error = this.buildExceptionMessage(e); + this.writeAsErrorMessage(message.getMessageHeader().getRequestID(), error); + if (this.logger.isErrorEnabled()) { + this.logger.error("error process host:" + hostAddress + ",message: " + message, e); + } + } + finally { + this.setLastError(error); + } + } + + private void doMessageReceived(final QueryMessage message, final String dbName, final String collectionName) throws DataStoreException { + if (collectionName.equals("$cmd")) { + this.onCommand(message, dbName); + return; + } + if (collectionName.equals("system.namespaces")) { + this.showCollections(message, dbName, collectionName); + } + else if (collectionName.equals("system.indexes")) { + this.getIndexes(message, dbName, collectionName); + } + else { + this.find(message, dbName, collectionName); + } + } + + private void onCommand(final QueryMessage message, final String dbName) throws DataStoreException { + if (dbName.equals("admin")) { + this.onAdminCommand(message); + return; + } + final BSONObject query = message.getQuery(); + if (query.containsField("getlasterror")) { + this.getLastError(message, dbName); + } + else if (query.containsField("getpreverror")) { + this.getPrevError(message, dbName); + } + else if (query.containsField("count")) { + this.count(message, dbName); + } + else if (query.containsField("getSchema")) { + this.getSchema(message, dbName); + } + else if (query.containsField("reIndex")) { + this.reIndex(message, dbName); + } + else if (query.containsField("reload")) { + this.reload(message, dbName); + } + else if (query.containsField("dbStats")) { + this.dbStats(message, dbName); + } + else if (query.containsField("listCollections")) { + this.listCollections(message, dbName); + } + else if (query.containsField("isMaster") || query.containsField("ismaster")) { + this.isMaster(message); + } + else if (query.containsField("buildinfo") || query.containsField("buildInfo")) { + this.buildinfo(message); + } + else if (query.containsField("serverStatus")) { + this.serverStatus(message); + } + else if (query.containsField("connectionStatus")) { + this.connectionStatus(message); + } + else if (query.containsField("update")) { + this.update(message, dbName); + } + else if (query.containsField("insert")) { + this.insert(message, dbName); + } + else if (query.containsField("ping")) { + this.ping(message); + } + else if (query.containsField("find")) { + this.find(message, dbName, query.get("find").toString()); + } + else { + this.delegateCommand(message, dbName); + } + } + + private void onAdminCommand(final QueryMessage message) throws DataStoreException { + final BSONObject query = message.getQuery(); + if (query.containsField("whatsmyuri")) { + this.whatsMyUri(message); + } + else if (query.containsField("listDatabases")) { + this.listDatabases(message); + } + else if (query.containsField("nonce")) { + this.nonce(message); + } + else if (query.containsField("isMaster") || query.containsField("ismaster")) { + this.isMaster(message); + } + else if (query.containsField("buildinfo") || query.containsField("buildInfo")) { + this.buildinfo(message); + } + else if (query.containsField("replSetGetStatus")) { + this.replSetGetStatus(message); + } + else if (query.containsField("getlasterror")) { + this.getLastError(message, "admin"); + } + else if (query.containsField("serverStatus")) { + this.serverStatus(message); + } + else if (query.containsField("connectionStatus")) { + this.connectionStatus(message); + } + else if (query.containsField("collstats")) { + this.collstats(message); + } + else if (query.containsField("getLog")) { + this.getLog(message); + } + else if (query.containsField("ping")) { + this.ping(message); + } + else { + this.delegateAdminCommand(message); + } + } + + private void replSetGetStatus(final QueryMessage message) { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)0); + doc.put("errmsg", (Object)"not running with --replSet"); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void delegateCommand(final QueryMessage message, final String dbName) throws DataStoreException { + throw new DataStoreException("unsupported command:" + message.getQuery()); + } + + private void collstats(final QueryMessage message) { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1); + doc.put("ns", (Object)message.getFullCollectionName()); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void ping(final QueryMessage message) throws DataStoreException { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void update(final QueryMessage message, final String dbName) throws DataStoreException { + final DataStore dataStore = this.dataStoreContext.getDataStore(dbName + "." + message.getQuery().get("update")); + final BasicBSONList updates = (BasicBSONList)message.getQuery().get("updates"); + final BSONObject selector = (BSONObject)((BasicBSONObject)updates.get(0)).get("q"); + final BSONObject update = (BSONObject)((BasicBSONObject)updates.get(0)).get("u"); + final int count = dataStore.update(selector, update); + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("nModified", (Object)count); + doc.put("n", (Object)count); + doc.put("ok", (Object)1); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void insert(final QueryMessage message, final String dbName) throws DataStoreException { + final DataStore dataStore = this.dataStoreContext.getDataStore(dbName + "." + message.getQuery().get("insert")); + final List documents = (List)message.getQuery().get("documents"); + dataStore.save(documents); + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("n", (Object)documents.size()); + doc.put("ok", (Object)1); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void getLog(final QueryMessage message) { + final BasicDBList names = new BasicDBList(); + names.add((Object)"global"); + names.add((Object)"startupWarnings"); + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1); + doc.put("names", (Object)names); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void dbStats(final QueryMessage message, final String dbName) throws DataStoreException { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("db", (Object)dbName); + doc.put("collections", (Object)this.dataStoreContext.getDataStoreConfig().getStoreNames()); + doc.put("ok", (Object)1.0); + final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress(); + doc.put("primary", (Object)localAddress.getAddress().getHostAddress()); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void delegateAdminCommand(final QueryMessage message) throws DataStoreException { + throw new DataStoreException("unsupported command:" + message.getQuery()); + } + + private void reload(final QueryMessage message, final String dbName) throws DataStoreException { + this.assertNotReadOnly(); + try { + this.dataStoreContext.reload(); + this.dataStoreContext.getDataStore(dbName + "." + message.getQuery().get("reload")).reload(); + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + catch (ConfigurationException e) { + throw new DataStoreException("error reload config: ", e); + } + } + + private void showCollections(final QueryMessage message, final String dbName, final String collectionName) throws DataStoreException { + final Set storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames(); + final List docList = new ArrayList(storeNames.size()); + for (final String sn : storeNames) { + if (sn.startsWith(dbName + ".")) { + docList.add((DBObject)new BasicDBObject("name", (Object)sn)); + } + } + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), (List)docList); + } + + private void getIndexes(final QueryMessage message, final String dbName, final String collectionName) throws DataStoreException { + final String ns = (String)message.getQuery().get("ns"); + final Collection indexes = this.dataStoreContext.getDataStore(ns).getIndexes(); + final List docList = new ArrayList(indexes.size()); + for (final Index i : indexes) { + docList.add(i.toBSONObject()); + } + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), docList); + } + + private void find(final QueryMessage message, final String dbName, final String collectionName) throws DataStoreException { + String fullCollectionName; + if (message.getFullCollectionName().contains("$cmd")) { + fullCollectionName = dbName + "." + collectionName; + } + else { + fullCollectionName = message.getFullCollectionName(); + } + final DataStore dataStore = this.dataStoreContext.getDataStore(fullCollectionName); + QueryResponse response; + if (this.useCursor()) { + response = dataStore.find(message.getRealQuery(), message.getReturnFieldSelector(), message.getOrderByQuery(), message.getFacetQuery(), message.getNumberToSkip(), message.getNumberToReturn(), message.isBulkQuery(), message.isFromMaster()); + } + else { + response = dataStore.find(message.getRealQuery(), message.getReturnFieldSelector(), message.getOrderByQuery(), message.getFacetQuery(), message.getNumberToSkip(), message.getNumberToReturn() - 1, message.isBulkQuery(), message.isFromMaster()); + } + final List doc = response.getDocList(); + if (this.useCursor()) { + doc.remove(0); + } + final ReplyMessage reply = this.prepareReplyMessage(message.getMessageHeader().getRequestID(), (List)doc); + reply.setStartingFrom(message.getNumberToSkip()); + if (response.hasMore() && this.useCursor()) { + final Cursor cursor = new Cursor(message.getNumberToSkip(), message.getNumberToReturn(), message, response.getNumFound()); + this.dataStoreContext.addCursor(cursor); + reply.setCursorID(cursor.getId()); + } + this.writeMessage(reply); + } + + private void getLastError(final QueryMessage message, final String dbName) { + final Object lastError = this.getLastError(); + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + doc.put("n", (Object)1); + doc.put("$err", (Object)((lastError == null) ? null : lastError.toString())); + doc.put("err", (Object)((lastError == null) ? null : lastError.toString())); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void getPrevError(final QueryMessage message, final String dbName) { + final Object prevError = this.getPrevError(); + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + doc.put("n", (Object)0); + doc.put("$err", (Object)((prevError == null) ? null : prevError.toString())); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void count(final QueryMessage message, final String dbName) throws DataStoreException { + final BSONObject query = message.getQuery(); + final int count = this.dataStoreContext.getDataStore(dbName + "." + query.get("count")).count(message.getRealQuery(), message.isFromMaster()); + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + doc.put("n", (Object)(double)count); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void getSchema(final QueryMessage message, final String dbName) throws DataStoreException { + final BSONObject query = message.getQuery(); + final String collectionName = (String)query.get("getSchema"); + final StoreConfig storeConfig = this.dataStoreContext.getDataStore(dbName + "." + collectionName).getStoreConfig(); + final Collection fields = storeConfig.getFields(); + final List docList = new ArrayList(fields.size()); + for (final Field field : fields) { + final BasicDBObject f = new BasicDBObject(); + f.put("name", field.getName()); + f.put("type", field.getType().getType()); + docList.add(f); + } + final BSONObject doc = new BasicBSONObject(); + doc.put("ok", 1.0); + doc.put("schema", docList); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void reIndex(final QueryMessage message, final String dbName) throws DataStoreException { + this.assertNotReadOnly(); + final BSONObject query = message.getQuery(); + final String collectionName = (String)query.get("reIndex"); + final DataStore dataStore = this.dataStoreContext.getDataStore(dbName + "." + collectionName); + final BSONObject doc = (BSONObject)new BasicBSONObject(); + final Object realQuery = query.get("query"); + if (null == realQuery) { + doc.put("version", (Object)dataStore.reIndex(null)); + } + else { + if (!(realQuery instanceof BSONObject)) { + throw new DataStoreException("invalid re-index query: " + realQuery); + } + dataStore.reIndex((BSONObject)realQuery); + } + doc.put("ok", (Object)1.0); + doc.put("n", (Object)0.0); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void whatsMyUri(final QueryMessage message) { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + final InetSocketAddress address = (InetSocketAddress)this.getSession().getRemoteAddress(); + doc.put("you", (Object)(address.getAddress().getHostAddress() + ":" + address.getPort())); + doc.put("ok", (Object)1.0); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void listDatabases(final QueryMessage message) { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + final Set storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames(); + final Set dbNames = new HashSet(); + for (final String sn : storeNames) { + final int i = sn.indexOf(46); + if (i > 0) { + final String name = sn.substring(0, i); + dbNames.add(name); + } + } + final List docs = new ArrayList(dbNames.size()); + for (final String dn : dbNames) { + docs.add((DBObject)new BasicDBObject("name", (Object)dn)); + } + doc.put("databases", (Object)docs); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void listCollections(final QueryMessage message, final String dbName) { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + final BSONObject cursor = (BSONObject)new BasicDBObject(); + doc.put("cursor", (Object)cursor); + cursor.put("id", (Object)(long)message.getMessageHeader().getRequestID()); + cursor.put("ns", (Object)(dbName + ".$cmd.listCollections")); + final BasicDBList firstBatch = new BasicDBList(); + cursor.put("firstBatch", (Object)firstBatch); + final Set storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames(); + for (final String sn : storeNames) { + if (sn.startsWith(dbName + ".")) { + final BSONObject collection = (BSONObject)new BasicDBObject(); + collection.put("name", (Object)sn.replace(dbName + ".", "")); + collection.put("options", (Object)new BasicDBObject()); + firstBatch.add((Object)collection); + } + } + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void nonce(final QueryMessage message) { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + doc.put("errmsg", (Object)"no such cmd"); + doc.put("bad cmd", (Object)new BasicBSONObject("nonce", (Object)true)); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void isMaster(final QueryMessage message) { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + doc.put("ismaster", (Object)true); + doc.put("secondary", (Object)false); + final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress(); + doc.put("primary", (Object)localAddress.getAddress().getHostAddress()); + doc.put("maxWireVersion", (Object)2); + doc.put("minWireVersion", (Object)0); + final BasicBSONObject client = (BasicBSONObject)message.getQuery().get("client"); + if (client != null) { + final String platform = client.getString("platform"); + if (platform != null && (platform.contains("python") || platform.contains("Python"))) { + this.getSession().setAttribute((Object)"useCursor", (Object)true); + } + final BasicBSONObject applicationInfo = (BasicBSONObject)client.get("application"); + if (applicationInfo != null && StringUtils.isNotBlank(applicationInfo.getString("name"))) { + this.getSession().setAttribute((Object)"useCursor", (Object)true); + } + } + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void buildinfo(final QueryMessage message) { + final BasicDBList versionArray = new BasicDBList(); + versionArray.add((Object)2); + versionArray.add((Object)6); + versionArray.add((Object)8); + versionArray.add((Object)0); + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + doc.put("bits", (Object)64.0); + doc.put("debug", (Object)false); + doc.put("version", (Object)"2.6.8"); + doc.put("versionArray", (Object)versionArray); + doc.put("javascriptEngine", (Object)"V8"); + final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress(); + doc.put("primary", (Object)localAddress.getAddress().getHostAddress()); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void serverStatus(final QueryMessage message) { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress(); + doc.put("primary", (Object)localAddress.getAddress().getHostAddress()); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void connectionStatus(final QueryMessage message) { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress(); + doc.put("primary", (Object)localAddress.getAddress().getHostAddress()); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/UpdateMessageHandler.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/UpdateMessageHandler.java index 58f3a45df..1641938a6 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/UpdateMessageHandler.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/UpdateMessageHandler.java @@ -1,27 +1,27 @@ package com.fangjia.datastore.handler; +import com.fangjia.datastore.mongo.protocol.MongoMessage; import com.fangjia.datastore.DataStore; -import com.fangjia.datastore.DataStoreContext; import com.fangjia.datastore.DataStoreException; +import com.fangjia.datastore.DataStoreContext; import com.fangjia.datastore.mongo.protocol.UpdateMessage; -public class UpdateMessageHandler extends MessageHandler { - - public UpdateMessageHandler(DataStoreContext dataStoreContext) { - super(dataStoreContext); - } - - public void messageReceived(UpdateMessage message,String ip) throws Exception { - assertNotReadOnly(); - if (message.isUpsert()) { - throw new DataStoreException("upsert is not supported."); - } - if (!message.isMultiUpdate()) { - throw new DataStoreException("only multi update is supported."); - } - - DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName()); - dataStore.update(message.getSelector(), message.getUpdate(),ip); - } - +public class UpdateMessageHandler extends MessageHandler +{ + public UpdateMessageHandler(final DataStoreContext dataStoreContext) { + super(dataStoreContext); + } + + @Override + public void messageReceived(final UpdateMessage message) throws Exception { + this.assertNotReadOnly(); + if (message.isUpsert()) { + throw new DataStoreException("upsert is not supported."); + } + if (!message.isMultiUpdate()) { + throw new DataStoreException("only multi update is supported."); + } + final DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName()); + dataStore.update(message.getSelector(), message.getUpdate()); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONInputStream.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONInputStream.java index b77a769f9..b32f524c5 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONInputStream.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONInputStream.java @@ -1,92 +1,75 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.codec; -import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.ByteArrayOutputStream; +import org.apache.mina.core.buffer.IoBuffer; import java.io.InputStream; -import org.apache.mina.core.buffer.IoBuffer; - -public class BSONInputStream extends InputStream { - private final IoBuffer buffer; - - public BSONInputStream(IoBuffer buffer) { - this.buffer = buffer; - } - - @Override - public int read() { - return buffer.get(); - } - - public synchronized int read(byte b[], int off, int len) { - if (b == null) { - throw new NullPointerException(); - } else if (off < 0 || len < 0 || len > b.length - off) { - throw new IndexOutOfBoundsException(); - } - buffer.get(b, off, len); - return len; - } - - public int readInteger() { - return this.buffer.getInt(); - } - - public long readLong() { - return this.buffer.getLong(); - } - - public double readDouble() { - return this.buffer.getDouble(); - } - - public String readCString() throws IOException { - ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); - int b; - while (true) { - b = read(); - if (b != 0) { - byteStream.write(b); - } else { - return new String(byteStream.toByteArray(), "UTF-8"); - } - } - } - - public String readString() throws IOException { - int len = readInteger(); - if (len < 0) { - throw new IOException("can not read string from the stream"); - } - if (len == 0) { - read(); - return ""; - } - byte[] buf = new byte[len]; - read(buf); - return new String(buf, 0, len - 1, "UTF-8"); - } - - @Override - public int available() throws IOException { - return buffer.remaining(); - } - +public class BSONInputStream extends InputStream +{ + private final IoBuffer buffer; + + public BSONInputStream(final IoBuffer buffer) { + this.buffer = buffer; + } + + @Override + public int read() { + return this.buffer.get(); + } + + @Override + public synchronized int read(final byte[] b, final int off, final int len) { + if (b == null) { + throw new NullPointerException(); + } + if (off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException(); + } + this.buffer.get(b, off, len); + return len; + } + + public int readInteger() { + return this.buffer.getInt(); + } + + public long readLong() { + return this.buffer.getLong(); + } + + public double readDouble() { + return this.buffer.getDouble(); + } + + public String readCString() throws IOException { + final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + while (true) { + final int b = this.read(); + if (b == 0) { + break; + } + byteStream.write(b); + } + return new String(byteStream.toByteArray(), "UTF-8"); + } + + public String readString() throws IOException { + final int len = this.readInteger(); + if (len < 0) { + throw new IOException("can not read string from the stream"); + } + if (len == 0) { + this.read(); + return ""; + } + final byte[] buf = new byte[len]; + this.read(buf); + return new String(buf, 0, len - 1, "UTF-8"); + } + + @Override + public int available() throws IOException { + return this.buffer.remaining(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONIoBufferAllocator.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONIoBufferAllocator.java index 042264ec6..97e452e9f 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONIoBufferAllocator.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONIoBufferAllocator.java @@ -1,95 +1,83 @@ package com.fangjia.datastore.mongo.codec; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - import org.apache.mina.core.buffer.AbstractIoBuffer; +import java.nio.ByteOrder; +import java.nio.ByteBuffer; import org.apache.mina.core.buffer.IoBuffer; import org.apache.mina.core.buffer.IoBufferAllocator; -public class BSONIoBufferAllocator implements IoBufferAllocator { - - public IoBuffer allocate(int capacity, boolean direct) { - return wrap(allocateNioBuffer(capacity, direct)); - } - - public ByteBuffer allocateNioBuffer(int capacity, boolean direct) { - ByteBuffer nioBuffer; - if (direct) { - nioBuffer = ByteBuffer.allocateDirect(capacity); - } else { - nioBuffer = ByteBuffer.allocate(capacity); - } - nioBuffer.order(ByteOrder.LITTLE_ENDIAN); - return nioBuffer; - } - - public IoBuffer wrap(ByteBuffer nioBuffer) { - return new SimpleBuffer(nioBuffer); - } - - public void dispose() { - // Do nothing - } - - private class SimpleBuffer extends AbstractIoBuffer { - private ByteBuffer buf; - - protected SimpleBuffer(ByteBuffer buf) { - super(BSONIoBufferAllocator.this, buf.capacity()); - this.buf = buf; - this.buf.order(ByteOrder.LITTLE_ENDIAN); - } - - protected SimpleBuffer(SimpleBuffer parent, ByteBuffer buf) { - super(parent); - this.buf = buf; - this.buf.order(ByteOrder.LITTLE_ENDIAN); - } - - @Override - public ByteBuffer buf() { - return buf; - } - - @Override - protected void buf(ByteBuffer buf) { - this.buf = buf; - } - - @Override - protected IoBuffer duplicate0() { - return new SimpleBuffer(this, this.buf.duplicate()); - } - - @Override - protected IoBuffer slice0() { - return new SimpleBuffer(this, this.buf.slice()); - } - - @Override - protected IoBuffer asReadOnlyBuffer0() { - return new SimpleBuffer(this, this.buf.asReadOnlyBuffer()); - } - - @Override - public byte[] array() { - return buf.array(); - } - - @Override - public int arrayOffset() { - return buf.arrayOffset(); - } - - @Override - public boolean hasArray() { - return buf.hasArray(); - } - - @Override - public void free() { - // Do nothing - } - } +public class BSONIoBufferAllocator implements IoBufferAllocator +{ + public IoBuffer allocate(final int capacity, final boolean direct) { + return this.wrap(this.allocateNioBuffer(capacity, direct)); + } + + public ByteBuffer allocateNioBuffer(final int capacity, final boolean direct) { + ByteBuffer nioBuffer; + if (direct) { + nioBuffer = ByteBuffer.allocateDirect(capacity); + } + else { + nioBuffer = ByteBuffer.allocate(capacity); + } + nioBuffer.order(ByteOrder.LITTLE_ENDIAN); + return nioBuffer; + } + + public IoBuffer wrap(final ByteBuffer nioBuffer) { + return (IoBuffer)new SimpleBuffer(nioBuffer); + } + + public void dispose() { + } + + private class SimpleBuffer extends AbstractIoBuffer + { + private ByteBuffer buf; + + protected SimpleBuffer(final ByteBuffer buf) { + super((IoBufferAllocator)BSONIoBufferAllocator.this, buf.capacity()); + (this.buf = buf).order(ByteOrder.LITTLE_ENDIAN); + } + + protected SimpleBuffer(final SimpleBuffer parent, final ByteBuffer buf) { + super((AbstractIoBuffer)parent); + (this.buf = buf).order(ByteOrder.LITTLE_ENDIAN); + } + + public ByteBuffer buf() { + return this.buf; + } + + protected void buf(final ByteBuffer buf) { + this.buf = buf; + } + + protected IoBuffer duplicate0() { + return (IoBuffer)new SimpleBuffer(this, this.buf.duplicate()); + } + + protected IoBuffer slice0() { + return (IoBuffer)new SimpleBuffer(this, this.buf.slice()); + } + + protected IoBuffer asReadOnlyBuffer0() { + return (IoBuffer)new SimpleBuffer(this, this.buf.asReadOnlyBuffer()); + } + + public byte[] array() { + return this.buf.array(); + } + + public int arrayOffset() { + return this.buf.arrayOffset(); + } + + public boolean hasArray() { + return this.buf.hasArray(); + } + + public void free() { + } + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONOutputStream.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONOutputStream.java index 77254eb6d..6e501ea79 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONOutputStream.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONOutputStream.java @@ -1,102 +1,84 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.codec; -import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.util.Arrays; +import java.io.IOException; +import java.io.ByteArrayOutputStream; -public class BSONOutputStream extends ByteArrayOutputStream { - - private static final String ENCDOING = "UTF-8"; - - public BSONOutputStream() { - super(); - } - - public BSONOutputStream(int size) { - super(size); - } - - public void writeCString(String s) throws IOException { - write(s.getBytes(ENCDOING)); - write(0); - } - - public void writeInteger(int i) throws IOException { - write(i); - write(i >> 8); - write(i >> 16); - write(i >> 24); - } - - public void writeLong(long l) throws IOException { - write((byte) l); - write((byte) (l >> 8)); - write((byte) (l >> 16)); - write((byte) (l >> 24)); - write((byte) (l >> 32)); - write((byte) (l >> 40)); - write((byte) (l >> 48)); - write((byte) (l >> 56)); - } - - public void writeString(String s) throws IOException { - byte[] bs = s.getBytes(ENCDOING); - writeInteger(bs.length + 1); - write(bs); - write(0); - } - - public void set(int index, int i) { - int newcount = Math.max(count, index + 4); - if (newcount > buf.length) { - buf = Arrays.copyOf(buf, Math.max(buf.length << 1, newcount)); - } - buf[index] = (byte) i; - buf[index + 1] = (byte) (i >> 8); - buf[index + 2] = (byte) (i >> 16); - buf[index + 3] = (byte) (i >> 24); - count = newcount; - } - - public void set(int index, byte[] b, int off, int len) { - if ((off < 0) || (off > b.length) || (len < 0) || ((off + len) > b.length) || ((off + len) < 0)) { - throw new IndexOutOfBoundsException(); - } else if (len == 0) { - return; - } - int newcount = Math.max(count, index + len); - if (newcount > buf.length) { - buf = Arrays.copyOf(buf, Math.max(buf.length << 1, newcount)); - } - System.arraycopy(b, off, buf, index, len); - count = newcount; - } - - public void setInteger(int index, int i) { - byte[] b = new byte[] { (byte) i, (byte) (i >> 8), (byte) (i >> 16), (byte) (i >> 24) }; - set(index, b, 0, b.length); - } - - public void setLong(int index, long l) { - byte[] b = new byte[] { (byte) l, (byte) (l >> 8), (byte) (l >> 16), (byte) (l >> 24), (byte) (l >> 32), - (byte) (l >> 40), (byte) (l >> 48), (byte) (l >> 56) }; - set(index, b, 0, b.length); - } +public class BSONOutputStream extends ByteArrayOutputStream +{ + private static final String ENCDOING = "UTF-8"; + + public BSONOutputStream() { + } + + public BSONOutputStream(final int size) { + super(size); + } + + public void writeCString(final String s) throws IOException { + this.write(s.getBytes("UTF-8")); + this.write(0); + } + + public void writeInteger(final int i) throws IOException { + this.write(i); + this.write(i >> 8); + this.write(i >> 16); + this.write(i >> 24); + } + + public void writeLong(final long l) throws IOException { + this.write((byte)l); + this.write((byte)(l >> 8)); + this.write((byte)(l >> 16)); + this.write((byte)(l >> 24)); + this.write((byte)(l >> 32)); + this.write((byte)(l >> 40)); + this.write((byte)(l >> 48)); + this.write((byte)(l >> 56)); + } + + public void writeString(final String s) throws IOException { + final byte[] bs = s.getBytes("UTF-8"); + this.writeInteger(bs.length + 1); + this.write(bs); + this.write(0); + } + + public void set(final int index, final int i) { + final int newcount = Math.max(this.count, index + 4); + if (newcount > this.buf.length) { + this.buf = Arrays.copyOf(this.buf, Math.max(this.buf.length << 1, newcount)); + } + this.buf[index] = (byte)i; + this.buf[index + 1] = (byte)(i >> 8); + this.buf[index + 2] = (byte)(i >> 16); + this.buf[index + 3] = (byte)(i >> 24); + this.count = newcount; + } + + public void set(final int index, final byte[] b, final int off, final int len) { + if (off < 0 || off > b.length || len < 0 || off + len > b.length || off + len < 0) { + throw new IndexOutOfBoundsException(); + } + if (len == 0) { + return; + } + final int newcount = Math.max(this.count, index + len); + if (newcount > this.buf.length) { + this.buf = Arrays.copyOf(this.buf, Math.max(this.buf.length << 1, newcount)); + } + System.arraycopy(b, off, this.buf, index, len); + this.count = newcount; + } + + public void setInteger(final int index, final int i) { + final byte[] b = { (byte)i, (byte)(i >> 8), (byte)(i >> 16), (byte)(i >> 24) }; + this.set(index, b, 0, b.length); + } + + public void setLong(final int index, final long l) { + final byte[] b = { (byte)l, (byte)(l >> 8), (byte)(l >> 16), (byte)(l >> 24), (byte)(l >> 32), (byte)(l >> 40), (byte)(l >> 48), (byte)(l >> 56) }; + this.set(index, b, 0, b.length); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/MongoProtocolDecoder.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/MongoProtocolDecoder.java index dbd156d26..d9302ac7f 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/MongoProtocolDecoder.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/MongoProtocolDecoder.java @@ -1,111 +1,108 @@ package com.fangjia.datastore.mongo.codec; +import org.slf4j.LoggerFactory; import org.apache.mina.core.buffer.BufferDataException; +import com.fangjia.datastore.mongo.protocol.UpdateMessage; +import com.fangjia.datastore.mongo.protocol.ReplyMessage; +import com.fangjia.datastore.mongo.protocol.QueryMessage; +import com.fangjia.datastore.mongo.protocol.KillCursorsMessage; +import com.fangjia.datastore.mongo.protocol.InsertMessage; +import com.fangjia.datastore.mongo.protocol.GetMoreMessage; +import com.fangjia.datastore.mongo.protocol.DeleteMessage; +import com.fangjia.datastore.mongo.protocol.MongoMessage; +import org.apache.mina.filter.codec.ProtocolDecoderException; +import com.fangjia.datastore.mongo.protocol.OperationCode; +import org.apache.mina.filter.codec.ProtocolDecoderOutput; import org.apache.mina.core.buffer.IoBuffer; import org.apache.mina.core.session.IoSession; -import org.apache.mina.filter.codec.CumulativeProtocolDecoder; -import org.apache.mina.filter.codec.ProtocolDecoderException; -import org.apache.mina.filter.codec.ProtocolDecoderOutput; import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fangjia.datastore.mongo.protocol.DeleteMessage; -import com.fangjia.datastore.mongo.protocol.GetMoreMessage; -import com.fangjia.datastore.mongo.protocol.InsertMessage; -import com.fangjia.datastore.mongo.protocol.KillCursorsMessage; -import com.fangjia.datastore.mongo.protocol.MongoMessage; -import com.fangjia.datastore.mongo.protocol.OperationCode; -import com.fangjia.datastore.mongo.protocol.QueryMessage; -import com.fangjia.datastore.mongo.protocol.ReplyMessage; -import com.fangjia.datastore.mongo.protocol.UpdateMessage; - -public class MongoProtocolDecoder extends CumulativeProtocolDecoder { - - private final static Logger LOGGER = LoggerFactory.getLogger(MongoProtocolDecoder.class); - - @Override - protected boolean doDecode(IoSession session, IoBuffer in, ProtocolDecoderOutput out) throws Exception { - try { - if (!prefixedDataAvailable(in, 4, true)) { - return false; - } - int start = in.position(); - int limit = in.limit(); - int messageSize = in.getInt(start); - in.limit(start + messageSize); - try { - final IoBuffer buf = in.slice(); - int opCodeValue = buf.getInt(buf.position() + 12); - OperationCode opCode = OperationCode.valueOf(opCodeValue); - MongoMessage message = newInstance(opCode); - BSONInputStream input = new BSONInputStream(buf); - message.read(input); - out.write(message); - return true; - } finally { - in.position(start + messageSize); - in.limit(limit); - } - } catch (Exception e) { - ProtocolDecoderException pde = new ProtocolDecoderException("error decode message:", e); - pde.setHexdump(in.getHexDump()); - LOGGER.error("", pde); - throw pde; - } - } - - private MongoMessage newInstance(OperationCode opCode) { - switch (opCode) { - case OP_DELETE: - return new DeleteMessage(); - case OP_GET_MORE: - return new GetMoreMessage(); - case OP_INSERT: - return new InsertMessage(); - case OP_KILL_CURSORS: - return new KillCursorsMessage(); - case OP_QUERY: - return new QueryMessage(); - case OP_REPLY: - return new ReplyMessage(); - case OP_UPDATE: - return new UpdateMessage(); - case OP_MSG: - case RESERVED: - default: - return null; - } - } - - public boolean prefixedDataAvailable(IoBuffer buffer, int prefixLength, boolean includePrefix) { - if (buffer.remaining() < prefixLength) { - return false; - } - - int dataLength; - switch (prefixLength) { - case 1: - dataLength = buffer.getUnsigned(buffer.position()); - break; - case 2: - dataLength = buffer.getUnsignedShort(buffer.position()); - break; - case 4: - dataLength = buffer.getInt(buffer.position()); - break; - default: - throw new IllegalArgumentException("prefixLength: " + prefixLength); - } - - if (dataLength < 0) { - throw new BufferDataException("dataLength: " + dataLength); - } - - if (includePrefix) { - return buffer.remaining() >= dataLength; - } else { - return buffer.remaining() - prefixLength >= dataLength; - } - } +import org.apache.mina.filter.codec.CumulativeProtocolDecoder; +public class MongoProtocolDecoder extends CumulativeProtocolDecoder +{ + private static final Logger LOGGER; + + protected boolean doDecode(final IoSession session, final IoBuffer in, final ProtocolDecoderOutput out) throws Exception { + try { + if (!this.prefixedDataAvailable(in, 4, true)) { + return false; + } + final int start = in.position(); + final int limit = in.limit(); + final int messageSize = in.getInt(start); + in.limit(start + messageSize); + try { + final IoBuffer buf = in.slice(); + final int opCodeValue = buf.getInt(buf.position() + 12); + final OperationCode opCode = OperationCode.valueOf(opCodeValue); + final MongoMessage message = this.newInstance(opCode); + final BSONInputStream input = new BSONInputStream(buf); + message.read(input); + out.write((Object)message); + return true; + } + finally { + in.position(start + messageSize); + in.limit(limit); + } + } + catch (Exception e) { + final ProtocolDecoderException pde = new ProtocolDecoderException("error decode message:", (Throwable)e); + pde.setHexdump(in.getHexDump()); + MongoProtocolDecoder.LOGGER.error("", (Throwable)pde); + throw pde; + } + } + + private MongoMessage newInstance(final OperationCode opCode) { + switch (opCode) { + case OP_DELETE: + return new DeleteMessage(); + case OP_GET_MORE: + return new GetMoreMessage(); + case OP_INSERT: + return new InsertMessage(); + case OP_KILL_CURSORS: + return new KillCursorsMessage(); + case OP_QUERY: + return new QueryMessage(); + case OP_REPLY: + return new ReplyMessage(); + case OP_UPDATE: + return new UpdateMessage(); + default: + return null; + } + } + + public boolean prefixedDataAvailable(final IoBuffer buffer, final int prefixLength, final boolean includePrefix) { + if (buffer.remaining() < prefixLength) { + return false; + } + int dataLength = 0; + switch (prefixLength) { + case 1: + dataLength = buffer.getUnsigned(buffer.position()); + break; + case 2: + dataLength = buffer.getUnsignedShort(buffer.position()); + break; + case 4: + dataLength = buffer.getInt(buffer.position()); + break; + default: + throw new IllegalArgumentException("prefixLength: " + prefixLength); + } + if (dataLength < 0) { + throw new BufferDataException("dataLength: " + dataLength); + } + if (includePrefix) { + return buffer.remaining() >= dataLength; + } + return buffer.remaining() - prefixLength >= dataLength; + } + + static { + LOGGER = LoggerFactory.getLogger((Class)MongoProtocolDecoder.class); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/MongoProtocolEncoder.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/MongoProtocolEncoder.java index 786e1043a..0973fdb0d 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/MongoProtocolEncoder.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/MongoProtocolEncoder.java @@ -1,36 +1,38 @@ package com.fangjia.datastore.mongo.codec; -import org.apache.mina.core.buffer.IoBuffer; -import org.apache.mina.core.session.IoSession; -import org.apache.mina.filter.codec.ProtocolEncoder; -import org.apache.mina.filter.codec.ProtocolEncoderException; -import org.apache.mina.filter.codec.ProtocolEncoderOutput; -import org.slf4j.Logger; import org.slf4j.LoggerFactory; - +import org.apache.mina.filter.codec.ProtocolEncoderException; +import org.apache.mina.core.buffer.IoBuffer; import com.fangjia.datastore.mongo.protocol.MongoMessage; +import org.apache.mina.filter.codec.ProtocolEncoderOutput; +import org.apache.mina.core.session.IoSession; +import org.slf4j.Logger; +import org.apache.mina.filter.codec.ProtocolEncoder; -public class MongoProtocolEncoder implements ProtocolEncoder { - - private final static Logger LOGGER = LoggerFactory.getLogger(MongoProtocolEncoder.class); - - public void encode(IoSession session, Object message, ProtocolEncoderOutput out) throws Exception { - try { - if (message instanceof MongoMessage) { - MongoMessage mongoMessage = (MongoMessage) message; - BSONOutputStream output = new BSONOutputStream(mongoMessage.getMessageHeader().getMessageLength()); - mongoMessage.write(output); - out.write(IoBuffer.wrap(output.toByteArray())); - } - } catch (Exception e) { - ProtocolEncoderException pe = new ProtocolEncoderException("error encode message: " + message, e); - LOGGER.error("", pe); - throw pe; - } - } - - public void dispose(IoSession session) throws Exception { - // empty - } - +public class MongoProtocolEncoder implements ProtocolEncoder +{ + private static final Logger LOGGER; + + public void encode(final IoSession session, final Object message, final ProtocolEncoderOutput out) throws Exception { + try { + if (message instanceof MongoMessage) { + final MongoMessage mongoMessage = (MongoMessage)message; + final BSONOutputStream output = new BSONOutputStream(mongoMessage.getMessageHeader().getMessageLength()); + mongoMessage.write(output); + out.write((Object)IoBuffer.wrap(output.toByteArray())); + } + } + catch (Exception e) { + final ProtocolEncoderException pe = new ProtocolEncoderException("error encode message: " + message, (Throwable)e); + MongoProtocolEncoder.LOGGER.error("", (Throwable)pe); + throw pe; + } + } + + public void dispose(final IoSession session) throws Exception { + } + + static { + LOGGER = LoggerFactory.getLogger((Class)MongoProtocolEncoder.class); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/DeleteMessage.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/DeleteMessage.java index e421cd55b..43b369f75 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/DeleteMessage.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/DeleteMessage.java @@ -1,111 +1,98 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; -import java.io.IOException; - -import org.bson.BSONDecoder; -import org.bson.BSONEncoder; -import org.bson.BSONObject; -import org.bson.BasicBSONDecoder; -import org.bson.BasicBSONObject; - -import com.fangjia.datastore.mongo.codec.BSONInputStream; -import com.fangjia.datastore.mongo.codec.BSONOutputStream; import com.fangjia.datastore.utils.BSONUtils; +import org.bson.BSONDecoder; +import java.io.InputStream; +import org.bson.BasicBSONDecoder; +import com.fangjia.datastore.mongo.codec.BSONInputStream; +import java.io.IOException; +import org.bson.BSONEncoder; +import org.bson.BasicBSONEncoder; +import com.fangjia.datastore.mongo.codec.BSONOutputStream; +import org.bson.BasicBSONObject; +import org.bson.BSONObject; -public class DeleteMessage implements MongoMessage { - - private static final long serialVersionUID = -3350216587439425208L; - private MessageHeader messageHeader; - private String fullCollectionName; - private int flags; - private BSONObject selector; - - public DeleteMessage() { - messageHeader = new MessageHeader(OperationCode.OP_DELETE); - selector = new BasicBSONObject(); - } - - public void write(BSONOutputStream out) throws IOException { - messageHeader.write(out); - out.writeInteger(0); - out.writeCString(this.fullCollectionName); - out.writeInteger(this.flags); - BSONEncoder encoder = new BSONEncoder(); - out.write(encoder.encode(selector)); - out.set(0, out.size()); - } - - public void read(BSONInputStream in) throws IOException { - messageHeader.read(in); - in.readInteger(); - this.fullCollectionName = in.readCString(); - this.flags = in.readInteger(); - BSONDecoder decoder = new BasicBSONDecoder(); - selector = decoder.readObject(in); - } - - public MessageHeader getMessageHeader() { - return messageHeader; - } - - public String getFullCollectionName() { - return fullCollectionName; - } - - public void setFullCollectionName(String fullCollectionName) { - this.fullCollectionName = fullCollectionName; - } - - public int getFlags() { - return flags; - } - - public void setFlags(int flags) { - this.flags = flags; - } - - public void setSingleRemove(boolean b) { - this.flags = (int) BSONUtils.bitSet(this.flags, 0, b); - } - - public BSONObject getSelector() { - return selector; - } - - public void setSelector(BSONObject selector) { - this.selector = selector; - } - - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("messageHeader:").append(messageHeader); - builder.append(", fullCollectionName:").append(fullCollectionName); - builder.append(", flags:").append(flags); - builder.append(", selector:").append(selector); - builder.append("}"); - return builder.toString(); - } - - public String toPrettyString() { - return toString(); - } - +public class DeleteMessage implements MongoMessage +{ + private static final long serialVersionUID = -3350216587439425208L; + private MessageHeader messageHeader; + private String fullCollectionName; + private int flags; + private BSONObject selector; + + public DeleteMessage() { + this.messageHeader = new MessageHeader(OperationCode.OP_DELETE); + this.selector = (BSONObject)new BasicBSONObject(); + } + + @Override + public void write(final BSONOutputStream out) throws IOException { + this.messageHeader.write(out); + out.writeInteger(0); + out.writeCString(this.fullCollectionName); + out.writeInteger(this.flags); + final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder(); + out.write(encoder.encode(this.selector)); + out.set(0, out.size()); + } + + @Override + public void read(final BSONInputStream in) throws IOException { + this.messageHeader.read(in); + in.readInteger(); + this.fullCollectionName = in.readCString(); + this.flags = in.readInteger(); + final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder(); + this.selector = decoder.readObject((InputStream)in); + } + + @Override + public MessageHeader getMessageHeader() { + return this.messageHeader; + } + + public String getFullCollectionName() { + return this.fullCollectionName; + } + + public void setFullCollectionName(final String fullCollectionName) { + this.fullCollectionName = fullCollectionName; + } + + public int getFlags() { + return this.flags; + } + + public void setFlags(final int flags) { + this.flags = flags; + } + + public void setSingleRemove(final boolean b) { + this.flags = (int)BSONUtils.bitSet(this.flags, 0, b); + } + + public BSONObject getSelector() { + return this.selector; + } + + public void setSelector(final BSONObject selector) { + this.selector = selector; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("messageHeader:").append(this.messageHeader); + builder.append(", fullCollectionName:").append(this.fullCollectionName); + builder.append(", flags:").append(this.flags); + builder.append(", selector:").append(this.selector); + builder.append("}"); + return builder.toString(); + } + + @Override + public String toPrettyString() { + return this.toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/GetMoreMessage.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/GetMoreMessage.java index dff96af93..8d44fe088 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/GetMoreMessage.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/GetMoreMessage.java @@ -1,97 +1,83 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; -import java.io.IOException; - import com.fangjia.datastore.mongo.codec.BSONInputStream; +import java.io.IOException; import com.fangjia.datastore.mongo.codec.BSONOutputStream; -public class GetMoreMessage implements MongoMessage { - - private static final long serialVersionUID = -3350216587439425208L; - private MessageHeader messageHeader; - private String fullCollectionName; - private int numberToReturn; - private long cursorID; - - public GetMoreMessage() { - messageHeader = new MessageHeader(OperationCode.OP_GET_MORE); - } - - public void write(BSONOutputStream out) throws IOException { - messageHeader.write(out); - out.writeInteger(0); - out.writeCString(this.fullCollectionName); - out.writeInteger(this.numberToReturn); - out.writeLong(cursorID); - out.set(0, out.size()); - } - - public void read(BSONInputStream in) throws IOException { - messageHeader.read(in); - in.readInteger(); - this.fullCollectionName = in.readCString(); - this.numberToReturn = in.readInteger(); - this.cursorID = in.readLong(); - } - - public MessageHeader getMessageHeader() { - return messageHeader; - } - - public String getFullCollectionName() { - return fullCollectionName; - } - - public void setFullCollectionName(String fullCollectionName) { - this.fullCollectionName = fullCollectionName; - } - - public int getNumberToReturn() { - return numberToReturn; - } - - public void setNumberToReturn(int numberToReturn) { - this.numberToReturn = numberToReturn; - } - - public long getCursorID() { - return cursorID; - } - - public void setCursorID(long cursorID) { - this.cursorID = cursorID; - } - - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("messageHeader:").append(messageHeader); - builder.append(", fullCollectionName:").append(fullCollectionName); - builder.append(", numberToReturn:").append(numberToReturn); - builder.append(", cursorID:").append(cursorID); - builder.append("}"); - return builder.toString(); - } - - public String toPrettyString() { - return toString(); - } - +public class GetMoreMessage implements MongoMessage +{ + private static final long serialVersionUID = -3350216587439425208L; + private MessageHeader messageHeader; + private String fullCollectionName; + private int numberToReturn; + private long cursorID; + + public GetMoreMessage() { + this.messageHeader = new MessageHeader(OperationCode.OP_GET_MORE); + } + + @Override + public void write(final BSONOutputStream out) throws IOException { + this.messageHeader.write(out); + out.writeInteger(0); + out.writeCString(this.fullCollectionName); + out.writeInteger(this.numberToReturn); + out.writeLong(this.cursorID); + out.set(0, out.size()); + } + + @Override + public void read(final BSONInputStream in) throws IOException { + this.messageHeader.read(in); + in.readInteger(); + this.fullCollectionName = in.readCString(); + this.numberToReturn = in.readInteger(); + this.cursorID = in.readLong(); + } + + @Override + public MessageHeader getMessageHeader() { + return this.messageHeader; + } + + public String getFullCollectionName() { + return this.fullCollectionName; + } + + public void setFullCollectionName(final String fullCollectionName) { + this.fullCollectionName = fullCollectionName; + } + + public int getNumberToReturn() { + return this.numberToReturn; + } + + public void setNumberToReturn(final int numberToReturn) { + this.numberToReturn = numberToReturn; + } + + public long getCursorID() { + return this.cursorID; + } + + public void setCursorID(final long cursorID) { + this.cursorID = cursorID; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("messageHeader:").append(this.messageHeader); + builder.append(", fullCollectionName:").append(this.fullCollectionName); + builder.append(", numberToReturn:").append(this.numberToReturn); + builder.append(", cursorID:").append(this.cursorID); + builder.append("}"); + return builder.toString(); + } + + @Override + public String toPrettyString() { + return this.toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/InsertMessage.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/InsertMessage.java index 9c9093e88..ce30fc4a6 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/InsertMessage.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/InsertMessage.java @@ -1,116 +1,105 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; -import java.io.IOException; +import org.bson.BSONDecoder; +import java.io.InputStream; +import org.bson.BasicBSONDecoder; import java.util.ArrayList; +import com.fangjia.datastore.mongo.codec.BSONInputStream; +import java.io.IOException; +import java.util.Iterator; +import org.bson.BSONEncoder; +import org.bson.BasicBSONEncoder; +import com.fangjia.datastore.mongo.codec.BSONOutputStream; +import org.bson.BSONObject; import java.util.List; -import org.bson.BSONDecoder; -import org.bson.BSONEncoder; -import org.bson.BSONObject; -import org.bson.BasicBSONDecoder; - -import com.fangjia.datastore.mongo.codec.BSONInputStream; -import com.fangjia.datastore.mongo.codec.BSONOutputStream; - -public class InsertMessage implements MongoMessage { - - private static final long serialVersionUID = -3350216587439425208L; - private MessageHeader messageHeader; - private String fullCollectionName; - private List documents; - - public InsertMessage() { - messageHeader = new MessageHeader(OperationCode.OP_INSERT); - } - - public void write(BSONOutputStream out) throws IOException { - messageHeader.write(out); - out.writeInteger(0); - out.writeCString(this.fullCollectionName); - if (null != documents && documents.size() > 0) { - BSONEncoder encoder = new BSONEncoder(); - for (BSONObject doc : documents) { - out.write(encoder.encode(doc)); - } - } - out.setInteger(0, out.size()); - } - - public void read(BSONInputStream in) throws IOException { - messageHeader.read(in); - in.readInteger(); - this.fullCollectionName = in.readCString(); - while (in.available() > 0) { - if (null == documents) { - documents = new ArrayList(); - } - BSONDecoder decoder = new BasicBSONDecoder(); - documents.add(decoder.readObject(in)); - } - } - - public MessageHeader getMessageHeader() { - return messageHeader; - } - - public String getFullCollectionName() { - return fullCollectionName; - } - - public void setFullCollectionName(String fullCollectionName) { - this.fullCollectionName = fullCollectionName; - } - - public List getDocuments() { - return documents; - } - - public void setDocuments(List documents) { - this.documents = documents; - } - - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("messageHeader:").append(messageHeader); - builder.append(", fullCollectionName:").append(fullCollectionName); - builder.append(", documents:").append(documents); - builder.append("}"); - return builder.toString(); - } - - public String toPrettyString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("messageHeader:").append(messageHeader); - builder.append(", fullCollectionName:").append(fullCollectionName); - if (null != documents && documents.size() > 0) { - builder.append(", documents:").append(documents.get(0).get("_id")); - if (documents.size() > 1) { - builder.append("...").append(documents.size() - 1).append(" more"); - } - } else { - builder.append(", documents:").append(documents); - } - builder.append("}"); - return builder.toString(); - } - +public class InsertMessage implements MongoMessage +{ + private static final long serialVersionUID = -3350216587439425208L; + private MessageHeader messageHeader; + private String fullCollectionName; + private List documents; + + public InsertMessage() { + this.messageHeader = new MessageHeader(OperationCode.OP_INSERT); + } + + @Override + public void write(final BSONOutputStream out) throws IOException { + this.messageHeader.write(out); + out.writeInteger(0); + out.writeCString(this.fullCollectionName); + if (null != this.documents && this.documents.size() > 0) { + final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder(); + for (final BSONObject doc : this.documents) { + out.write(encoder.encode(doc)); + } + } + out.setInteger(0, out.size()); + } + + @Override + public void read(final BSONInputStream in) throws IOException { + this.messageHeader.read(in); + in.readInteger(); + this.fullCollectionName = in.readCString(); + while (in.available() > 0) { + if (null == this.documents) { + this.documents = new ArrayList(); + } + final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder(); + this.documents.add(decoder.readObject((InputStream)in)); + } + } + + @Override + public MessageHeader getMessageHeader() { + return this.messageHeader; + } + + public String getFullCollectionName() { + return this.fullCollectionName; + } + + public void setFullCollectionName(final String fullCollectionName) { + this.fullCollectionName = fullCollectionName; + } + + public List getDocuments() { + return this.documents; + } + + public void setDocuments(final List documents) { + this.documents = documents; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("messageHeader:").append(this.messageHeader); + builder.append(", fullCollectionName:").append(this.fullCollectionName); + builder.append(", documents:").append(this.documents); + builder.append("}"); + return builder.toString(); + } + + @Override + public String toPrettyString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("messageHeader:").append(this.messageHeader); + builder.append(", fullCollectionName:").append(this.fullCollectionName); + if (null != this.documents && this.documents.size() > 0) { + builder.append(", documents:").append(this.documents.get(0).get("_id")); + if (this.documents.size() > 1) { + builder.append("...").append(this.documents.size() - 1).append(" more"); + } + } + else { + builder.append(", documents:").append(this.documents); + } + builder.append("}"); + return builder.toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/KillCursorsMessage.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/KillCursorsMessage.java index ee5f41a8d..3a36d8d4d 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/KillCursorsMessage.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/KillCursorsMessage.java @@ -1,96 +1,83 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; -import java.io.IOException; import java.util.ArrayList; +import com.fangjia.datastore.mongo.codec.BSONInputStream; +import java.io.IOException; +import java.util.Iterator; +import com.fangjia.datastore.mongo.codec.BSONOutputStream; import java.util.List; -import com.fangjia.datastore.mongo.codec.BSONInputStream; -import com.fangjia.datastore.mongo.codec.BSONOutputStream; - -public class KillCursorsMessage implements MongoMessage { - - private static final long serialVersionUID = -3350216587439425208L; - private MessageHeader messageHeader; - private int numberOfCursorIDs; - private List cursorIDs; - - public KillCursorsMessage() { - messageHeader = new MessageHeader(OperationCode.OP_KILL_CURSORS); - } - - public MessageHeader getMessageHeader() { - return messageHeader; - } - - public void write(BSONOutputStream out) throws IOException { - messageHeader.write(out); - out.writeInteger(0); - out.writeInteger(numberOfCursorIDs); - if (null != cursorIDs) { - for (Long id : cursorIDs) { - out.writeLong(id); - } - } - out.set(0, out.size()); - } - - public void read(BSONInputStream in) throws IOException { - messageHeader.read(in); - in.readInteger(); - numberOfCursorIDs = in.readInteger(); - while (in.available() > 0) { - if (null == cursorIDs) { - cursorIDs = new ArrayList(); - } - cursorIDs.add(in.readLong()); - } - } - - public int getNumberOfCursorIDs() { - return numberOfCursorIDs; - } - - public void setNumberOfCursorIDs(int numberOfCursorIDs) { - this.numberOfCursorIDs = numberOfCursorIDs; - } - - public List getCursorIDs() { - return cursorIDs; - } - - public void setCursorIDs(List cursorIDs) { - this.cursorIDs = cursorIDs; - } - - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("messageHeader:").append(messageHeader); - builder.append(", numberOfCursorIDs:").append(numberOfCursorIDs); - builder.append(", cursorIDs:").append(cursorIDs); - builder.append("}"); - return builder.toString(); - } - - public String toPrettyString() { - return toString(); - } - +public class KillCursorsMessage implements MongoMessage +{ + private static final long serialVersionUID = -3350216587439425208L; + private MessageHeader messageHeader; + private int numberOfCursorIDs; + private List cursorIDs; + + public KillCursorsMessage() { + this.messageHeader = new MessageHeader(OperationCode.OP_KILL_CURSORS); + } + + @Override + public MessageHeader getMessageHeader() { + return this.messageHeader; + } + + @Override + public void write(final BSONOutputStream out) throws IOException { + this.messageHeader.write(out); + out.writeInteger(0); + out.writeInteger(this.numberOfCursorIDs); + if (null != this.cursorIDs) { + for (final Long id : this.cursorIDs) { + out.writeLong(id); + } + } + out.set(0, out.size()); + } + + @Override + public void read(final BSONInputStream in) throws IOException { + this.messageHeader.read(in); + in.readInteger(); + this.numberOfCursorIDs = in.readInteger(); + while (in.available() > 0) { + if (null == this.cursorIDs) { + this.cursorIDs = new ArrayList(); + } + this.cursorIDs.add(in.readLong()); + } + } + + public int getNumberOfCursorIDs() { + return this.numberOfCursorIDs; + } + + public void setNumberOfCursorIDs(final int numberOfCursorIDs) { + this.numberOfCursorIDs = numberOfCursorIDs; + } + + public List getCursorIDs() { + return this.cursorIDs; + } + + public void setCursorIDs(final List cursorIDs) { + this.cursorIDs = cursorIDs; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("messageHeader:").append(this.messageHeader); + builder.append(", numberOfCursorIDs:").append(this.numberOfCursorIDs); + builder.append(", cursorIDs:").append(this.cursorIDs); + builder.append("}"); + return builder.toString(); + } + + @Override + public String toPrettyString() { + return this.toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/MessageHeader.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/MessageHeader.java index 9430053af..97fa2c238 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/MessageHeader.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/MessageHeader.java @@ -1,106 +1,93 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; -import java.io.IOException; -import java.io.Serializable; -import java.util.concurrent.atomic.AtomicInteger; - -import com.fangjia.datastore.mongo.codec.BSONInputStream; import com.fangjia.datastore.mongo.codec.BSONOutputStream; +import java.io.IOException; +import com.fangjia.datastore.mongo.codec.BSONInputStream; +import java.util.concurrent.atomic.AtomicInteger; +import java.io.Serializable; -public final class MessageHeader implements Writable, Readable, Serializable { - private static final long serialVersionUID = 8505270142383450097L; - private static final AtomicInteger ID_GENERATOR = new AtomicInteger(1); - - private int messageLength; - private int requestID; - private int responseTo; - private OperationCode opCode; - - public MessageHeader(OperationCode opCode) { - this.opCode = opCode; - this.requestID = ID_GENERATOR.getAndIncrement(); - } - - public MessageHeader() { - } - - public void read(BSONInputStream in) throws IOException { - messageLength = in.readInteger(); - requestID = in.readInteger(); - responseTo = in.readInteger(); - opCode = OperationCode.valueOf(in.readInteger()); - } - - public void write(BSONOutputStream out) throws IOException { - out.writeInteger(messageLength); - out.writeInteger(requestID); - out.writeInteger(responseTo); - out.writeInteger(opCode.getValue()); - } - - public int size() { - return 16; - } - - public int getMessageLength() { - return messageLength; - } - - public void setMessageLength(int messageLength) { - this.messageLength = messageLength; - } - - public int getRequestID() { - return requestID; - } - - public void setRequestID(int requestID) { - this.requestID = requestID; - } - - public int getResponseTo() { - return responseTo; - } - - public void setResponseTo(int responseTo) { - this.responseTo = responseTo; - } - - public OperationCode getOpCode() { - return opCode; - } - - public void setOpCode(OperationCode opCode) { - this.opCode = opCode; - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("messageLength:").append(messageLength); - builder.append(", requestID:").append(requestID); - builder.append(", responseTo:").append(responseTo); - builder.append(", opCode:").append(opCode); - builder.append("}"); - return builder.toString(); - } - +public final class MessageHeader implements Writable, Readable, Serializable +{ + private static final long serialVersionUID = 8505270142383450097L; + private static final AtomicInteger ID_GENERATOR; + private int messageLength; + private int requestID; + private int responseTo; + private OperationCode opCode; + + public MessageHeader(final OperationCode opCode) { + this.opCode = opCode; + this.requestID = MessageHeader.ID_GENERATOR.getAndIncrement(); + } + + public MessageHeader() { + } + + @Override + public void read(final BSONInputStream in) throws IOException { + this.messageLength = in.readInteger(); + this.requestID = in.readInteger(); + this.responseTo = in.readInteger(); + this.opCode = OperationCode.valueOf(in.readInteger()); + } + + @Override + public void write(final BSONOutputStream out) throws IOException { + out.writeInteger(this.messageLength); + out.writeInteger(this.requestID); + out.writeInteger(this.responseTo); + out.writeInteger(this.opCode.getValue()); + } + + public int size() { + return 16; + } + + public int getMessageLength() { + return this.messageLength; + } + + public void setMessageLength(final int messageLength) { + this.messageLength = messageLength; + } + + public int getRequestID() { + return this.requestID; + } + + public void setRequestID(final int requestID) { + this.requestID = requestID; + } + + public int getResponseTo() { + return this.responseTo; + } + + public void setResponseTo(final int responseTo) { + this.responseTo = responseTo; + } + + public OperationCode getOpCode() { + return this.opCode; + } + + public void setOpCode(final OperationCode opCode) { + this.opCode = opCode; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("messageLength:").append(this.messageLength); + builder.append(", requestID:").append(this.requestID); + builder.append(", responseTo:").append(this.responseTo); + builder.append(", opCode:").append(this.opCode); + builder.append("}"); + return builder.toString(); + } + + static { + ID_GENERATOR = new AtomicInteger(1); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/MongoMessage.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/MongoMessage.java index 35f01104a..ea571f11c 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/MongoMessage.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/MongoMessage.java @@ -1,26 +1,10 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; import java.io.Serializable; -public interface MongoMessage extends Readable, Writable, Serializable { - public MessageHeader getMessageHeader(); - - public String toPrettyString(); +public interface MongoMessage extends Readable, Writable, Serializable +{ + MessageHeader getMessageHeader(); + + String toPrettyString(); } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/OperationCode.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/OperationCode.java index 0ceb27e6d..cf7fbde2b 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/OperationCode.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/OperationCode.java @@ -1,82 +1,39 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; import java.util.HashMap; import java.util.Map; -public enum OperationCode { - /** - * Reply to a client request. responseTo is set - */ - OP_REPLY(1), - /** - * generic msg command followed by a string - */ - OP_MSG(1000), - /** - * update document - */ - OP_UPDATE(2001), - /** - * insert new document - */ - OP_INSERT(2002), - /** - * formerly used for OP_GET_BY_OID - */ - RESERVED(2003), - /** - * query a collection - */ - OP_QUERY(2004), - /** - * Get more data from a query. See Cursors - */ - OP_GET_MORE(2005), - /** - * Delete documents - */ - OP_DELETE(2006), - /** - * Tell database client is done with a cursor - */ - OP_KILL_CURSORS(2007); - - private final int value; - - private final static Map TYPE_MAP = new HashMap(); - static { - for (OperationCode e : OperationCode.values()) { - TYPE_MAP.put(e.value, e); - } - } - - private OperationCode(int value) { - this.value = value; - } - - public int getValue() { - return value; - } - - public static OperationCode valueOf(int value) { - return TYPE_MAP.get(value); - } - +public enum OperationCode +{ + OP_REPLY(1), + OP_MSG(1000), + OP_UPDATE(2001), + OP_INSERT(2002), + RESERVED(2003), + OP_QUERY(2004), + OP_GET_MORE(2005), + OP_DELETE(2006), + OP_KILL_CURSORS(2007); + + private final int value; + private static final Map TYPE_MAP; + + private OperationCode(final int value) { + this.value = value; + } + + public int getValue() { + return this.value; + } + + public static OperationCode valueOf(final int value) { + return OperationCode.TYPE_MAP.get(value); + } + + static { + TYPE_MAP = new HashMap(); + for (final OperationCode e : values()) { + OperationCode.TYPE_MAP.put(e.value, e); + } + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/QueryMessage.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/QueryMessage.java index e82245fa0..ddf724569 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/QueryMessage.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/QueryMessage.java @@ -1,213 +1,197 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; -import java.io.IOException; - -import org.bson.BSONDecoder; -import org.bson.BSONEncoder; -import org.bson.BSONObject; -import org.bson.BasicBSONDecoder; - -import com.fangjia.datastore.mongo.codec.BSONInputStream; -import com.fangjia.datastore.mongo.codec.BSONOutputStream; import com.fangjia.datastore.utils.BSONUtils; +import org.bson.BSONDecoder; +import java.io.InputStream; +import org.bson.BasicBSONDecoder; +import com.fangjia.datastore.mongo.codec.BSONInputStream; +import java.io.IOException; +import org.bson.BSONEncoder; +import org.bson.BasicBSONEncoder; +import com.fangjia.datastore.mongo.codec.BSONOutputStream; +import org.bson.BSONObject; -public class QueryMessage implements MongoMessage { - - private static final long serialVersionUID = -3350216587439425208L; - private MessageHeader messageHeader; - private int options; - private int numberToSkip; - private int numberToReturn; - private String fullCollectionName; - private BSONObject query; - private BSONObject returnFieldSelector; - - public QueryMessage() { - messageHeader = new MessageHeader(OperationCode.OP_QUERY); - } - - public void write(BSONOutputStream out) throws IOException { - messageHeader.write(out); - out.writeInteger(options); - out.writeCString(this.fullCollectionName); - out.writeInteger(numberToSkip); - out.writeInteger(numberToReturn); - BSONEncoder encoder = new BSONEncoder(); - out.write(encoder.encode(query)); - if (null != returnFieldSelector) { - out.write(encoder.encode(returnFieldSelector)); - } - out.set(0, out.size()); - } - - public void read(BSONInputStream in) throws IOException { - messageHeader.read(in); - options = in.readInteger(); - this.fullCollectionName = in.readCString(); - this.numberToSkip = in.readInteger(); - this.numberToReturn = in.readInteger(); - BSONDecoder decoder = new BasicBSONDecoder(); - query = decoder.readObject(in); - if (in.available() > 0) { - returnFieldSelector = decoder.readObject(in); - } - } - - public MessageHeader getMessageHeader() { - return messageHeader; - } - - public String getFullCollectionName() { - return fullCollectionName; - } - - public void setFullCollectionName(String fullCollectionName) { - this.fullCollectionName = fullCollectionName; - } - - public int getOptions() { - return options; - } - - public void setOptions(int options) { - this.options = options; - } - - public void setNoCursorTimeout(boolean b) { - this.options = (int) BSONUtils.bitSet(options, 4, b); - } - - public void setTailable(boolean b) { - this.options = (int) BSONUtils.bitSet(options, 1, b); - } - - public void setSlaveOk(boolean b) { - this.options = (int) BSONUtils.bitSet(options, 2, b); - } - - public void setAwaitData(boolean b) { - this.options = (int) BSONUtils.bitSet(options, 5, b); - } - - public int getNumberToSkip() { - return numberToSkip; - } - - public void setNumberToSkip(int numberToSkip) { - this.numberToSkip = numberToSkip; - } - - public int getNumberToReturn() { - return numberToReturn; - } - - public void setNumberToReturn(int numberToReturn) { - this.numberToReturn = numberToReturn; - } - - public BSONObject getQuery() { - return query; - } - - public void setQuery(BSONObject query) { - this.query = query; - } - - public BSONObject getRealQuery() { - if (null == query) { - return null; - } - BSONObject q = (BSONObject) query.get("$query"); - if (null == q) { - q = (BSONObject) query.get("query"); - if (null == q) { - q = query; - } - } - return q; - } - - public BSONObject getOrderByQuery() { - if (null == query) { - return null; - } - BSONObject q = (BSONObject) query.get("$orderby"); - if (null == q) { - q = (BSONObject) query.get("orderby"); - } - return q; - } - - public BSONObject getFacetQuery() { - BSONObject q = getRealQuery(); - if (null == q) { - return null; - } - return (BSONObject) q.get("$facet"); - } - - public boolean isBulkQuery() { - BSONObject q = getRealQuery(); - if (null == q) { - return true; - } - Boolean bulk = (Boolean) q.get("$bulk"); - - return null == bulk ? true : bulk; - } - - public boolean isFromMaster() { - BSONObject q = getRealQuery(); - if (null == q) { - return false; - } - Boolean master = (Boolean) q.get("$master"); - - return null == master ? false : master; - } - - public BSONObject getReturnFieldSelector() { - return returnFieldSelector; - } - - public void setReturnFieldSelector(BSONObject returnFieldSelector) { - this.returnFieldSelector = returnFieldSelector; - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("messageHeader:").append(messageHeader); - builder.append(", options:").append(options); - builder.append(", numberToSkip:").append(numberToSkip); - builder.append(", numberToReturn:").append(numberToReturn); - builder.append(", fullCollectionName:").append(fullCollectionName); - builder.append(", query:").append(query); - builder.append(", returnFieldSelector:").append(returnFieldSelector); - builder.append("}"); - return builder.toString(); - } - - public String toPrettyString() { - return toString(); - } - +public class QueryMessage implements MongoMessage +{ + private static final long serialVersionUID = -3350216587439425208L; + private MessageHeader messageHeader; + private int options; + private int numberToSkip; + private int numberToReturn; + private String fullCollectionName; + private BSONObject query; + private BSONObject returnFieldSelector; + + public QueryMessage() { + this.messageHeader = new MessageHeader(OperationCode.OP_QUERY); + } + + @Override + public void write(final BSONOutputStream out) throws IOException { + this.messageHeader.write(out); + out.writeInteger(this.options); + out.writeCString(this.fullCollectionName); + out.writeInteger(this.numberToSkip); + out.writeInteger(this.numberToReturn); + final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder(); + out.write(encoder.encode(this.query)); + if (null != this.returnFieldSelector) { + out.write(encoder.encode(this.returnFieldSelector)); + } + out.set(0, out.size()); + } + + @Override + public void read(final BSONInputStream in) throws IOException { + this.messageHeader.read(in); + this.options = in.readInteger(); + this.fullCollectionName = in.readCString(); + this.numberToSkip = in.readInteger(); + this.numberToReturn = in.readInteger(); + final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder(); + this.query = decoder.readObject((InputStream)in); + if (in.available() > 0) { + this.returnFieldSelector = decoder.readObject((InputStream)in); + } + } + + @Override + public MessageHeader getMessageHeader() { + return this.messageHeader; + } + + public String getFullCollectionName() { + return this.fullCollectionName; + } + + public void setFullCollectionName(final String fullCollectionName) { + this.fullCollectionName = fullCollectionName; + } + + public int getOptions() { + return this.options; + } + + public void setOptions(final int options) { + this.options = options; + } + + public void setNoCursorTimeout(final boolean b) { + this.options = (int)BSONUtils.bitSet(this.options, 4, b); + } + + public void setTailable(final boolean b) { + this.options = (int)BSONUtils.bitSet(this.options, 1, b); + } + + public void setSlaveOk(final boolean b) { + this.options = (int)BSONUtils.bitSet(this.options, 2, b); + } + + public void setAwaitData(final boolean b) { + this.options = (int)BSONUtils.bitSet(this.options, 5, b); + } + + public int getNumberToSkip() { + return this.numberToSkip; + } + + public void setNumberToSkip(final int numberToSkip) { + this.numberToSkip = numberToSkip; + } + + public int getNumberToReturn() { + return this.numberToReturn; + } + + public void setNumberToReturn(final int numberToReturn) { + this.numberToReturn = numberToReturn; + } + + public BSONObject getQuery() { + return this.query; + } + + public void setQuery(final BSONObject query) { + this.query = query; + } + + public BSONObject getRealQuery() { + if (null == this.query) { + return null; + } + BSONObject q = (BSONObject)this.query.get("$query"); + if (null == q) { + q = (BSONObject)this.query.get("query"); + if (null == q) { + q = this.query; + } + } + return q; + } + + public BSONObject getOrderByQuery() { + if (null == this.query) { + return null; + } + BSONObject q = (BSONObject)this.query.get("$orderby"); + if (null == q) { + q = (BSONObject)this.query.get("orderby"); + } + return q; + } + + public BSONObject getFacetQuery() { + final BSONObject q = this.getRealQuery(); + if (null == q) { + return null; + } + return (BSONObject)q.get("$facet"); + } + + public boolean isBulkQuery() { + final BSONObject q = this.getRealQuery(); + if (null == q) { + return true; + } + final Boolean bulk = (Boolean)q.get("$bulk"); + return null == bulk || bulk; + } + + public boolean isFromMaster() { + final BSONObject q = this.getRealQuery(); + if (null == q) { + return false; + } + final Boolean master = (Boolean)q.get("$master"); + return null != master && master; + } + + public BSONObject getReturnFieldSelector() { + return this.returnFieldSelector; + } + + public void setReturnFieldSelector(final BSONObject returnFieldSelector) { + this.returnFieldSelector = returnFieldSelector; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("messageHeader:").append(this.messageHeader); + builder.append(", options:").append(this.options); + builder.append(", numberToSkip:").append(this.numberToSkip); + builder.append(", numberToReturn:").append(this.numberToReturn); + builder.append(", fullCollectionName:").append(this.fullCollectionName); + builder.append(", query:").append(this.query); + builder.append(", returnFieldSelector:").append(this.returnFieldSelector); + builder.append("}"); + return builder.toString(); + } + + @Override + public String toPrettyString() { + return this.toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/Readable.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/Readable.java index 0aa14d6c1..cb0dbacf3 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/Readable.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/Readable.java @@ -1,26 +1,9 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; import java.io.IOException; - import com.fangjia.datastore.mongo.codec.BSONInputStream; -public interface Readable { - public void read(BSONInputStream in) throws IOException; +public interface Readable +{ + void read(final BSONInputStream p0) throws IOException; } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/ReplyMessage.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/ReplyMessage.java index 806849823..d133315a3 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/ReplyMessage.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/ReplyMessage.java @@ -1,138 +1,125 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; -import java.io.IOException; +import org.bson.BSONDecoder; +import java.io.InputStream; +import org.bson.BasicBSONDecoder; import java.util.ArrayList; +import com.fangjia.datastore.mongo.codec.BSONInputStream; +import java.io.IOException; +import java.util.Iterator; +import org.bson.BSONEncoder; +import org.bson.BasicBSONEncoder; +import com.fangjia.datastore.mongo.codec.BSONOutputStream; +import org.bson.BSONObject; import java.util.List; -import org.bson.BSONDecoder; -import org.bson.BSONEncoder; -import org.bson.BSONObject; -import org.bson.BasicBSONDecoder; - -import com.fangjia.datastore.mongo.codec.BSONInputStream; -import com.fangjia.datastore.mongo.codec.BSONOutputStream; - -public class ReplyMessage implements MongoMessage { - - private static final long serialVersionUID = -3350216587439425208L; - private MessageHeader messageHeader; - private int responseFlag; - private long cursorID; - private int startingFrom; - private int numberReturned; - private List documents; - - public ReplyMessage() { - messageHeader = new MessageHeader(OperationCode.OP_REPLY); - } - - public void write(BSONOutputStream out) throws IOException { - messageHeader.write(out); - out.writeInteger(responseFlag); - out.writeLong(cursorID); - out.writeInteger(startingFrom); - out.writeInteger(numberReturned); - if (null != this.documents) { - BSONEncoder encoder = new BSONEncoder(); - for (BSONObject doc : this.documents) { - out.write(encoder.encode(doc)); - } - } - out.set(0, out.size()); - } - - public void read(BSONInputStream in) throws IOException { - messageHeader.read(in); - responseFlag = in.readInteger(); - cursorID = in.readLong(); - startingFrom = in.readInteger(); - numberReturned = in.readInteger(); - documents = new ArrayList(numberReturned); - if (numberReturned > 0) { - BSONDecoder decoder = new BasicBSONDecoder(); - for (int i = 0; i < numberReturned; i++) { - documents.add(decoder.readObject(in)); - } - } - } - - public MessageHeader getMessageHeader() { - return messageHeader; - } - - public int getResponseFlag() { - return responseFlag; - } - - public void setResponseFlag(int responseFlag) { - this.responseFlag = responseFlag; - } - - public long getCursorID() { - return cursorID; - } - - public void setCursorID(long cursorID) { - this.cursorID = cursorID; - } - - public int getStartingFrom() { - return startingFrom; - } - - public void setStartingFrom(int startingFrom) { - this.startingFrom = startingFrom; - } - - public int getNumberReturned() { - return numberReturned; - } - - public void setNumberReturned(int numberReturned) { - this.numberReturned = numberReturned; - } - - public List getDocuments() { - return documents; - } - - public void setDocuments(List documents) { - this.documents = documents; - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("messageHeader:").append(messageHeader); - builder.append(", responseFlag:").append(responseFlag); - builder.append(", cursorID:").append(cursorID); - builder.append(", startingFrom:").append(startingFrom); - builder.append(", numberReturned:").append(numberReturned); - builder.append(", documents:").append(documents); - builder.append("}"); - return builder.toString(); - } - - public String toPrettyString() { - return toString(); - } - +public class ReplyMessage implements MongoMessage +{ + private static final long serialVersionUID = -3350216587439425208L; + private MessageHeader messageHeader; + private int responseFlag; + private long cursorID; + private int startingFrom; + private int numberReturned; + private List documents; + + public ReplyMessage() { + this.messageHeader = new MessageHeader(OperationCode.OP_REPLY); + } + + @Override + public void write(final BSONOutputStream out) throws IOException { + this.messageHeader.write(out); + out.writeInteger(this.responseFlag); + out.writeLong(this.cursorID); + out.writeInteger(this.startingFrom); + out.writeInteger(this.numberReturned); + if (null != this.documents) { + final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder(); + for (final BSONObject doc : this.documents) { + out.write(encoder.encode(doc)); + } + } + out.set(0, out.size()); + } + + @Override + public void read(final BSONInputStream in) throws IOException { + this.messageHeader.read(in); + this.responseFlag = in.readInteger(); + this.cursorID = in.readLong(); + this.startingFrom = in.readInteger(); + this.numberReturned = in.readInteger(); + this.documents = new ArrayList(this.numberReturned); + if (this.numberReturned > 0) { + final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder(); + for (int i = 0; i < this.numberReturned; ++i) { + this.documents.add(decoder.readObject((InputStream)in)); + } + } + } + + @Override + public MessageHeader getMessageHeader() { + return this.messageHeader; + } + + public int getResponseFlag() { + return this.responseFlag; + } + + public void setResponseFlag(final int responseFlag) { + this.responseFlag = responseFlag; + } + + public long getCursorID() { + return this.cursorID; + } + + public void setCursorID(final long cursorID) { + this.cursorID = cursorID; + } + + public int getStartingFrom() { + return this.startingFrom; + } + + public void setStartingFrom(final int startingFrom) { + this.startingFrom = startingFrom; + } + + public int getNumberReturned() { + return this.numberReturned; + } + + public void setNumberReturned(final int numberReturned) { + this.numberReturned = numberReturned; + } + + public List getDocuments() { + return this.documents; + } + + public void setDocuments(final List documents) { + this.documents = documents; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("messageHeader:").append(this.messageHeader); + builder.append(", responseFlag:").append(this.responseFlag); + builder.append(", cursorID:").append(this.cursorID); + builder.append(", startingFrom:").append(this.startingFrom); + builder.append(", numberReturned:").append(this.numberReturned); + builder.append(", documents:").append(this.documents); + builder.append("}"); + return builder.toString(); + } + + @Override + public String toPrettyString() { + return this.toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/UpdateMessage.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/UpdateMessage.java index 536d498fd..33bd4fdbd 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/UpdateMessage.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/UpdateMessage.java @@ -1,137 +1,123 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; -import java.io.IOException; - -import org.bson.BSONDecoder; -import org.bson.BSONEncoder; -import org.bson.BSONObject; -import org.bson.BasicBSONDecoder; -import org.bson.BasicBSONObject; - -import com.fangjia.datastore.mongo.codec.BSONInputStream; -import com.fangjia.datastore.mongo.codec.BSONOutputStream; import com.fangjia.datastore.utils.BSONUtils; +import org.bson.BSONDecoder; +import java.io.InputStream; +import org.bson.BasicBSONDecoder; +import com.fangjia.datastore.mongo.codec.BSONInputStream; +import java.io.IOException; +import org.bson.BSONEncoder; +import org.bson.BasicBSONEncoder; +import com.fangjia.datastore.mongo.codec.BSONOutputStream; +import org.bson.BasicBSONObject; +import org.bson.BSONObject; -public class UpdateMessage implements MongoMessage { - - private static final long serialVersionUID = -5278293474222136247L; - - private MessageHeader messageHeader; - private String fullCollectionName; - private int flags; - private BSONObject selector; - private BSONObject update; - - public UpdateMessage() { - messageHeader = new MessageHeader(OperationCode.OP_UPDATE); - this.selector = new BasicBSONObject(); - this.update = new BasicBSONObject(); - } - - public void write(BSONOutputStream out) throws IOException { - messageHeader.write(out); - out.writeInteger(0); - out.writeCString(this.fullCollectionName); - out.writeInteger(flags); - BSONEncoder encoder = new BSONEncoder(); - out.write(encoder.encode(selector)); - out.write(encoder.encode(update)); - out.set(0, out.size()); - } - - public void read(BSONInputStream in) throws IOException { - messageHeader.read(in); - in.readInteger(); - this.fullCollectionName = in.readCString(); - this.flags = in.readInteger(); - BSONDecoder decoder = new BasicBSONDecoder(); - selector = decoder.readObject(in); - update = decoder.readObject(in); - } - - public void setUpsert(boolean upsert) { - this.flags = (int) BSONUtils.bitSet(this.flags, 0, upsert); - } - - public boolean isUpsert() { - return BSONUtils.isBitSet(flags, 0); - } - - public void setMultiUpdate(boolean multiUpdate) { - this.flags = (int) BSONUtils.bitSet(this.flags, 1, multiUpdate); - } - - public boolean isMultiUpdate() { - return BSONUtils.isBitSet(flags, 1); - } - - public MessageHeader getMessageHeader() { - return messageHeader; - } - - public String getFullCollectionName() { - return fullCollectionName; - } - - public void setFullCollectionName(String fullCollectionName) { - this.fullCollectionName = fullCollectionName; - } - - public int getFlags() { - return flags; - } - - public void setFlags(int flags) { - this.flags = flags; - } - - public BSONObject getSelector() { - return selector; - } - - public void setSelector(BSONObject selector) { - this.selector = selector; - } - - public BSONObject getUpdate() { - return update; - } - - public void setUpdate(BSONObject update) { - this.update = update; - } - - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("messageHeader:").append(messageHeader); - builder.append(", fullCollectionName:").append(fullCollectionName); - builder.append(", flags:").append(flags); - builder.append(", selector:").append(selector); - builder.append(", update:").append(update); - builder.append("}"); - return builder.toString(); - } - - public String toPrettyString() { - return toString(); - } - +public class UpdateMessage implements MongoMessage +{ + private static final long serialVersionUID = -5278293474222136247L; + private MessageHeader messageHeader; + private String fullCollectionName; + private int flags; + private BSONObject selector; + private BSONObject update; + + public UpdateMessage() { + this.messageHeader = new MessageHeader(OperationCode.OP_UPDATE); + this.selector = (BSONObject)new BasicBSONObject(); + this.update = (BSONObject)new BasicBSONObject(); + } + + @Override + public void write(final BSONOutputStream out) throws IOException { + this.messageHeader.write(out); + out.writeInteger(0); + out.writeCString(this.fullCollectionName); + out.writeInteger(this.flags); + final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder(); + out.write(encoder.encode(this.selector)); + out.write(encoder.encode(this.update)); + out.set(0, out.size()); + } + + @Override + public void read(final BSONInputStream in) throws IOException { + this.messageHeader.read(in); + in.readInteger(); + this.fullCollectionName = in.readCString(); + this.flags = in.readInteger(); + final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder(); + this.selector = decoder.readObject((InputStream)in); + this.update = decoder.readObject((InputStream)in); + } + + public void setUpsert(final boolean upsert) { + this.flags = (int)BSONUtils.bitSet(this.flags, 0, upsert); + } + + public boolean isUpsert() { + return BSONUtils.isBitSet(this.flags, 0); + } + + public void setMultiUpdate(final boolean multiUpdate) { + this.flags = (int)BSONUtils.bitSet(this.flags, 1, multiUpdate); + } + + public boolean isMultiUpdate() { + return BSONUtils.isBitSet(this.flags, 1); + } + + @Override + public MessageHeader getMessageHeader() { + return this.messageHeader; + } + + public String getFullCollectionName() { + return this.fullCollectionName; + } + + public void setFullCollectionName(final String fullCollectionName) { + this.fullCollectionName = fullCollectionName; + } + + public int getFlags() { + return this.flags; + } + + public void setFlags(final int flags) { + this.flags = flags; + } + + public BSONObject getSelector() { + return this.selector; + } + + public void setSelector(final BSONObject selector) { + this.selector = selector; + } + + public BSONObject getUpdate() { + return this.update; + } + + public void setUpdate(final BSONObject update) { + this.update = update; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("messageHeader:").append(this.messageHeader); + builder.append(", fullCollectionName:").append(this.fullCollectionName); + builder.append(", flags:").append(this.flags); + builder.append(", selector:").append(this.selector); + builder.append(", update:").append(this.update); + builder.append("}"); + return builder.toString(); + } + + @Override + public String toPrettyString() { + return this.toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/Writable.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/Writable.java index 0b43ef8ba..aa55f121b 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/Writable.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/Writable.java @@ -1,26 +1,9 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; import java.io.IOException; - import com.fangjia.datastore.mongo.codec.BSONOutputStream; -public interface Writable { - public void write(BSONOutputStream out) throws IOException; +public interface Writable +{ + void write(final BSONOutputStream p0) throws IOException; } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/CommonsHttpSolrServer.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/CommonsHttpSolrServer.java index 3c92e83bc..369c8af94 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/CommonsHttpSolrServer.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/CommonsHttpSolrServer.java @@ -1,11 +1,7 @@ package com.fangjia.datastore.solr; -import java.io.IOException; -import java.net.MalformedURLException; - import org.apache.commons.io.IOUtils; import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.solr.client.solrj.SolrRequest; @@ -15,66 +11,61 @@ import org.apache.solr.common.util.NamedList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class CommonsHttpSolrServer extends HttpSolrServer { - private static final long serialVersionUID = -3881911436089005421L; +import java.io.IOException; +import java.net.MalformedURLException; - private final static Logger LOGGER = LoggerFactory.getLogger(CommonsHttpSolrServer.class); +public class CommonsHttpSolrServer extends HttpSolrServer +{ + private static final long serialVersionUID = -3881911436089005421L; + private static final Logger LOGGER; + private final HttpSolrClient solrServer; + private final String host; + private final String coreName; + + public CommonsHttpSolrServer(final String host, final String coreName, final HttpClient httpClient) throws MalformedURLException { + this.host = host; + this.coreName = coreName; + (this.solrServer = new HttpSolrClient("http://" + host + '/' + coreName)).setAllowCompression(true); + } + + @Override + public String getHost() { + return this.host; + } + + @Override + public String getCoreName() { + return this.coreName; + } + + @Override + public void reloadCore() throws SolrException { + try { + final HttpGet method = new HttpGet("http://" + this.host + "/admin/cores?action=RELOAD&core=" + this.coreName); + final HttpResponse response = this.solrServer.getHttpClient().execute(method); + if (response.getStatusLine().getStatusCode() != 200) { + throw new SolrException("[" + this.coreName + "] error reload core of solr[" + this.host + "]: " + IOUtils.toString(response.getEntity().getContent())); + } + if (LOGGER.isInfoEnabled()) { + LOGGER.info("[" + this.coreName + "] solr core of server[" + this.host + "] reloaded."); + } + if (LOGGER.isInfoEnabled()) { + LOGGER.info("[" + this.coreName + "] core reload success"); + } + } + catch (IOException e) { + throw new SolrException("[" + this.coreName + "] error reload core", e); + } + } - private final HttpSolrClient solrServer; - private final String host; - private final String coreName; + public NamedList request(final SolrRequest request, final String arg1) throws SolrServerException, IOException { + return this.solrServer.request(request); + } - public CommonsHttpSolrServer(String host, String coreName, HttpClient httpClient) throws MalformedURLException { - this.host = host; - this.coreName = coreName; - this.solrServer = new HttpSolrClient("http://" + host + '/' + coreName); - this.solrServer.setAllowCompression(true); - } - - @Override - public String getHost() { - return host; - } - - @Override - public String getCoreName() { - return coreName; - } - - @Override - public void reloadCore() throws SolrException { - try { - HttpGet method = new HttpGet("http://" + host + "/admin/cores?action=RELOAD&core=" + this.coreName); - HttpResponse response = solrServer.getHttpClient().execute(method); - if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { - throw new SolrException("[" + this.coreName + "] error reload core of solr[" + host + "]: " - + IOUtils.toString(response.getEntity().getContent())); - } else if (LOGGER.isInfoEnabled()) { - LOGGER.info("[" + this.coreName + "] solr core of server[" + host + "] reloaded."); - } - if (LOGGER.isInfoEnabled()) { - LOGGER.info("[" + this.coreName + "] core reload success"); - } - } catch (IOException e) { - throw new SolrException("[" + this.coreName + "] error reload core", e); - } - } - - /*@Override - public NamedList request(SolrRequest request) throws SolrServerException, IOException { - return solrServer.request(request); - - }*/ - - @Override - public NamedList request(SolrRequest request, String arg1) - throws SolrServerException, IOException { - return solrServer.request(request); - } - - @Override - public void shutdown() { - - } + public void shutdown() { + } + static { + LOGGER = LoggerFactory.getLogger(CommonsHttpSolrServer.class); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/Core.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/Core.java index 740fe9854..f217c6248 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/Core.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/Core.java @@ -1,138 +1,32 @@ package com.fangjia.datastore.solr; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; - -import org.apache.commons.lang.time.FastDateFormat; -import org.bson.BSONObject; -import org.bson.types.ObjectId; - import com.fangjia.datastore.QueryResponse; +import org.bson.types.ObjectId; +import java.util.List; +import org.bson.BSONObject; +import java.util.Iterator; +import java.util.Collection; +import org.apache.commons.lang.time.FastDateFormat; import com.fangjia.datastore.config.Reloadable; -/** - * 一个Core对应一个solr索引 - * - * @author velna - * - */ -public interface Core extends Reloadable { - - public final static int INDEX_BATCH_SIZE = 30000; - public final static FastDateFormat VERSION_DATE_FORMAT = FastDateFormat.getInstance("yyMMddHHmmssSSS"); - - /** - * 得到索引名称 - * - * @return - */ - public String getName(); - - /** - * 得到所有索引字段 - * - * @return - * @throws SolrException - */ - Collection getIndexes() throws SolrException; - - /** - * 添加索引字段,必要的话重建索引 - * - * @param indexes - * 添加的索引字段列表 - * @param iterator - * 用于遍历数据 - * @throws SolrException - */ - void ensureIndex(Collection indexes, Iterator iterator) throws SolrException; - - /** - * 添加索引字段,必要的话重建索引 - * - * @param index - * 添加的索引字段列表 - * @param iterator - * 用于遍历数据 - * @throws SolrException - */ - void ensureIndex(Index index, Iterator iterator) throws SolrException; - - /** - * 重建索引 - * - * @param objIdList - * 待重建的ObjIdList,全部重建时为null - * @param iterator - * 用于遍历数据 - */ - void reIndex(List objIdList, Iterator iterator) throws SolrException; - - /** - * 保存索引文档 - * - * @param doc - * @throws SolrException - */ - // void save(BSONObject doc) throws SolrException; - /** - * 保存多个索引文档 - * - * @param docList - * @throws SolrException - */ - // void save(List docList) throws SolrException; - /** - * 保存多个索引文档 - * - * @param iterator - * @throws SolrException - */ - void save(Iterator iterator) throws SolrException; - - /** - * 根据ID删除索引文档 - * - * @param id - * @throws SolrException - */ - // void deleteById(String id) throws SolrException; - /** - * 根据ID列表删除索引文档 - * - * @param ids - * @throws SolrException - */ - // void deleteById(List ids) throws SolrException; - /** - * 根据查询删除索引文档 - * - * @param query - * @throws SolrException - */ - void deleteByQuery(BSONObject query) throws SolrException; - - /** - * 查询索引 - * - * @param query - * 查询条件 - * @param sort - * 排序条件 - * @param facet - * facet查询条件 - * @param skip - * 从第skip条开始返回结果 - * @param limit - * 返回limit条结果 - * @param fromMaster - * 是否从主索引中读取数据 - * @param bulk - * 是否使用bulk查询 - * @return - * @throws SolrException - */ - QueryResponse find(BSONObject query, BSONObject sort, BSONObject facet, int skip, int limit, boolean fromMaster, boolean bulk) - throws SolrException; +public interface Core extends Reloadable +{ + public static final int INDEX_BATCH_SIZE = 30000; + public static final FastDateFormat VERSION_DATE_FORMAT = FastDateFormat.getInstance("yyMMddHHmmssSSS"); + + String getName(); + + Collection getIndexes() throws SolrException; + + void ensureIndex(final Collection p0, final Iterator p1) throws SolrException; + + void ensureIndex(final Index p0, final Iterator p1) throws SolrException; + + void reIndex(final List p0, final Iterator p1) throws SolrException; + + void save(final Iterator p0) throws SolrException; + + void deleteByQuery(final BSONObject p0) throws SolrException; + + QueryResponse find(final BSONObject p0, final BSONObject p1, final BSONObject p2, final int p3, final int p4, final boolean p5, final boolean p6) throws SolrException; } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/CoreImpl.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/CoreImpl.java index 7b4ea2efd..6467ed951 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/CoreImpl.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/CoreImpl.java @@ -1,485 +1,480 @@ package com.fangjia.datastore.solr; +import com.fangjia.datastore.utils.LoggerFactory; +import com.fangjia.datastore.TriggerExecuteException; +import com.fangjia.datastore.config.Trigger; +import com.fangjia.datastore.TriggerExecutor; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.client.solrj.response.FacetField; +import org.apache.solr.common.SolrDocument; import java.util.ArrayList; +import com.fangjia.datastore.QueryResponse; +import org.apache.commons.lang.StringUtils; +import com.mongodb.BasicDBObject; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.commons.collections.CollectionUtils; +import org.bson.types.ObjectId; +import java.util.List; import java.util.Arrays; import java.util.Collection; import java.util.Iterator; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Executor; -import java.util.concurrent.LinkedBlockingQueue; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrQuery.ORDER; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.SpatialParams; -import org.bson.BSONObject; -import org.bson.types.ObjectId; -import org.slf4j.Logger; -import org.sothis.core.util.BlockingThreadPoolExecutor; - import com.fangjia.datastore.ConfigurationException; -import com.fangjia.datastore.QueryResponse; -import com.fangjia.datastore.TriggerExecuteException; -import com.fangjia.datastore.TriggerExecutor; -import com.fangjia.datastore.config.ServerConfig; -import com.fangjia.datastore.config.StoreConfig; -import com.fangjia.datastore.config.Trigger; +import java.util.concurrent.LinkedBlockingQueue; +import org.sothis.core.util.BlockingThreadPoolExecutor; +import java.util.concurrent.Executor; +import org.bson.BSONObject; +import java.util.concurrent.BlockingQueue; import com.fangjia.datastore.utils.AsyncOperation; -import com.fangjia.datastore.utils.LoggerFactory; -import com.mongodb.BasicDBObject; +import com.fangjia.datastore.config.StoreConfig; +import com.fangjia.datastore.config.ServerConfig; +import org.slf4j.Logger; -public class CoreImpl implements Core { - private final static Logger LOGGER = LoggerFactory.getLogger(CoreImpl.class); - private final static Logger PERFORMANCE_LOGGER = LoggerFactory.getPerformanceLogger(CoreImpl.class); - private final static int TRIIGER_POOL_SIZE = Integer.getInteger("datastore.triggerPoolSize", 10); +public class CoreImpl implements Core +{ + private static final Logger LOGGER; + private static final Logger PERFORMANCE_LOGGER; + private static final int TRIIGER_POOL_SIZE; + private final String name; + private final SolrServer solrServer; + private final ServerConfig serverConfig; + private final StoreConfig storeConfig; + private final AsyncOperation asynchOperation; + private final BlockingQueue solrAddQueue; + private final BlockingQueue triggerQueue; + private final Executor triggerExecutorService; + + public CoreImpl(final StoreConfig storeConfig) throws ConfigurationException { + this.name = storeConfig.getName(); + this.storeConfig = storeConfig; + this.solrServer = new SolrServerImpl(this.storeConfig); + this.serverConfig = storeConfig.getDataStoreConfig().getServerConfig(); + if (CoreImpl.LOGGER.isInfoEnabled()) { + CoreImpl.LOGGER.info("init solr core: " + this.name + ", storeConfig: " + storeConfig); + } + this.asynchOperation = AsyncOperation.getInstance(); + this.triggerExecutorService = new BlockingThreadPoolExecutor(this.name + "-Trigger", 1, CoreImpl.TRIIGER_POOL_SIZE, CoreImpl.TRIIGER_POOL_SIZE, 60000L); + this.solrAddQueue = new LinkedBlockingQueue(60000); + final Thread solrIndexAddThread = new Thread(new SolrAddRunnable(), this.name + "-SolrAdd"); + solrIndexAddThread.setDaemon(true); + solrIndexAddThread.start(); + this.triggerQueue = new LinkedBlockingQueue(this.serverConfig.getTriggerBatchSize() * 4); + final Thread triggerExecuteThread = new Thread(new TriggerRunnable(), this.name + "-Trigger"); + triggerExecuteThread.setDaemon(true); + triggerExecuteThread.start(); + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + while (!CoreImpl.this.solrAddQueue.isEmpty()) { + CoreImpl.LOGGER.info(CoreImpl.this.name + ": wait for solr queue, current size: " + CoreImpl.this.solrAddQueue.size()); + try { + Thread.sleep(1000L); + } + catch (InterruptedException e) { + CoreImpl.LOGGER.error("", (Throwable)e); + } + } + CoreImpl.LOGGER.info(CoreImpl.this.name + ": shutdown."); + } + }); + if (CoreImpl.LOGGER.isInfoEnabled()) { + CoreImpl.LOGGER.info("solr core " + this.name + " initialized"); + } + } + + private void saveToSolr(final Iterator iterator, final String operationName) throws SolrException { + long c = 0L; + final long startTime = System.currentTimeMillis(); + while (iterator.hasNext()) { + try { + this.triggerQueue.put(iterator.next()); + } + catch (InterruptedException e) { + throw new SolrException("error put object: ", e); + } + ++c; + if (null != operationName && c % this.serverConfig.getTriggerBatchSize() == 0L && CoreImpl.LOGGER.isInfoEnabled()) { + CoreImpl.LOGGER.info("{}: {}, {}/s", new Object[] { operationName, c, (int)(c / ((System.currentTimeMillis() - startTime) / 1000.0)) }); + } + } + } + + @Override + public void reload() throws ConfigurationException { + this.solrServer.reload(); + try { + this.solrServer.reloadCores(); + } + catch (SolrException e) { + throw new ConfigurationException(e); + } + } + + @Override + public Collection getIndexes() throws SolrException { + return this.solrServer.getSolrSchema().getIndexes(); + } + + @Override + public void ensureIndex(final Collection indexes, final Iterator iterator) throws SolrException { + try { + final String operationName = "ensureIndex_" + System.currentTimeMillis(); + if (CoreImpl.LOGGER.isInfoEnabled()) { + CoreImpl.LOGGER.info("{}: ensure index[{}] {} ...", new Object[] { this.getName(), operationName, indexes }); + } + if (this.solrServer.getSolrSchema().saveIndex(indexes) > 0) { + this.solrServer.reloadCores(); + } + this.saveToSolr(iterator, operationName); + if (CoreImpl.LOGGER.isInfoEnabled()) { + CoreImpl.LOGGER.info("{}: ensure index[{}] {} finished", new Object[] { this.getName(), operationName, indexes }); + } + } + catch (SolrException e) { + CoreImpl.LOGGER.error("ensureIndex: ", (Throwable)e); + } + } + + @Override + public void ensureIndex(final Index index, final Iterator iterator) throws SolrException { + this.ensureIndex(Arrays.asList(index), iterator); + } + + @Override + public void reIndex(final List objIdList, final Iterator iterator) throws SolrException { + final boolean logInfo = CollectionUtils.isEmpty((Collection)objIdList); + final long startTime = System.currentTimeMillis(); + final String operationName = logInfo ? (this.getName() + "_re-index_" + startTime) : null; + if (logInfo && CoreImpl.LOGGER.isInfoEnabled()) { + CoreImpl.LOGGER.info(operationName + " started."); + } + this.saveToSolr(iterator, operationName); + if (logInfo && CoreImpl.LOGGER.isInfoEnabled()) { + CoreImpl.LOGGER.info("{} finished, total time: {}s.", operationName, (System.currentTimeMillis() - startTime) / 1000L); + } + } + + @Override + public void save(final Iterator iterator) throws SolrException { + this.asynchOperation.run(new Runnable() { + @Override + public void run() { + try { + CoreImpl.this.saveToSolr(iterator, "save_" + System.currentTimeMillis()); + } + catch (SolrException e) { + CoreImpl.LOGGER.error(CoreImpl.this.name + ": error save objects into solr:", (Throwable)e); + } + } + }); + } + + @Override + public void deleteByQuery(final BSONObject query) throws SolrException { + final String solrQuery = this.toSolrQuery(query).get("q"); + if (CoreImpl.LOGGER.isDebugEnabled()) { + CoreImpl.LOGGER.debug("deleteByQuery: {}", (Object)solrQuery); + } + this.solrServer.deleteByQuery(solrQuery); + } + + private SolrQuery toSolrQuery(final BSONObject query) throws SolrException { + return this.toSolrQuery(query, false); + } + + private SolrQuery toSolrQuery(final BSONObject orgQuery, final boolean useFilterQuery) throws SolrException { + System.err.println(orgQuery.toString()); + final BSONObject query = (BSONObject)new BasicDBObject(); + query.putAll(orgQuery); + final SolrQueryBuilder queryBuilder = new SolrQueryBuilder(); + final SolrQuery solrQuery = new SolrQuery(); + if (useFilterQuery && query != null && query.containsField("visible") && query.containsField("city") && query.containsField("cat") && query.get("cat") instanceof String) { + final StringBuilder fq = new StringBuilder(); + fq.append("+cat:").append(query.get("cat")); + fq.append(" +city:").append(query.get("city")); + fq.append(" +visible:").append(((Number)query.get("visible")).intValue()); + solrQuery.addFilterQuery(fq.toString()); + query.removeField("cat"); + query.removeField("city"); + query.removeField("visible"); + } + for (final String key : query.keySet()) { + if (key.charAt(0) == '$') { + continue; + } + final Object value = query.get(key); + final Index index = this.assertFieldExistence(key); + if (value instanceof BSONObject) { + final BSONObject subQuery = (BSONObject)value; + Object g = subQuery.get("$gt"); + boolean ge = false; + if (null == g) { + g = subQuery.get("$gte"); + ge = (g != null); + } + Object l = subQuery.get("$lt"); + boolean le = false; + if (null == l) { + l = subQuery.get("$lte"); + le = (l != null); + } + if (ge && null == l) { + le = true; + } + if (le && null == g) { + ge = true; + } + if (null != g || null != l) { + queryBuilder.range(key, index.normalizeValue(g), index.normalizeValue(l), ge, le); + } + final Object in = subQuery.get("$in"); + if (null != in) { + queryBuilder.in(key, index.normalizeValue((List)in)); + } + final Object ne = subQuery.get("$ne"); + if (null == ne) { + continue; + } + queryBuilder.ne(key, index.normalizeValue(ne)); + } + else { + queryBuilder.eq(key, index.normalizeValue(value)); + } + } + String q = queryBuilder.toQuery().toString(); + if (query.get("$sr") != null) { + final Object value2 = query.get("$sr"); + final BSONObject bsonObject = (BSONObject)value2; + final String val = bsonObject.get("sr.query").toString(); + if (val != null && !"".equals(val)) { + q = q + " +(" + val + ")"; + } + } + if (query.get("$sd") != null) { + final Object value2 = query.get("$sd"); + final BSONObject bsonObject = (BSONObject)value2; + final String distance = (String)bsonObject.get("sd.distance"); + final String field = (String)bsonObject.get("sd.sfield"); + final String point = (String)bsonObject.get("sd.point"); + if (!StringUtils.isNotBlank(point) || !StringUtils.isNotBlank(distance) || !StringUtils.isNotBlank(field)) { + throw new IllegalArgumentException("paramName $sd value illegal : " + value2); + } + solrQuery.setParam("sfield", field); + solrQuery.setParam("d", distance); + solrQuery.setParam("pt", point); + solrQuery.addSort("geodist()", SolrQuery.ORDER.asc); + solrQuery.addFilterQuery("{!geofilt}"); + } + if (query.get("$fangjiawang_query_type") != null) { + final Object value2 = query.get("$fangjiawang_query_type"); + solrQuery.add("q.op", value2.toString()); + } + if (StringUtils.isEmpty(q)) { + solrQuery.setQuery("*:*"); + } + else { + solrQuery.setQuery(q); + } + return solrQuery; + } + + private Index assertFieldExistence(final String key) throws SolrException { + final Index index = this.solrServer.getSolrSchema().getIndex(key); + if (null == index) { + throw new SolrException("index '" + key + "' is not exist!"); + } + return index; + } + + @Override + public QueryResponse find(final BSONObject query, final BSONObject sort, final BSONObject facet, int skip, int limit, final boolean fromMaster, final boolean bulk) throws SolrException { + final long start = System.currentTimeMillis(); + if (limit < 0) { + limit = this.serverConfig.getDefaultCursorLimit(); + } + if (skip < 0) { + skip = 0; + } + final SolrQuery solrQuery = this.toSolrQuery(query, true); + solrQuery.setFields("_id"); + solrQuery.setStart(skip); + solrQuery.setRows(limit); + if (null != sort) { + for (final String key : sort.keySet()) { + final int value = ((Number)sort.get(key)).intValue(); + if (!"score".equals(key)) { + this.assertFieldExistence(key); + } + solrQuery.addSort(key, (value > 0) ? SolrQuery.ORDER.asc : SolrQuery.ORDER.desc); + } + } + if (null != facet) { + final List fields = (List)facet.get("fields"); + if (null != fields) { + solrQuery.setFacet(true); + Integer facetMinCount = null; + if (null != facet.get("minCount")) { + facetMinCount = ((Number)facet.get("minCount")).intValue(); + } + solrQuery.setFacetMinCount((null == facetMinCount) ? 1 : ((int)facetMinCount)); + Integer facetLimit = null; + if (null != facet.get("limit")) { + facetLimit = ((Number)facet.get("limit")).intValue(); + } + solrQuery.setFacetLimit((null == facetLimit) ? 100 : facetLimit); + for (final String field : fields) { + this.assertFieldExistence(field); + solrQuery.addFacetField(field); + } + } + } + final org.apache.solr.client.solrj.response.QueryResponse queryResponse = this.solrServer.query(solrQuery, fromMaster, bulk); + final long time = System.currentTimeMillis() - start; + if (time > 100L) { + String host = solrQuery.get("__host"); + solrQuery.remove("__host"); + final StringBuilder sb = new StringBuilder(); + sb.append("query solr: ").append(solrQuery); + if(StringUtils.isNotBlank(host)){ + sb.append(", host: ").append(host); + } + sb.append(", time: ").append(time); + sb.append(", qtime: ").append(queryResponse.getQTime()); + sb.append(", numFound: ").append(queryResponse.getResults().getNumFound()); + sb.append(", numReturned: ").append(queryResponse.getResults().size()); + sb.append(", master: ").append(fromMaster); + CoreImpl.PERFORMANCE_LOGGER.info(sb.toString()); + if (time < 1000L) { + CoreImpl.PERFORMANCE_LOGGER.info(sb.toString()); + } + else if (time < 3000L) { + CoreImpl.PERFORMANCE_LOGGER.warn(sb.toString()); + } + else { + CoreImpl.PERFORMANCE_LOGGER.error(sb.toString()); + } + } + final QueryResponse response = this.toSolrResponse(queryResponse); + return response; + } + + private QueryResponse toSolrResponse(final org.apache.solr.client.solrj.response.QueryResponse response) { + if (response == null) { + return QueryResponse.EMPTY_RESPONSE; + } + final SolrDocumentList solrDocumentList = response.getResults(); + final int numReturned = solrDocumentList.size(); + final List idList = new ArrayList(numReturned); + for (int i = 0; i < numReturned; ++i) { + final String value = ((SolrDocument)solrDocumentList.get(i)).getFirstValue("_id").toString(); + idList.add(new ObjectId(value)); + } + final QueryResponse solrResponse = new QueryResponse(); + solrResponse.setNumFound((int)solrDocumentList.getNumFound()); + solrResponse.setNumReturned(numReturned); + solrResponse.setNumToSkip((int)solrDocumentList.getStart()); + solrResponse.setIdList(idList); + solrResponse.setFacetFieldList(response.getFacetFields()); + return solrResponse; + } + + @Override + public String getName() { + return this.name; + } + + private List executeTrigger(final List list) throws TriggerExecuteException { + final Trigger trigger = this.storeConfig.getTrigger(); + if (null == trigger) { + if (CoreImpl.LOGGER.isWarnEnabled()) { + CoreImpl.LOGGER.warn("null trigger!"); + } + return list; + } + if (null == trigger.getInterface()) { + return list; + } + return TriggerExecutor.getInstance().execute(trigger, list); + } + + static { + LOGGER = LoggerFactory.getLogger(CoreImpl.class); + PERFORMANCE_LOGGER = LoggerFactory.getPerformanceLogger(CoreImpl.class); + TRIIGER_POOL_SIZE = Integer.getInteger("datastore.triggerPoolSize", 10); + } - private final String name; - private final SolrServer solrServer; - private final ServerConfig serverConfig; - private final StoreConfig storeConfig; - private final AsyncOperation asynchOperation; - private final BlockingQueue solrAddQueue; - private final BlockingQueue triggerQueue; - private final Executor triggerExecutorService; + private class TriggerRunnable implements Runnable { - public CoreImpl(StoreConfig storeConfig) throws ConfigurationException { - this.name = storeConfig.getName(); - this.storeConfig = storeConfig; - this.solrServer = new SolrServerImpl(this.storeConfig); - this.serverConfig = storeConfig.getDataStoreConfig().getServerConfig(); - if (LOGGER.isInfoEnabled()) { - LOGGER.info("init solr core: " + name + ", storeConfig: " + storeConfig); - } + public void run() { + int sleepTime = 0; + while (true) { + try { + final List dbObjects = new ArrayList(serverConfig.getTriggerBatchSize()); + triggerQueue.drainTo(dbObjects, serverConfig.getTriggerBatchSize()); + if (!dbObjects.isEmpty()) { + triggerExecutorService.execute(new Runnable() { - this.asynchOperation = AsyncOperation.getInstance(); + public void run() { + try { + long time = System.currentTimeMillis(); + List list = executeTrigger(dbObjects); + time = System.currentTimeMillis() - time; + if (LOGGER.isInfoEnabled() && time > 0) { + LOGGER.info("execute trigger: {}/{} objects, {}ms, {}/s", new Object[] { list.size(), + dbObjects.size(), time, dbObjects.size() * 1000 / time }); + } + for (BSONObject obj : list) { + solrAddQueue.put(obj); + } + } catch (Exception e) { + LOGGER.error("error execute trigger: ", e); + } + } - // trigger executor - this.triggerExecutorService = new BlockingThreadPoolExecutor(name + "-Trigger", 1, TRIIGER_POOL_SIZE, TRIIGER_POOL_SIZE, - 60000); + }); + sleepTime = serverConfig.getTriggerBatchSize() - dbObjects.size(); + if (sleepTime > 100) { + Thread.sleep(sleepTime * 3); + } + } else { + Thread.sleep(serverConfig.getTriggerBatchSize()); + } + } catch (Exception e) { + LOGGER.error("error execute trigger: ", e); + } + } + } - // solr add thread - this.solrAddQueue = new LinkedBlockingQueue(INDEX_BATCH_SIZE * 2); - Thread solrIndexAddThread = new Thread(new SolrAddRunnable(), name + "-SolrAdd"); - solrIndexAddThread.setDaemon(true); - solrIndexAddThread.start(); - - // trigger thread - this.triggerQueue = new LinkedBlockingQueue(this.serverConfig.getTriggerBatchSize() * 4); - Thread triggerExecuteThread = new Thread(new TriggerRunnable(), name + "-Trigger"); - triggerExecuteThread.setDaemon(true); - triggerExecuteThread.start(); - - // shutdown hook - Runtime.getRuntime().addShutdownHook(new Thread() { - - @Override - public void run() { - while (!solrAddQueue.isEmpty()) { - LOGGER.info(CoreImpl.this.name + ": wait for solr queue, current size: " + solrAddQueue.size()); - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - LOGGER.error("", e); - } - } - LOGGER.info(CoreImpl.this.name + ": shutdown."); - } - - }); - if (LOGGER.isInfoEnabled()) { - LOGGER.info("solr core " + name + " initialized"); - } - } - - private class TriggerRunnable implements Runnable { - - public void run() { - int sleepTime = 0; - while (true) { - try { - final List dbObjects = new ArrayList(serverConfig.getTriggerBatchSize()); - triggerQueue.drainTo(dbObjects, serverConfig.getTriggerBatchSize()); - if (!dbObjects.isEmpty()) { - triggerExecutorService.execute(new Runnable() { - - public void run() { - try { - long time = System.currentTimeMillis(); - List list = executeTrigger(dbObjects); - time = System.currentTimeMillis() - time; - if (LOGGER.isInfoEnabled() && time > 0) { - LOGGER.info("execute trigger: {}/{} objects, {}ms, {}/s", new Object[] { list.size(), - dbObjects.size(), time, dbObjects.size() * 1000 / time }); - } - for (BSONObject obj : list) { - solrAddQueue.put(obj); - } - } catch (Exception e) { - LOGGER.error("error execute trigger: ", e); - } - } - - }); - sleepTime = serverConfig.getTriggerBatchSize() - dbObjects.size(); - if (sleepTime > 100) { - Thread.sleep(sleepTime * 3); - } - } else { - Thread.sleep(serverConfig.getTriggerBatchSize()); - } - } catch (Exception e) { - LOGGER.error("error execute trigger: ", e); - } - } - } - - } - - private class SolrAddRunnable implements Runnable { - - public void run() { - final List dbObjects = new ArrayList(INDEX_BATCH_SIZE); - final int maxSleepTime = 10000; - int sleepTime = 0; - while (true) { - try { - solrAddQueue.drainTo(dbObjects, INDEX_BATCH_SIZE); - if (!dbObjects.isEmpty()) { - solrServer.add(dbObjects); - sleepTime = maxSleepTime - dbObjects.size(); - if (sleepTime > 100) { - Thread.sleep(sleepTime); - } - } else { - Thread.sleep(maxSleepTime); - } - } catch (Exception e) { - LOGGER.error("error save to solr: ", e); - } finally { - dbObjects.clear(); - } - } - } - } - - private void saveToSolr(final Iterator iterator, String operationName) throws SolrException { - long c = 0; - long startTime = System.currentTimeMillis(); - while (iterator.hasNext()) { - try { - triggerQueue.put(iterator.next()); - } catch (InterruptedException e) { - throw new SolrException("error put object: ", e); - } - c++; - if (null != operationName && c % serverConfig.getTriggerBatchSize() == 0 && LOGGER.isInfoEnabled()) { - LOGGER.info("{}: {}, {}/s", new Object[] { operationName, c, - (int) (c / ((System.currentTimeMillis() - startTime) / 1000.0)) }); - } - } - } - - public void reload() throws ConfigurationException { - this.solrServer.reload(); - try { - this.solrServer.reloadCores(); - } catch (SolrException e) { - throw new ConfigurationException(e); - } - } - - public Collection getIndexes() throws SolrException { - return this.solrServer.getSolrSchema().getIndexes(); - } - - public void ensureIndex(final Collection indexes, final Iterator iterator) throws SolrException { - try { - String operationName = "ensureIndex_" + System.currentTimeMillis(); - if (LOGGER.isInfoEnabled()) { - LOGGER.info("{}: ensure index[{}] {} ...", new Object[] { CoreImpl.this.getName(), operationName, indexes }); - } - if (solrServer.getSolrSchema().saveIndex(indexes) > 0) { - this.solrServer.reloadCores(); - } - saveToSolr(iterator, operationName); - if (LOGGER.isInfoEnabled()) { - LOGGER.info("{}: ensure index[{}] {} finished", new Object[] { CoreImpl.this.getName(), operationName, indexes }); - } - } catch (SolrException e) { - LOGGER.error("ensureIndex: ", e); - } - } - - public void ensureIndex(Index index, Iterator iterator) throws SolrException { - ensureIndex(Arrays.asList(index), iterator); - } - - public void reIndex(final List objIdList, final Iterator iterator) throws SolrException { - boolean logInfo = CollectionUtils.isEmpty(objIdList); - long startTime = System.currentTimeMillis(); - String operationName = logInfo ? CoreImpl.this.getName() + "_re-index_" + startTime : null; - if (logInfo && LOGGER.isInfoEnabled()) { - LOGGER.info(operationName + " started."); - } - saveToSolr(iterator, operationName); - if (logInfo && LOGGER.isInfoEnabled()) { - LOGGER.info("{} finished, total time: {}s.", operationName, (System.currentTimeMillis() - startTime) / 1000); - } - } - - public void save(final Iterator iterator) throws SolrException { - asynchOperation.run(new Runnable() { - public void run() { - try { - saveToSolr(iterator, "save_" + System.currentTimeMillis()); - } catch (SolrException e) { - LOGGER.error(name + ": error save objects into solr:", e); - } - } - }); - } - - public void deleteByQuery(final BSONObject query) throws SolrException { - String solrQuery = toSolrQuery(query).get(CommonParams.Q); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("deleteByQuery: {}", solrQuery); - } - this.solrServer.deleteByQuery(solrQuery); - } - - private SolrQuery toSolrQuery(BSONObject query) throws SolrException { - return toSolrQuery(query, false); - } - - private SolrQuery toSolrQuery(final BSONObject orgQuery, boolean useFilterQuery) throws SolrException { - System.err.println(orgQuery.toString()); - BSONObject query = new BasicDBObject(); - query.putAll(orgQuery); - SolrQueryBuilder queryBuilder = new SolrQueryBuilder(); - - SolrQuery solrQuery = new SolrQuery(); - if (useFilterQuery && query != null && query.containsField("visible") && query.containsField("city") - && query.containsField("cat") && query.get("cat") instanceof String) { - StringBuilder fq = new StringBuilder(); - fq.append("+cat:").append(query.get("cat")); - fq.append(" +city:").append(query.get("city")); - fq.append(" +visible:").append(((Number) query.get("visible")).intValue()); - solrQuery.addFilterQuery(fq.toString()); - query.removeField("cat"); - query.removeField("city"); - query.removeField("visible"); - } - - if (null != query) { - for (String key : query.keySet()) { - if (key.charAt(0) == '$') { - continue; - } - Object value = query.get(key); - Index index = assertFieldExistence(key); - if (value instanceof BSONObject) { - BSONObject subQuery = (BSONObject) value; - Object g = subQuery.get("$gt"); - boolean ge = false; - if (null == g) { - g = subQuery.get("$gte"); - ge = g != null; - } - Object l = subQuery.get("$lt"); - boolean le = false; - if (null == l) { - l = subQuery.get("$lte"); - le = l != null; - } - if (ge && null == l) { - le = true; - } - if (le && null == g) { - ge = true; - } - if (null != g || null != l) { - queryBuilder.range(key, index.normalizeValue(g), index.normalizeValue(l), ge, le); - } - Object in = subQuery.get("$in"); - if (null != in) { - queryBuilder.in(key, index.normalizeValue((List) in)); - } - Object ne = subQuery.get("$ne"); - if (null != ne) { - queryBuilder.ne(key, index.normalizeValue(ne)); - } - - } else { - queryBuilder.eq(key, index.normalizeValue(value)); - } - } - } - String q = queryBuilder.toQuery().toString(); - - // $sr模式 - if (null != query) { - if (query.get("$sr") != null) { - Object value = query.get("$sr"); - BSONObject bsonObject = (BSONObject) value; - String val = bsonObject.get("sr.query").toString(); - if (val != null && !"".equals(val)) { - q += " +(" + val + ")"; - } - } - if (query.get("$sd") != null) { - Object value = query.get("$sd"); - BSONObject bsonObject = (BSONObject) value; - String distance = (String) bsonObject.get("sd.distance"); - String field = (String) bsonObject.get("sd.sfield"); - String point = (String) bsonObject.get("sd.point"); - if (StringUtils.isNotBlank(point) && StringUtils.isNotBlank(distance) && StringUtils.isNotBlank(field)) { - solrQuery.setParam(SpatialParams.FIELD, field); - solrQuery.setParam(SpatialParams.DISTANCE, distance); - solrQuery.setParam(SpatialParams.POINT, point); - solrQuery.addSort("geodist()", ORDER.asc); - solrQuery.addFilterQuery("{!geofilt}"); - } else { - throw new IllegalArgumentException("paramName $sd value illegal : " + value); - } - } - - if (query.get("$fangjiawang_query_type") != null) { - Object value = query.get("$fangjiawang_query_type"); - solrQuery.add("q.op", value.toString()); - } - } - - - - if (StringUtils.isEmpty(q)) { - solrQuery.setQuery("*:*"); - } else { - solrQuery.setQuery(q); - } - return solrQuery; - } - - private Index assertFieldExistence(String key) throws SolrException { - Index index = this.solrServer.getSolrSchema().getIndex(key); - if (null == index) { - throw new SolrException("index '" + key + "' is not exist!"); - } - return index; - } - - public QueryResponse find(BSONObject query, BSONObject sort, BSONObject facet, int skip, int limit, boolean fromMaster, - boolean bulk) throws SolrException { - long start = System.currentTimeMillis(); - if (limit < 0) { - limit = serverConfig.getDefaultCursorLimit(); - } - if (skip < 0) { - skip = 0; - } - SolrQuery solrQuery = toSolrQuery(query, true); - - // fields, start, limit - solrQuery.setFields(SolrSchema.ID); - solrQuery.setStart(skip); - solrQuery.setRows(limit); - - // sort - if (null != sort) { - for (String key : sort.keySet()) { - int value = ((Number) sort.get(key)).intValue(); - if (!"score".equals(key)) { // pseudo-field score - assertFieldExistence(key); - } - solrQuery.addSort(key, value > 0 ? ORDER.asc : ORDER.desc); - } - } - - // facet - if (null != facet) { - List fields = (List) facet.get("fields"); - if (null != fields) { - solrQuery.setFacet(true); - Integer facetMinCount = null; - if (null != facet.get("minCount")) { - facetMinCount = ((Number) facet.get("minCount")).intValue(); - } - solrQuery.setFacetMinCount(null == facetMinCount ? 1 : facetMinCount); - Integer facetLimit = null; - if (null != facet.get("limit")) { - facetLimit = ((Number) facet.get("limit")).intValue(); - } - solrQuery.setFacetLimit(null == facetLimit ? 100 : facetLimit); - for (String field : fields) { - assertFieldExistence(field); - solrQuery.addFacetField(field); - } - } - } - - org.apache.solr.client.solrj.response.QueryResponse queryResponse = this.solrServer.query(solrQuery, fromMaster, bulk); - long time = System.currentTimeMillis() - start; - if (time > 100) { - StringBuilder sb = new StringBuilder(); - sb.append("query solr: ").append(solrQuery); - sb.append(", time: ").append(time); - sb.append(", qtime: ").append(queryResponse.getQTime()); - sb.append(", numFound: ").append(queryResponse.getResults().getNumFound()); - sb.append(", numReturned: ").append(queryResponse.getResults().size()); - sb.append(", master: ").append(fromMaster); - if (time < 1000) { - PERFORMANCE_LOGGER.info(sb.toString()); - } else if (time < 3000) { - PERFORMANCE_LOGGER.warn(sb.toString()); - } else { - PERFORMANCE_LOGGER.error(sb.toString()); - } - } - QueryResponse response = toSolrResponse(queryResponse); - return response; - } - - private QueryResponse toSolrResponse(org.apache.solr.client.solrj.response.QueryResponse response) { - if (response == null) { - return QueryResponse.EMPTY_RESPONSE; - } - - SolrDocumentList solrDocumentList = response.getResults(); - int numReturned = solrDocumentList.size(); - List idList = new ArrayList(numReturned); - for (int i = 0; i < numReturned; i++) { - String value = solrDocumentList.get(i).getFirstValue(SolrSchema.ID).toString(); - idList.add(new ObjectId(value)); - } - - QueryResponse solrResponse = new QueryResponse(); - solrResponse.setNumFound((int) solrDocumentList.getNumFound()); - solrResponse.setNumReturned(numReturned); - solrResponse.setNumToSkip((int) solrDocumentList.getStart()); - solrResponse.setIdList(idList); - solrResponse.setFacetFieldList(response.getFacetFields()); - return solrResponse; - } - - public String getName() { - return name; - } - - private List executeTrigger(List list) throws TriggerExecuteException { - Trigger trigger = this.storeConfig.getTrigger(); - if (null == trigger) { - if (LOGGER.isWarnEnabled()) { - LOGGER.warn("null trigger!"); - } - return list; - } - - if (null == trigger.getInterface()) { - return list; - } - return TriggerExecutor.getInstance().execute(trigger, list); - } + } + + private class SolrAddRunnable implements Runnable + { + @Override + public void run() { + final List dbObjects = new ArrayList(30000); + final int maxSleepTime = 10000; + int sleepTime = 0; + while (true) { + try { + CoreImpl.this.solrAddQueue.drainTo(dbObjects, 30000); + if (!dbObjects.isEmpty()) { + CoreImpl.this.solrServer.add(dbObjects); + sleepTime = 10000 - dbObjects.size(); + if (sleepTime <= 100) { + continue; + } + Thread.sleep(sleepTime); + } + else { + Thread.sleep(10000L); + } + } + catch (Exception e) { + CoreImpl.LOGGER.error("error save to solr: ", (Throwable)e); + } + finally { + dbObjects.clear(); + } + } + } + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/HttpSolrServer.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/HttpSolrServer.java index 12a416398..388c29f65 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/HttpSolrServer.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/HttpSolrServer.java @@ -2,13 +2,13 @@ package com.fangjia.datastore.solr; import org.apache.solr.client.solrj.SolrServer; -public abstract class HttpSolrServer extends SolrServer { - - private static final long serialVersionUID = -3967599614339379847L; - - abstract public void reloadCore() throws SolrException; - - abstract public String getCoreName(); - - abstract public String getHost(); +public abstract class HttpSolrServer extends SolrServer +{ + private static final long serialVersionUID = -3967599614339379847L; + + public abstract void reloadCore() throws SolrException; + + public abstract String getCoreName(); + + public abstract String getHost(); } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/Index.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/Index.java index 0ece19d97..a8a7c1415 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/Index.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/Index.java @@ -1,191 +1,196 @@ package com.fangjia.datastore.solr; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; -import org.bson.BSONObject; import org.bson.BasicBSONObject; +import org.bson.BSONObject; +import java.util.Date; +import java.util.Iterator; +import java.util.ArrayList; +import java.util.List; -public class Index { - public static enum IndexTypeEnum { - STRING, INTEGER, INT, DOUBLE, TEXT_STD, TEXT_CJK, DATE, LONG, LOCATION; - public String getType() { - return this.name().toLowerCase(); - } - - public static IndexTypeEnum parse(String type) { - for (IndexTypeEnum e : IndexTypeEnum.values()) { - if (e.getType().equals(type)) { - return e; - } - } - return null; - } - } - - private String name; - private IndexTypeEnum type; - private boolean stored; - private boolean indexed; - private boolean multiValued; - private String comments; - private boolean dynamic; - - public Index(String name, IndexTypeEnum type, boolean stored, boolean indexed, boolean multiValued, boolean dynamic, - String comments) { - this.name = name; - this.type = type; - this.stored = stored; - this.indexed = indexed; - this.multiValued = multiValued; - this.dynamic = dynamic; - this.comments = comments; - } - - public Index(String name, IndexTypeEnum type) { - this.name = name; - this.type = type; - this.stored = false; - this.indexed = true; - this.multiValued = false; - this.dynamic = false; - this.comments = ""; - } - - public List normalizeValue(List values) { - if (null == values) { - return null; - } - List ret = new ArrayList(values.size()); - for (Object v : values) { - ret.add(normalizeValue(v)); - } - return ret; - } - - public Object normalizeValue(Object value) { - if (null == value) { - return null; - } - switch (this.type) { - case STRING: - case TEXT_STD: - return value.toString(); - case INTEGER: - return ((Number) value).intValue(); - case LONG: - return ((Number) value).longValue(); - case DOUBLE: - return ((Number) value).doubleValue(); - case DATE: - return (Date) value; - } - return value; - } - - public Index() { - this(null, null); - } - - public String getName() { - return name; - } - - public IndexTypeEnum getType() { - return type; - } - - public boolean isStored() { - return stored; - } - - public boolean isIndexed() { - return indexed; - } - - public boolean isMultiValued() { - return multiValued; - } - - public String getComments() { - return comments; - } - - public boolean isDynamic() { - return dynamic; - } - - public void setName(String name) { - this.name = name; - } - - public void setType(IndexTypeEnum type) { - this.type = type; - } - - public void setStored(boolean stored) { - this.stored = stored; - } - - public void setIndexed(boolean indexed) { - this.indexed = indexed; - } - - public void setMultiValued(boolean multiValued) { - this.multiValued = multiValued; - } - - public void setComments(String comments) { - this.comments = comments; - } - - public void setDynamic(boolean dynamic) { - this.dynamic = dynamic; - } - - public BSONObject toBSONObject() { - BSONObject ret = new BasicBSONObject(); - ret.put("name", name); - ret.put("type", this.type.getType()); - ret.put("stored", this.stored); - ret.put("indexed", this.indexed); - ret.put("multiValued", this.multiValued); - ret.put("dynamic", this.dynamic); - ret.put("comments", this.comments); - return ret; - } - - @Override - public int hashCode() { - return new HashCodeBuilder().append(this.dynamic).append(this.indexed).append(this.multiValued).append(this.stored) - .append(this.comments).append(this.name).append(this.type).toHashCode(); - } - - @Override - public boolean equals(Object obj) { - if (null == obj) { - return false; - } - if (this == obj) { - return true; - } - if (!(obj instanceof Index)) { - return false; - } - Index other = (Index) obj; - return new EqualsBuilder().append(this.dynamic, other.dynamic).append(this.indexed, other.indexed).append( - this.multiValued, other.multiValued).append(this.stored, other.stored).append(this.comments, other.comments) - .append(this.name, other.name).append(this.type, other.type).isEquals(); - } - - @Override - public String toString() { - return new StringBuilder().append("{name=").append(name).append(", type=").append(type).append(", stored=") - .append(stored).append(", indexed=").append(indexed).append(", multiValued=").append(multiValued).append( - ", dynamic=").append(dynamic).append(", comments=").append(this.comments).append("}").toString(); - } - +public class Index +{ + private String name; + private IndexTypeEnum type; + private boolean stored; + private boolean indexed; + private boolean multiValued; + private String comments; + private boolean dynamic; + + public Index(final String name, final IndexTypeEnum type, final boolean stored, final boolean indexed, final boolean multiValued, final boolean dynamic, final String comments) { + this.name = name; + this.type = type; + this.stored = stored; + this.indexed = indexed; + this.multiValued = multiValued; + this.dynamic = dynamic; + this.comments = comments; + } + + public Index(final String name, final IndexTypeEnum type) { + this.name = name; + this.type = type; + this.stored = false; + this.indexed = true; + this.multiValued = false; + this.dynamic = false; + this.comments = ""; + } + + public List normalizeValue(final List values) { + if (null == values) { + return null; + } + final List ret = new ArrayList(values.size()); + for (final Object v : values) { + ret.add(this.normalizeValue(v)); + } + return ret; + } + + public Object normalizeValue(final Object value) { + if (null == value) { + return null; + } + switch (this.type) { + case STRING: + case TEXT_STD: + return value.toString(); + case INTEGER: + return ((Number)value).intValue(); + case LONG: + return ((Number)value).longValue(); + case DOUBLE: + return ((Number)value).doubleValue(); + case DATE: + return value; + default: + return value; + } + } + + public Index() { + this(null, null); + } + + public String getName() { + return this.name; + } + + public IndexTypeEnum getType() { + return this.type; + } + + public boolean isStored() { + return this.stored; + } + + public boolean isIndexed() { + return this.indexed; + } + + public boolean isMultiValued() { + return this.multiValued; + } + + public String getComments() { + return this.comments; + } + + public boolean isDynamic() { + return this.dynamic; + } + + public void setName(final String name) { + this.name = name; + } + + public void setType(final IndexTypeEnum type) { + this.type = type; + } + + public void setStored(final boolean stored) { + this.stored = stored; + } + + public void setIndexed(final boolean indexed) { + this.indexed = indexed; + } + + public void setMultiValued(final boolean multiValued) { + this.multiValued = multiValued; + } + + public void setComments(final String comments) { + this.comments = comments; + } + + public void setDynamic(final boolean dynamic) { + this.dynamic = dynamic; + } + + public BSONObject toBSONObject() { + final BSONObject ret = (BSONObject)new BasicBSONObject(); + ret.put("name", (Object)this.name); + ret.put("type", (Object)this.type.getType()); + ret.put("stored", (Object)this.stored); + ret.put("indexed", (Object)this.indexed); + ret.put("multiValued", (Object)this.multiValued); + ret.put("dynamic", (Object)this.dynamic); + ret.put("comments", (Object)this.comments); + return ret; + } + + @Override + public int hashCode() { + return new HashCodeBuilder().append(this.dynamic).append(this.indexed).append(this.multiValued).append(this.stored).append((Object)this.comments).append((Object)this.name).append((Object)this.type).toHashCode(); + } + + @Override + public boolean equals(final Object obj) { + if (null == obj) { + return false; + } + if (this == obj) { + return true; + } + if (!(obj instanceof Index)) { + return false; + } + final Index other = (Index)obj; + return new EqualsBuilder().append(this.dynamic, other.dynamic).append(this.indexed, other.indexed).append(this.multiValued, other.multiValued).append(this.stored, other.stored).append((Object)this.comments, (Object)other.comments).append((Object)this.name, (Object)other.name).append((Object)this.type, (Object)other.type).isEquals(); + } + + @Override + public String toString() { + return "{name=" + this.name + ", type=" + this.type + ", stored=" + this.stored + ", indexed=" + this.indexed + ", multiValued=" + this.multiValued + ", dynamic=" + this.dynamic + ", comments=" + this.comments + "}"; + } + + public enum IndexTypeEnum + { + STRING, + INTEGER, + INT, + DOUBLE, + TEXT_STD, + TEXT_CJK, + DATE, + LONG, + LOCATION; + + public String getType() { + return this.name().toLowerCase(); + } + + public static IndexTypeEnum parse(final String type) { + for (final IndexTypeEnum e : values()) { + if (e.getType().equals(type)) { + return e; + } + } + return null; + } + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/LBHttpSolrServer.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/LBHttpSolrServer.java index bbfaae8af..f54925e00 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/LBHttpSolrServer.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/LBHttpSolrServer.java @@ -1,262 +1,223 @@ package com.fangjia.datastore.solr; -import java.io.IOException; -import java.lang.ref.WeakReference; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; - -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.math.RandomUtils; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrRequest; +import com.fangjia.datastore.utils.LoggerFactory; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.params.ShardParams; +import java.io.IOException; +import org.apache.commons.lang.StringUtils; import org.apache.solr.common.util.NamedList; +import org.apache.solr.client.solrj.SolrRequest; +import java.util.concurrent.TimeUnit; +import java.lang.ref.WeakReference; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.Executors; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.params.SolrParams; +import java.util.Iterator; +import org.apache.commons.lang.math.RandomUtils; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.locks.ReentrantLock; +import org.apache.solr.client.solrj.SolrQuery; import org.slf4j.Logger; -import com.fangjia.datastore.utils.LoggerFactory; - -/** - * - * @author velna - * @see org.apache.solr.client.solrj.impl.LBHttpSolrServer - */ -public class LBHttpSolrServer extends HttpSolrServer { - private static final long serialVersionUID = -6175098554678462150L; - private final static Logger LOGGER = LoggerFactory.getLogger(LBHttpSolrServer.class); - - private static final SolrQuery SOLR_QUERY = new SolrQuery("*:*"); - private static final int CHECK_INTERVAL = 10 * 1000; - - private final String coreName; - - private final ReentrantLock checkLock = new ReentrantLock(); - private ScheduledExecutorService aliveCheckExecutor; - private final CopyOnWriteArrayList aliveServers = new CopyOnWriteArrayList(); - private final CopyOnWriteArrayList zombieServers = new CopyOnWriteArrayList(); - - public LBHttpSolrServer(HttpSolrServer... httpSolrServers) { - if (null == httpSolrServers || httpSolrServers.length == 0) { - throw new IllegalArgumentException("length of httpSolrServers must greater than 0"); - } - for (HttpSolrServer httpSolrServer : httpSolrServers) { - this.aliveServers.add(new WrappedHttpSolrServer(httpSolrServer)); - } - coreName = httpSolrServers[0].getCoreName(); - } - - private WrappedHttpSolrServer selectRandomServer() { - for (;;) { - if (aliveServers.isEmpty()) { - throw new RuntimeException("no solr server available, all are down!"); - } - try { - if (aliveServers.size() == 1) { - return aliveServers.get(0); - } - return aliveServers.get(RandomUtils.nextInt(aliveServers.size())); - } catch (IndexOutOfBoundsException e) { - continue; - } - } - } - - /*@Override - public NamedList request(final SolrRequest request) throws SolrServerException, IOException { - int attempts = 0; - int startSize = aliveServers.size(); - Exception ex; - String zombieHost = null; - for (;;) { - WrappedHttpSolrServer httpSolrServer = selectRandomServer(); - try { - if (null != zombieHost && request.getParams() instanceof SolrQuery) { - SolrQuery query = (SolrQuery) request.getParams(); - String shards = query.get(ShardParams.SHARDS); - if (null != shards) { - shards = StringUtils.replace(shards, zombieHost, httpSolrServer.httpSolrServer.getHost()); - query.set(ShardParams.SHARDS, shards); - } - } - return httpSolrServer.httpSolrServer.request(request); - } catch (SolrServerException e) { - if (e.getRootCause() instanceof IOException) { - ex = e; - zombieHost = httpSolrServer.httpSolrServer.getHost(); - LOGGER.error("error request solr server: " + zombieHost, e); - moveAliveToDead(httpSolrServer); - } else { - throw e; - } - } - attempts++; - if (attempts >= startSize) - throw new SolrServerException("No live SolrServers available to handle this request", ex); - } - }*/ - - @Override - public String getHost() { - for (WrappedHttpSolrServer server : aliveServers) { - LOGGER.info("---"+server.httpSolrServer.getHost()); - } - return selectRandomServer().httpSolrServer.getHost(); - } - - @Override - public String getCoreName() { - return coreName; - } - - @Override - public void reloadCore() throws com.fangjia.datastore.solr.SolrException { - // does noting - } - - private void moveAliveToDead(WrappedHttpSolrServer solrServer) { - checkLock.lock(); - try { - boolean result = aliveServers.remove(solrServer); - if (result) { - if (zombieServers.addIfAbsent(solrServer)) { - if (LOGGER.isWarnEnabled()) { - LOGGER.warn("solr server [" + solrServer.httpSolrServer.getHost() + "] is marked as down."); - } - startAliveCheckExecutor(); - } - } - } finally { - checkLock.unlock(); - } - } - - private void checkAZombieServer(WrappedHttpSolrServer zombieServer) { - checkLock.lock(); - try { - if (LOGGER.isInfoEnabled()) { - LOGGER.info("check solr server [" + zombieServer.httpSolrServer.getHost() + "] ..."); - } - QueryResponse resp = zombieServer.httpSolrServer.query(SOLR_QUERY); - if (resp.getStatus() == 0) { - // server has come back up - zombieServers.remove(zombieServer); - aliveServers.add(zombieServer); - if (LOGGER.isInfoEnabled()) { - LOGGER.info("solr server [" + zombieServer.httpSolrServer.getHost() + "] has come back."); - } - zombieServer.failedPings = 0; - } - } catch (Exception e) { - zombieServer.failedPings++; - // Expected . The server is still down - } finally { - checkLock.unlock(); - } - } - - private void startAliveCheckExecutor() { - if (aliveCheckExecutor == null) { - synchronized (this) { - if (aliveCheckExecutor == null) { - aliveCheckExecutor = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory()); - aliveCheckExecutor.scheduleAtFixedRate(getAliveCheckRunner(new WeakReference(this)), - CHECK_INTERVAL, CHECK_INTERVAL, TimeUnit.MILLISECONDS); - } - } - } - } - - private static Runnable getAliveCheckRunner(final WeakReference lbHttpSolrServer) { - return new Runnable() { - public void run() { - LBHttpSolrServer solrServer = lbHttpSolrServer.get(); - if (solrServer != null && solrServer.zombieServers != null) { - for (WrappedHttpSolrServer zombieServer : solrServer.zombieServers) { - solrServer.checkAZombieServer(zombieServer); - } - } - } - }; - } - - /** - * The default thread factory - */ - static class DefaultThreadFactory implements ThreadFactory { - static final AtomicInteger poolNumber = new AtomicInteger(1); - final ThreadGroup group; - final AtomicInteger threadNumber = new AtomicInteger(1); - final String namePrefix; - - DefaultThreadFactory() { - SecurityManager s = System.getSecurityManager(); - group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup(); - namePrefix = "solrcheck-" + poolNumber.getAndIncrement() + "-thread-"; - } - - public Thread newThread(Runnable r) { - Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0); - t.setDaemon(true); - if (t.getPriority() != Thread.NORM_PRIORITY) - t.setPriority(Thread.NORM_PRIORITY); - return t; - } - } - - private class WrappedHttpSolrServer { - private final HttpSolrServer httpSolrServer; - private int failedPings = 0; - - public WrappedHttpSolrServer(HttpSolrServer httpSolrServer) { - this.httpSolrServer = httpSolrServer; - } - - } - - @Override - public NamedList request(SolrRequest paramSolrRequest, - String paramString) throws SolrServerException, IOException { - int attempts = 0; - int startSize = aliveServers.size(); - Exception ex; - String zombieHost = null; - for (;;) { - WrappedHttpSolrServer httpSolrServer = selectRandomServer(); - try { - if (null != zombieHost && paramSolrRequest.getParams() instanceof SolrQuery) { - SolrQuery query = (SolrQuery) paramSolrRequest.getParams(); - String shards = query.get(ShardParams.SHARDS); - if (null != shards) { - shards = StringUtils.replace(shards, zombieHost, httpSolrServer.httpSolrServer.getHost()); - query.set(ShardParams.SHARDS, shards); - } - } - return httpSolrServer.httpSolrServer.request(paramSolrRequest); - } catch (SolrServerException e) { - if (e.getRootCause() instanceof IOException) { - ex = e; - zombieHost = httpSolrServer.httpSolrServer.getHost(); - LOGGER.error("error request solr server: " + zombieHost, e); - moveAliveToDead(httpSolrServer); - } else { - throw e; - } - } - attempts++; - if (attempts >= startSize) - throw new SolrServerException("No live SolrServers available to handle this request", ex); - } - } - - @Override - public void shutdown() { - // TODO Auto-generated method stub - - } -} \ No newline at end of file +public class LBHttpSolrServer extends HttpSolrServer +{ + private static final long serialVersionUID = -6175098554678462150L; + private static final Logger LOGGER; + private static final SolrQuery SOLR_QUERY; + private static final int CHECK_INTERVAL = 10000; + private final String coreName; + private final ReentrantLock checkLock; + private ScheduledExecutorService aliveCheckExecutor; + private final CopyOnWriteArrayList aliveServers; + private final CopyOnWriteArrayList zombieServers; + + public LBHttpSolrServer(final HttpSolrServer... httpSolrServers) { + this.checkLock = new ReentrantLock(); + this.aliveServers = new CopyOnWriteArrayList(); + this.zombieServers = new CopyOnWriteArrayList(); + if (null == httpSolrServers || httpSolrServers.length == 0) { + throw new IllegalArgumentException("length of httpSolrServers must greater than 0"); + } + for (final HttpSolrServer httpSolrServer : httpSolrServers) { + this.aliveServers.add(new WrappedHttpSolrServer(httpSolrServer)); + } + this.coreName = httpSolrServers[0].getCoreName(); + } + + private WrappedHttpSolrServer selectRandomServer() { + while (!this.aliveServers.isEmpty()) { + try { + if (this.aliveServers.size() == 1) { + return this.aliveServers.get(0); + } + return this.aliveServers.get(RandomUtils.nextInt(this.aliveServers.size())); + } + catch (IndexOutOfBoundsException e) {} + } + throw new RuntimeException("no solr server available, all are down!"); + } + + @Override + public String getHost() { + for (final WrappedHttpSolrServer server : this.aliveServers) { + LBHttpSolrServer.LOGGER.info("---" + server.httpSolrServer.getHost()); + } + return this.selectRandomServer().httpSolrServer.getHost(); + } + + @Override + public String getCoreName() { + return this.coreName; + } + + @Override + public void reloadCore() throws SolrException { + } + + private void moveAliveToDead(final WrappedHttpSolrServer solrServer) { + this.checkLock.lock(); + try { + final boolean result = this.aliveServers.remove(solrServer); + if (result && this.zombieServers.addIfAbsent(solrServer)) { + if (LBHttpSolrServer.LOGGER.isWarnEnabled()) { + LBHttpSolrServer.LOGGER.warn("solr server [" + solrServer.httpSolrServer.getHost() + "] is marked as down."); + } + this.startAliveCheckExecutor(); + } + } + finally { + this.checkLock.unlock(); + } + } + + private void checkAZombieServer(final WrappedHttpSolrServer zombieServer) { + this.checkLock.lock(); + try { + if (LBHttpSolrServer.LOGGER.isInfoEnabled()) { + LBHttpSolrServer.LOGGER.info("check solr server [" + zombieServer.httpSolrServer.getHost() + "] ..."); + } + final QueryResponse resp = zombieServer.httpSolrServer.query((SolrParams)LBHttpSolrServer.SOLR_QUERY); + if (resp.getStatus() == 0) { + this.zombieServers.remove(zombieServer); + this.aliveServers.add(zombieServer); + if (LBHttpSolrServer.LOGGER.isInfoEnabled()) { + LBHttpSolrServer.LOGGER.info("solr server [" + zombieServer.httpSolrServer.getHost() + "] has come back."); + } + zombieServer.failedPings = 0; + } + } + catch (Exception e) { + zombieServer.failedPings++; + } + finally { + this.checkLock.unlock(); + } + } + + private void startAliveCheckExecutor() { + if (this.aliveCheckExecutor == null) { + synchronized (this) { + if (this.aliveCheckExecutor == null) { + (this.aliveCheckExecutor = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory())).scheduleAtFixedRate(getAliveCheckRunner(new WeakReference(this)), 10000L, 10000L, TimeUnit.MILLISECONDS); + } + } + } + } + + private static Runnable getAliveCheckRunner(final WeakReference lbHttpSolrServer) { + return new Runnable() { + @Override + public void run() { + final LBHttpSolrServer solrServer = (LBHttpSolrServer)lbHttpSolrServer.get(); + if (solrServer != null && solrServer.zombieServers != null) { + for (final WrappedHttpSolrServer zombieServer : solrServer.zombieServers) { + solrServer.checkAZombieServer(zombieServer); + } + } + } + }; + } + + public NamedList request(final SolrRequest paramSolrRequest, final String paramString) throws SolrServerException, IOException { + int attempts = 0; + final int startSize = this.aliveServers.size(); + String zombieHost = null; + while (true) { + final WrappedHttpSolrServer httpSolrServer = this.selectRandomServer(); + try { + if (null != zombieHost && paramSolrRequest.getParams() instanceof SolrQuery) { + final SolrQuery query = (SolrQuery)paramSolrRequest.getParams(); + String shards = query.get("shards"); + if (null != shards) { + shards = StringUtils.replace(shards, zombieHost, httpSolrServer.httpSolrServer.getHost()); + query.set("shards", shards); + } + } + return httpSolrServer.httpSolrServer.request(paramSolrRequest); + } + catch (SolrServerException e) { + if (!(e.getRootCause() instanceof IOException)) { + throw e; + } + final Exception ex = (Exception)e; + zombieHost = httpSolrServer.httpSolrServer.getHost(); + LBHttpSolrServer.LOGGER.error("error request solr server: " + zombieHost, (Throwable)e); + this.moveAliveToDead(httpSolrServer); + if (++attempts >= startSize) { + throw new SolrServerException("No live SolrServers available to handle this request", (Throwable)ex); + } + continue; + } + } + } + + public void shutdown() { + } + + static { + LOGGER = LoggerFactory.getLogger(LBHttpSolrServer.class); + SOLR_QUERY = new SolrQuery("*:*"); + } + + static class DefaultThreadFactory implements ThreadFactory + { + static final AtomicInteger poolNumber; + final ThreadGroup group; + final AtomicInteger threadNumber; + final String namePrefix; + + DefaultThreadFactory() { + this.threadNumber = new AtomicInteger(1); + final SecurityManager s = System.getSecurityManager(); + this.group = ((s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup()); + this.namePrefix = "solrcheck-" + DefaultThreadFactory.poolNumber.getAndIncrement() + "-thread-"; + } + + @Override + public Thread newThread(final Runnable r) { + final Thread t = new Thread(this.group, r, this.namePrefix + this.threadNumber.getAndIncrement(), 0L); + t.setDaemon(true); + if (t.getPriority() != 5) { + t.setPriority(5); + } + return t; + } + + static { + poolNumber = new AtomicInteger(1); + } + } + + private class WrappedHttpSolrServer + { + private final HttpSolrServer httpSolrServer; + private int failedPings; + + public WrappedHttpSolrServer(final HttpSolrServer httpSolrServer) { + this.failedPings = 0; + this.httpSolrServer = httpSolrServer; + } + } +} diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrException.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrException.java index d69e3595f..4f29518db 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrException.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrException.java @@ -1,23 +1,21 @@ package com.fangjia.datastore.solr; -public class SolrException extends Exception { - - private static final long serialVersionUID = -1105081175581071875L; - - public SolrException() { - super(); - } - - public SolrException(String message, Throwable cause) { - super(message, cause); - } - - public SolrException(String message) { - super(message); - } - - public SolrException(Throwable cause) { - super(cause); - } - +public class SolrException extends Exception +{ + private static final long serialVersionUID = -1105081175581071875L; + + public SolrException() { + } + + public SolrException(final String message, final Throwable cause) { + super(message, cause); + } + + public SolrException(final String message) { + super(message); + } + + public SolrException(final Throwable cause) { + super(cause); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrQueryBuilder.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrQueryBuilder.java index 43a840b47..fa56f6ba6 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrQueryBuilder.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrQueryBuilder.java @@ -1,177 +1,176 @@ package com.fangjia.datastore.solr; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; import java.util.TimeZone; - -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.time.FastDateFormat; -import org.apache.lucene.index.Term; -import org.apache.lucene.queryParser.QueryParser; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.NumericRangeQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TermRangeQuery; +import java.util.ArrayList; import org.apache.solr.client.solrj.util.ClientUtils; +import org.apache.commons.lang.StringUtils; +import java.util.Iterator; +import java.util.List; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermRangeQuery; +import java.util.Date; +import org.apache.lucene.search.NumericRangeQuery; +import org.apache.lucene.search.BooleanClause; +import org.apache.commons.lang.time.FastDateFormat; +import org.apache.lucene.search.BooleanQuery; -public class SolrQueryBuilder { - private BooleanQuery booleanQuery = new BooleanQuery(); - public final static FastDateFormat DATE_FORMAT = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", TimeZone - .getTimeZone("UTC")); - /** - * 约定:以$$开头的字符串不需要特殊字符转义 - */ - public final static String NO_ESCAPSE = "$$"; - - public SolrQueryBuilder range(String field, Object min, Object max, boolean minInclusive, boolean maxInclusive, - BooleanClause.Occur occur) { - if (min != null || max != null) { - Object obj = null == min ? max : min; - Query query = null; - if (Integer.class.isInstance(obj)) { - query = NumericRangeQuery.newIntRange(field, (Integer) min, (Integer) max, minInclusive, maxInclusive); - } else if (Long.class.isInstance(obj)) { - query = NumericRangeQuery.newLongRange(field, (Long) min, (Long) max, minInclusive, maxInclusive); - } else if (Double.class.isInstance(obj)) { - query = NumericRangeQuery.newDoubleRange(field, (Double) min, (Double) max, minInclusive, maxInclusive); - } else if (Float.class.isInstance(obj)) { - query = NumericRangeQuery.newFloatRange(field, (Float) min, (Float) max, minInclusive, maxInclusive); - } else if (Date.class.isInstance(obj)) { - query = new TermRangeQuery(field, null == min ? null : DATE_FORMAT.format((Date) min), null == max ? null - : DATE_FORMAT.format((Date) max), minInclusive, maxInclusive); - } else { - query = new TermRangeQuery(field, null == min ? (String) min : min.toString(), null == max ? (String) max : max - .toString(), minInclusive, maxInclusive); - } - booleanQuery.add(query, occur); - } - return this; - } - - public SolrQueryBuilder range(String field, Object min, Object max, boolean minInclusive, boolean maxInclusive) { - return range(field, min, max, minInclusive, maxInclusive, BooleanClause.Occur.MUST); - } - - public SolrQueryBuilder range(String field, Object min, Object max) { - return range(field, min, max, true, true, BooleanClause.Occur.MUST); - } - - public SolrQueryBuilder range(String field, Object min, Object max, BooleanClause.Occur occur) { - return range(field, min, max, true, true, occur); - } - - public SolrQueryBuilder gt(String field, Object value) { - return range(field, value, null, false, true, BooleanClause.Occur.MUST); - } - - public SolrQueryBuilder gt(String field, Object value, BooleanClause.Occur occur) { - return range(field, value, null, false, true, occur); - } - - public SolrQueryBuilder gte(String field, Object value) { - return range(field, value, null, true, true, BooleanClause.Occur.MUST); - } - - public SolrQueryBuilder gte(String field, Object value, BooleanClause.Occur occur) { - return range(field, value, null, true, true, occur); - } - - public SolrQueryBuilder lt(String field, Object value) { - return range(field, null, value, true, false, BooleanClause.Occur.MUST); - } - - public SolrQueryBuilder lt(String field, Object value, BooleanClause.Occur occur) { - return range(field, null, value, true, false, occur); - } - - public SolrQueryBuilder lte(String field, Object value) { - return range(field, null, value, true, true, BooleanClause.Occur.MUST); - } - - public SolrQueryBuilder lte(String field, Object value, BooleanClause.Occur occur) { - return range(field, null, value, true, true, occur); - } - - public SolrQueryBuilder eq(String field, Object value, BooleanClause.Occur occur) { - if (null != value) { - Query query = new TermQuery(new Term(field, value.toString())); - booleanQuery.add(query, occur); - } - return this; - } - - public SolrQueryBuilder eq(String field, Object value) { - return eq(field, getRealValue(value), BooleanClause.Occur.MUST); - } - - public SolrQueryBuilder eq(String field, Object value, boolean escapse) { - return eq(field, escapse ? QueryParser.escape(value.toString()) : value.toString(), BooleanClause.Occur.MUST); - } - - public SolrQueryBuilder ne(String field, Object value) { - return eq(field, getRealValue(value), BooleanClause.Occur.MUST_NOT); - } - - public SolrQueryBuilder in(String field, List value, boolean escapse, BooleanClause.Occur occur) { - if (null == value || value.isEmpty()) { - throw new IllegalArgumentException("in query with null or empty list!!"); - } - BooleanQuery bQuery = new BooleanQuery(); - for (Object v : value) { - Query query = new TermQuery(new Term(field, escapse ? QueryParser.escape(v.toString()) : v.toString())); - bQuery.add(query, BooleanClause.Occur.SHOULD); - } - booleanQuery.add(bQuery, occur); - return this; - } - - public SolrQueryBuilder in(String field, List value) { - return in(field, getRealValue(value), false, BooleanClause.Occur.MUST); - } - - public SolrQueryBuilder in(String field, List value, BooleanClause.Occur occur) { - return in(field, getRealValue(value), false, occur); - } - - public SolrQueryBuilder in(String field, List value, boolean escapse) { - return in(field, value, escapse, BooleanClause.Occur.MUST); - } - - public Query toQuery() { - return this.booleanQuery; - } - - /** - * 获取真实value
- * 如果是非string的value,返回转义后的原值
- * 如果是string型,如果以$$开头,则截取$$后面的字符且不需要转义 - * - * @param value - * @return - */ - private Object getRealValue(Object value) { - if (value instanceof String) { - if (NO_ESCAPSE.equals(value)) { - return value; - } else if (StringUtils.startsWith((String) value, NO_ESCAPSE)) { - return StringUtils.substring((String) value, NO_ESCAPSE.length()); - } - } - return ClientUtils.escapeQueryChars(value.toString()); -// return QueryParser.escape(value.toString()); - } - - private List getRealValue(List value) { - if (null == value || value.isEmpty()) { - return value; - } - List ret = new ArrayList(value.size()); - for (Object v : value) { - ret.add(getRealValue(v)); - } - return ret; - } +public class SolrQueryBuilder +{ + private BooleanQuery booleanQuery; + public static final FastDateFormat DATE_FORMAT; + public static final String NO_ESCAPSE = "$$"; + + public SolrQueryBuilder() { + this.booleanQuery = new BooleanQuery(); + } + + public SolrQueryBuilder range(final String field, final Object min, final Object max, final boolean minInclusive, final boolean maxInclusive, final BooleanClause.Occur occur) { + if (min != null || max != null) { + final Object obj = (null == min) ? max : min; + Query query = null; + if (Integer.class.isInstance(obj)) { + query = (Query)NumericRangeQuery.newIntRange(field, (Integer)min, (Integer)max, minInclusive, maxInclusive); + } + else if (Long.class.isInstance(obj)) { + query = (Query)NumericRangeQuery.newLongRange(field, (Long)min, (Long)max, minInclusive, maxInclusive); + } + else if (Double.class.isInstance(obj)) { + query = (Query)NumericRangeQuery.newDoubleRange(field, (Double)min, (Double)max, minInclusive, maxInclusive); + } + else if (Float.class.isInstance(obj)) { + query = (Query)NumericRangeQuery.newFloatRange(field, (Float)min, (Float)max, minInclusive, maxInclusive); + } + else if (Date.class.isInstance(obj)) { + query = (Query)new TermRangeQuery(field, (null == min) ? null : SolrQueryBuilder.DATE_FORMAT.format((Date)min), (null == max) ? null : SolrQueryBuilder.DATE_FORMAT.format((Date)max), minInclusive, maxInclusive); + } + else { + query = (Query)new TermRangeQuery(field, (null == min) ? ((String)min) : min.toString(), (null == max) ? ((String)max) : max.toString(), minInclusive, maxInclusive); + } + this.booleanQuery.add(query, occur); + } + return this; + } + + public SolrQueryBuilder range(final String field, final Object min, final Object max, final boolean minInclusive, final boolean maxInclusive) { + return this.range(field, min, max, minInclusive, maxInclusive, BooleanClause.Occur.MUST); + } + + public SolrQueryBuilder range(final String field, final Object min, final Object max) { + return this.range(field, min, max, true, true, BooleanClause.Occur.MUST); + } + + public SolrQueryBuilder range(final String field, final Object min, final Object max, final BooleanClause.Occur occur) { + return this.range(field, min, max, true, true, occur); + } + + public SolrQueryBuilder gt(final String field, final Object value) { + return this.range(field, value, null, false, true, BooleanClause.Occur.MUST); + } + + public SolrQueryBuilder gt(final String field, final Object value, final BooleanClause.Occur occur) { + return this.range(field, value, null, false, true, occur); + } + + public SolrQueryBuilder gte(final String field, final Object value) { + return this.range(field, value, null, true, true, BooleanClause.Occur.MUST); + } + + public SolrQueryBuilder gte(final String field, final Object value, final BooleanClause.Occur occur) { + return this.range(field, value, null, true, true, occur); + } + + public SolrQueryBuilder lt(final String field, final Object value) { + return this.range(field, null, value, true, false, BooleanClause.Occur.MUST); + } + + public SolrQueryBuilder lt(final String field, final Object value, final BooleanClause.Occur occur) { + return this.range(field, null, value, true, false, occur); + } + + public SolrQueryBuilder lte(final String field, final Object value) { + return this.range(field, null, value, true, true, BooleanClause.Occur.MUST); + } + + public SolrQueryBuilder lte(final String field, final Object value, final BooleanClause.Occur occur) { + return this.range(field, null, value, true, true, occur); + } + + public SolrQueryBuilder eq(final String field, final Object value, final BooleanClause.Occur occur) { + if (null != value) { + final Query query = (Query)new TermQuery(new Term(field, value.toString())); + this.booleanQuery.add(query, occur); + } + return this; + } + + public SolrQueryBuilder eq(final String field, final Object value) { + return this.eq(field, this.getRealValue(value), BooleanClause.Occur.MUST); + } + + public SolrQueryBuilder eq(final String field, final Object value, final boolean escapse) { + return this.eq(field, escapse ? QueryParser.escape(value.toString()) : value.toString(), BooleanClause.Occur.MUST); + } + + public SolrQueryBuilder ne(final String field, final Object value) { + return this.eq(field, this.getRealValue(value), BooleanClause.Occur.MUST_NOT); + } + + public SolrQueryBuilder in(final String field, final List value, final boolean escapse, final BooleanClause.Occur occur) { + if (null == value || value.isEmpty()) { + throw new IllegalArgumentException("in query with null or empty list!!"); + } + final BooleanQuery bQuery = new BooleanQuery(); + for (final Object v : value) { + final Query query = (Query)new TermQuery(new Term(field, escapse ? QueryParser.escape(v.toString()) : v.toString())); + bQuery.add(query, BooleanClause.Occur.SHOULD); + } + this.booleanQuery.add((Query)bQuery, occur); + return this; + } + + public SolrQueryBuilder in(final String field, final List value) { + return this.in(field, this.getRealValue(value), false, BooleanClause.Occur.MUST); + } + + public SolrQueryBuilder in(final String field, final List value, final BooleanClause.Occur occur) { + return this.in(field, this.getRealValue(value), false, occur); + } + + public SolrQueryBuilder in(final String field, final List value, final boolean escapse) { + return this.in(field, value, escapse, BooleanClause.Occur.MUST); + } + + public Query toQuery() { + return (Query)this.booleanQuery; + } + + private Object getRealValue(final Object value) { + if (value instanceof String) { + if ("$$".equals(value)) { + return value; + } + if (StringUtils.startsWith((String)value, "$$")) { + return StringUtils.substring((String)value, "$$".length()); + } + } + return ClientUtils.escapeQueryChars(value.toString()); + } + + private List getRealValue(final List value) { + if (null == value || value.isEmpty()) { + return value; + } + final List ret = new ArrayList(value.size()); + for (final Object v : value) { + ret.add(this.getRealValue(v)); + } + return ret; + } + + static { + DATE_FORMAT = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", TimeZone.getTimeZone("UTC")); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrSchema.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrSchema.java index 5b4f06359..d04b48bb0 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrSchema.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrSchema.java @@ -1,26 +1,26 @@ package com.fangjia.datastore.solr; import java.io.File; -import java.util.Collection; import java.util.Set; - +import java.util.Collection; import com.fangjia.datastore.config.Reloadable; -public interface SolrSchema extends Reloadable { - public final static String VERTION = "_version"; - public final static String ID = "_id"; - - int saveIndex(Collection indexes) throws SolrException; - - boolean removeIndex(String name) throws SolrException; - - Index getIndex(String name); - - Collection getIndexes(); - - Set getIndexNames(); - - File getSchemaFile(); - - void setSchemaFile(File schemaFile); -} \ No newline at end of file +public interface SolrSchema extends Reloadable +{ + public static final String VERTION = "_version"; + public static final String ID = "_id"; + + int saveIndex(final Collection p0) throws SolrException; + + boolean removeIndex(final String p0) throws SolrException; + + Index getIndex(final String p0); + + Collection getIndexes(); + + Set getIndexNames(); + + File getSchemaFile(); + + void setSchemaFile(final File p0); +} diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrSchemaImpl.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrSchemaImpl.java index e2312d7d4..d76c7202b 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrSchemaImpl.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrSchemaImpl.java @@ -1,189 +1,194 @@ package com.fangjia.datastore.solr; -import java.io.File; -import java.io.FileOutputStream; +import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.lang.StringUtils; -import org.dom4j.Document; +import java.io.OutputStream; +import org.dom4j.io.XMLWriter; +import java.io.FileOutputStream; import org.dom4j.DocumentException; +import java.util.Set; +import java.util.List; +import org.apache.commons.lang.StringUtils; +import org.dom4j.Node; +import java.util.Iterator; import org.dom4j.DocumentHelper; import org.dom4j.Element; -import org.dom4j.Node; -import org.dom4j.io.SAXReader; -import org.dom4j.io.XMLWriter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import java.util.Collection; import com.fangjia.datastore.ConfigurationException; -import com.fangjia.datastore.solr.Index.IndexTypeEnum; +import java.util.HashMap; +import org.slf4j.Logger; +import org.dom4j.Document; +import java.util.Map; +import java.io.File; +import org.dom4j.io.SAXReader; -public class SolrSchemaImpl implements SolrSchema { - private final SAXReader reader = new SAXReader(); - private File schemaFile; - private final Map indexes = new HashMap(); - private Document document; - private final static Logger LOGGER = LoggerFactory.getLogger(SolrSchemaImpl.class); - - public static void main(String[] args) throws Exception { - new SolrSchemaImpl(new File("D:\\viewstore\\solrhome\\house\\conf\\schema.xml")); - } - - public SolrSchemaImpl(File schemaFile) throws ConfigurationException { - if (!schemaFile.exists() || schemaFile.isDirectory()) { - throw new IllegalArgumentException("schema file not exists or is a directory:" + schemaFile); - } - this.schemaFile = schemaFile; - reload(); - } - - public int saveIndex(Collection newIndexes) throws SolrException { - int ret = 0; - try { - for (Index index : newIndexes) { - Index i = indexes.get(index.getName()); - if (index.equals(i)) { - continue; - } - ret++; - Element e; - if (indexes.containsKey(index.getName())) { - String xpath = "//schema//fields//" + (index.isDynamic() ? "dynamicField" : "field") + "[name=\"" - + index.getName() + "\"]"; - e = (Element) this.document.selectSingleNode(xpath); - } else { - e = DocumentHelper.createElement(index.isDynamic() ? "dynamicField" : "field"); - Element fields = (Element) document.selectSingleNode("//schema//fields"); - fields.add(e); - } - e.addAttribute("name", index.getName()); - e.addAttribute("type", index.getType().getType()); - e.addAttribute("stored", String.valueOf(index.isStored())); - e.addAttribute("indexed", String.valueOf(index.isIndexed())); - e.addAttribute("multiValued", String.valueOf(index.isMultiValued())); - e.addAttribute("comments", String.valueOf(index.getComments())); - this.indexes.put(index.getName(), index); - } - } finally { - write(); - } - return ret; - } - - public synchronized boolean removeIndex(String name) throws SolrException { - Index index = this.indexes.get(name); - if (null == index) { - return false; - } - if (ID.equals(name) || VERTION.equals(name)) { - throw new SolrException("delete of field \"" + name + "\" is not allowed!"); - } - String xpath = "//schema//fields//" + (index.isDynamic() ? "dynamicField" : "field") + "[name=\"" + index.getName() - + "\"]"; - Element e = (Element) this.document.selectSingleNode(xpath); - if (null != e) { - e.detach(); - } - write(); - this.indexes.remove(name); - return true; - } - - public Collection getIndexes() { - return indexes.values(); - } - - @SuppressWarnings("unchecked") - public void reload() throws ConfigurationException { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("refresh solr schema: {}", this.schemaFile); - } - Map newIndexes = new HashMap(); - - Document newDocument = read(); - - List list = newDocument.selectNodes("//schema"); - if (null == list || list.size() > 1) { - throw new ConfigurationException("zero or more than one \"fields\" tag found, expect only one \"fields\" tag"); - } - Node fieldsNode = list.get(0); - list = fieldsNode.selectNodes("field"); - for (Node node : list) { - Element e = (Element) node; - - String name = e.attributeValue("name"); - if (StringUtils.isEmpty(name)) { - throw new ConfigurationException("invalid field name \"" + name + "\" in solr schema: " + this.schemaFile); - } - - IndexTypeEnum type = IndexTypeEnum.parse(e.attributeValue("type")); - if (null == type) { - throw new ConfigurationException("invalid field type \"" + e.attributeValue("type") + "\" in solr schema: " - + this.schemaFile); - } - - boolean stored = Boolean.parseBoolean(e.attributeValue("stored")); - boolean indexed = Boolean.parseBoolean(e.attributeValue("indexed")); - boolean multiValued = Boolean.parseBoolean(e.attributeValue("multiValued")); - String comments = e.attributeValue("comments"); - boolean dynamic = e.getName().equals("dynamicField"); - - Index index = new Index(name, type, stored, indexed, multiValued, dynamic, comments); - newIndexes.put(name, index); - } - if (!newIndexes.containsKey(ID)) { - throw new ConfigurationException("no field names \"" + ID + "\" find in schema: " + this.schemaFile); - } - if (!newIndexes.containsKey(VERTION)) { - throw new ConfigurationException("no field names \"" + VERTION + "\" find in schema: " + this.schemaFile); - } - indexes.clear(); - indexes.putAll(newIndexes); - this.document = newDocument; - - } - - public Set getIndexNames() { - return this.indexes.keySet(); - } - - private Document read() throws ConfigurationException { - try { - return reader.read(schemaFile); - } catch (DocumentException e) { - throw new ConfigurationException("error read solr xml file: " + this, e); - } - } - - private void write() throws SolrException { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("write solr schema: {}" + this.schemaFile); - } - XMLWriter writer; - try { - writer = new XMLWriter(new FileOutputStream(schemaFile)); - writer.write(document); - writer.close(); - } catch (IOException e) { - throw new SolrException("error write solr xml file: " + this, e); - } - } - - public Index getIndex(String name) { - return this.indexes.get(name); - } - - public File getSchemaFile() { - return schemaFile; - } - - public void setSchemaFile(File schemaFile) { - this.schemaFile = schemaFile; - } - -} \ No newline at end of file +public class SolrSchemaImpl implements SolrSchema +{ + private final SAXReader reader; + private File schemaFile; + private final Map indexes; + private Document document; + private static final Logger LOGGER; + + public static void main(final String[] args) throws Exception { + new SolrSchemaImpl(new File("D:\\viewstore\\solrhome\\house\\conf\\schema.xml")); + } + + public SolrSchemaImpl(final File schemaFile) throws ConfigurationException { + this.reader = new SAXReader(); + this.indexes = new HashMap(); + if (!schemaFile.exists() || schemaFile.isDirectory()) { + throw new IllegalArgumentException("schema file not exists or is a directory:" + schemaFile); + } + this.schemaFile = schemaFile; + this.reload(); + } + + @Override + public int saveIndex(final Collection newIndexes) throws SolrException { + int ret = 0; + try { + for (final Index index : newIndexes) { + final Index i = this.indexes.get(index.getName()); + if (index.equals(i)) { + continue; + } + ++ret; + Element e; + if (this.indexes.containsKey(index.getName())) { + final String xpath = "//schema//fields//" + (index.isDynamic() ? "dynamicField" : "field") + "[name=\"" + index.getName() + "\"]"; + e = (Element)this.document.selectSingleNode(xpath); + } + else { + e = DocumentHelper.createElement(index.isDynamic() ? "dynamicField" : "field"); + final Element fields = (Element)this.document.selectSingleNode("//schema//fields"); + fields.add(e); + } + e.addAttribute("name", index.getName()); + e.addAttribute("type", index.getType().getType()); + e.addAttribute("stored", String.valueOf(index.isStored())); + e.addAttribute("indexed", String.valueOf(index.isIndexed())); + e.addAttribute("multiValued", String.valueOf(index.isMultiValued())); + e.addAttribute("comments", String.valueOf(index.getComments())); + this.indexes.put(index.getName(), index); + } + } + finally { + this.write(); + } + return ret; + } + + @Override + public synchronized boolean removeIndex(final String name) throws SolrException { + final Index index = this.indexes.get(name); + if (null == index) { + return false; + } + if ("_id".equals(name) || "_version".equals(name)) { + throw new SolrException("delete of field \"" + name + "\" is not allowed!"); + } + final String xpath = "//schema//fields//" + (index.isDynamic() ? "dynamicField" : "field") + "[name=\"" + index.getName() + "\"]"; + final Element e = (Element)this.document.selectSingleNode(xpath); + if (null != e) { + e.detach(); + } + this.write(); + this.indexes.remove(name); + return true; + } + + @Override + public Collection getIndexes() { + return this.indexes.values(); + } + + @Override + public void reload() throws ConfigurationException { + if (SolrSchemaImpl.LOGGER.isDebugEnabled()) { + SolrSchemaImpl.LOGGER.debug("refresh solr schema: {}", (Object)this.schemaFile); + } + final Map newIndexes = new HashMap(); + final Document newDocument = this.read(); + List list = (List)newDocument.selectNodes("//schema"); + if (null == list || list.size() > 1) { + throw new ConfigurationException("zero or more than one \"fields\" tag found, expect only one \"fields\" tag"); + } + final Node fieldsNode = list.get(0); + list = (List)fieldsNode.selectNodes("field"); + for (final Node node : list) { + final Element e = (Element)node; + final String name = e.attributeValue("name"); + if (StringUtils.isEmpty(name)) { + throw new ConfigurationException("invalid field name \"" + name + "\" in solr schema: " + this.schemaFile); + } + final Index.IndexTypeEnum type = Index.IndexTypeEnum.parse(e.attributeValue("type")); + if (null == type) { + throw new ConfigurationException("invalid field type \"" + e.attributeValue("type") + "\" in solr schema: " + this.schemaFile); + } + final boolean stored = Boolean.parseBoolean(e.attributeValue("stored")); + final boolean indexed = Boolean.parseBoolean(e.attributeValue("indexed")); + final boolean multiValued = Boolean.parseBoolean(e.attributeValue("multiValued")); + final String comments = e.attributeValue("comments"); + final boolean dynamic = e.getName().equals("dynamicField"); + final Index index = new Index(name, type, stored, indexed, multiValued, dynamic, comments); + newIndexes.put(name, index); + } + if (!newIndexes.containsKey("_id")) { + throw new ConfigurationException("no field names \"_id\" find in schema: " + this.schemaFile); + } + if (!newIndexes.containsKey("_version")) { + throw new ConfigurationException("no field names \"_version\" find in schema: " + this.schemaFile); + } + this.indexes.clear(); + this.indexes.putAll(newIndexes); + this.document = newDocument; + } + + @Override + public Set getIndexNames() { + return this.indexes.keySet(); + } + + private Document read() throws ConfigurationException { + try { + return this.reader.read(this.schemaFile); + } + catch (DocumentException e) { + throw new ConfigurationException("error read solr xml file: " + this, (Throwable)e); + } + } + + private void write() throws SolrException { + if (SolrSchemaImpl.LOGGER.isDebugEnabled()) { + SolrSchemaImpl.LOGGER.debug("write solr schema: {}" + this.schemaFile); + } + try { + final XMLWriter writer = new XMLWriter((OutputStream)new FileOutputStream(this.schemaFile)); + writer.write(this.document); + writer.close(); + } + catch (IOException e) { + throw new SolrException("error write solr xml file: " + this, e); + } + } + + @Override + public Index getIndex(final String name) { + return this.indexes.get(name); + } + + @Override + public File getSchemaFile() { + return this.schemaFile; + } + + @Override + public void setSchemaFile(final File schemaFile) { + this.schemaFile = schemaFile; + } + + static { + LOGGER = LoggerFactory.getLogger((Class)SolrSchemaImpl.class); + } +} diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrServer.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrServer.java index 8ce424980..684ba0e35 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrServer.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrServer.java @@ -1,23 +1,22 @@ package com.fangjia.datastore.solr; -import java.util.List; - -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.response.QueryResponse; import org.bson.BSONObject; - +import java.util.List; import com.fangjia.datastore.ConfigurationException; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.SolrQuery; -public interface SolrServer { - QueryResponse query(SolrQuery params, boolean fromMaster, boolean bulk) throws SolrException; - - void deleteByQuery(String query) throws SolrException; - - void reload() throws ConfigurationException; - - void reloadCores() throws SolrException; - - SolrSchema getSolrSchema(); - - void add(List dbObjects) throws SolrException; +public interface SolrServer +{ + QueryResponse query(final SolrQuery p0, final boolean p1, final boolean p2) throws SolrException; + + void deleteByQuery(final String p0) throws SolrException; + + void reload() throws ConfigurationException; + + void reloadCores() throws SolrException; + + SolrSchema getSolrSchema(); + + void add(final List p0) throws SolrException; } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrServerImpl.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrServerImpl.java index 8bd4287db..b9da9e2f8 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrServerImpl.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrServerImpl.java @@ -1,475 +1,455 @@ package com.fangjia.datastore.solr; -import java.io.File; -import java.io.IOException; -import java.net.MalformedURLException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.math.RandomUtils; -import org.apache.http.client.HttpClient; -import org.apache.http.client.config.CookieSpecs; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.SolrInputField; -import org.apache.solr.common.params.ShardParams; -import org.bson.BSONObject; -import org.bson.types.ObjectId; -import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.sothis.core.util.BlockingThreadPoolExecutor; - -import com.fangjia.datastore.ConfigurationException; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.commons.lang.math.RandomUtils; +import java.io.IOException; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.commons.lang.StringUtils; +import java.util.Collections; +import org.apache.commons.collections.CollectionUtils; +import org.bson.types.ObjectId; +import org.apache.solr.common.SolrInputField; +import java.util.Iterator; +import java.util.Date; +import java.util.ArrayList; +import java.util.concurrent.CountDownLatch; +import org.bson.BSONObject; +import java.util.Collection; +import org.apache.solr.common.SolrInputDocument; +import java.util.List; +import java.util.HashMap; +import java.net.MalformedURLException; import com.fangjia.datastore.config.SolrConfig; +import com.fangjia.datastore.ConfigurationException; +import org.sothis.core.util.BlockingThreadPoolExecutor; +import java.io.File; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.Executor; +import java.util.Map; +import org.apache.http.client.HttpClient; import com.fangjia.datastore.config.StoreConfig; +import org.slf4j.Logger; -public class SolrServerImpl implements SolrServer { - - private final static Logger LOGGER = LoggerFactory.getLogger(SolrServerImpl.class); - private final static int SOLR_COMMIT_MAX_RETRY_COUNT = 3; - private final static int SOLR_QUERY_MAX_RETRY_COUNT = 3; - private final static int MAX_NONE_BULK_SKIP = 100000; - - private final String name; - private final StoreConfig storeConfig; - private final HttpClient httpClient; - private int masterCommitSolrServerCount; - private final Map allSolrServers; - private String[] masterCommitFilters; - private HttpSolrServer[][] masterCommitSolrServers; - private HttpSolrServer[] masterSolrServers; - private HttpSolrServer[] slaveSolrServers; - private SolrSchema[] solrSchemas; - private final SolrSchema solrSchema; - private final Executor solrAddExecutorService; - private final AtomicBoolean filtering = new AtomicBoolean(false); - private long lastFilterTime = 0; - - public SolrServerImpl(StoreConfig storeConfig) throws ConfigurationException { - this.storeConfig = storeConfig; - this.name = storeConfig.getName(); - - /*MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager(); - connectionManager.getParams().setTcpNoDelay(true); - connectionManager.getParams().setStaleCheckingEnabled(true); - this.httpClient = new HttpClient(connectionManager); - httpClient.getParams().setConnectionManagerTimeout(0); - DefaultHttpMethodRetryHandler retryhandler = new DefaultHttpMethodRetryHandler(0, false); - this.httpClient.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryhandler); - */ - PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); - cm.setMaxTotal(5); - /* RequestConfig requestConfig = RequestConfig.custom() - .setConnectTimeout(1000 * 10) - .setSocketTimeout(1000 * 10) - .setCookieSpec(CookieSpecs.IGNORE_COOKIES) - .build(); - - this.httpClient = HttpClients.custom() - .setDefaultRequestConfig(requestConfig) - .setConnectionManager(cm) - .build();*/ - this.httpClient = new DefaultHttpClient(); - this.allSolrServers = initSolrServers(this.storeConfig, httpClient); - this.solrSchema = new SolrSchemaImpl(new File(storeConfig.getMasterSolrConfig().getHome() + "/" + name - + "/conf/managed-schema")); - int size = (masterCommitSolrServerCount + 1) * 2; - this.solrAddExecutorService = new BlockingThreadPoolExecutor(name + "-SolrAdd", 1, size, size, 60000); - } - - private CommonsHttpSolrServer createSolrServer(String coreName, SolrConfig solronfig) throws ConfigurationException { - try { - CommonsHttpSolrServer solrServer = new CommonsHttpSolrServer(solronfig.getHost(), coreName, httpClient); - return solrServer; - } catch (MalformedURLException e) { - throw new ConfigurationException("[" + this.name + "] error create solr server: ", e); - } - } - - private HttpSolrServer[] initSolrServer(Map allSolrServers, SolrConfig solrConfig) - throws ConfigurationException { - SolrConfig[] shardsConfigs = solrConfig.getShardsConfigs(); - if (shardsConfigs.length == 0) { - shardsConfigs = new SolrConfig[] { solrConfig }; - } - HttpSolrServer[] shards = new HttpSolrServer[shardsConfigs.length]; - for (int i = 0; i < shardsConfigs.length; i++) { - SolrConfig shardConfig = shardsConfigs[i]; - if (!allSolrServers.containsKey(shardConfig.getId())) { - SolrConfig[] slaveConfigs = shardConfig.getSlaveConfigs(); - if (slaveConfigs.length == 0) { - allSolrServers.put(shardConfig.getId(), createSolrServer(name, shardConfig)); - } else { - CommonsHttpSolrServer[] slaveServers = new CommonsHttpSolrServer[slaveConfigs.length]; - for (int j = 0; j < slaveConfigs.length; j++) { - SolrConfig slaveConfig = slaveConfigs[j]; - if (!allSolrServers.containsKey(slaveConfig.getId())) { - allSolrServers.put(slaveConfig.getId(), createSolrServer(name, slaveConfig)); - } - slaveServers[j] = (CommonsHttpSolrServer) allSolrServers.get(slaveConfig.getId()); - } - allSolrServers.put(shardConfig.getId(), new LBHttpSolrServer(slaveServers)); - } - } - shards[i] = allSolrServers.get(shardConfig.getId()); - } - return shards; - } - - private synchronized Map initSolrServers(StoreConfig storeConfig, HttpClient httpClient) - throws ConfigurationException { - // http connections - //httpClient.getHttpConnectionManager().getParams().setDefaultMaxConnectionsPerHost( - //storeConfig.getSolrMaxConnectionsPerHost()); - //httpClient.getHttpConnectionManager().getParams().setMaxTotalConnections(storeConfig.getSolrMaxTotalConnections()); - - Map solrServers = new HashMap(); - SolrConfig[] masterConfigs = storeConfig.getMasterSolrConfigs(); - masterCommitSolrServers = new HttpSolrServer[masterConfigs.length][]; - masterCommitFilters = new String[masterConfigs.length]; - solrSchemas = new SolrSchema[masterConfigs.length]; - masterCommitSolrServerCount = 0; - for (int i = 0; i < masterConfigs.length; i++) { - masterCommitSolrServers[i] = initSolrServer(solrServers, masterConfigs[i]); - masterCommitFilters[i] = masterConfigs[i].getFilter(); - solrSchemas[i] = new SolrSchemaImpl(new File(masterConfigs[i].getHome() + "/" + name + "/conf/managed-schema")); - masterCommitSolrServerCount += masterCommitSolrServers[i].length; - } - masterSolrServers = initSolrServer(solrServers, storeConfig.getMasterSolrConfig()); - slaveSolrServers = initSolrServer(solrServers, storeConfig.getSlaveSolrConfig()); - - return solrServers; - } - - private void addToSolr0(HttpSolrServer solrServer, List documents, int errorCount) { - try { - if (null != documents && !documents.isEmpty()) { - solrServer.add(documents); - solrServer.commit(); - } - } catch (Exception e) { - if (errorCount <= SOLR_COMMIT_MAX_RETRY_COUNT) { - errorCount++; - long sleepTime = errorCount * errorCount; - if (LOGGER.isInfoEnabled()) { - LOGGER.info("[" + this.name + "] error commit to solr[" + solrServer.getHost() + "], sleep " + sleepTime - + "s and retry(" + e.getMessage() + ")"); - } - try { - Thread.sleep(1000 * sleepTime); - } catch (InterruptedException e1) { - LOGGER.error("[" + this.name + "] error save to solr: ", e1); - } - addToSolr0(solrServer, documents, errorCount); - } else { - if (LOGGER.isErrorEnabled()) { - LOGGER.error("[" + this.name + "] error save to solr[" + solrServer.getHost() + "] after " - + SOLR_COMMIT_MAX_RETRY_COUNT + " times retry: ", e); - } - } - } - } - - public void add(List dbObjects) throws SolrException { - if (null == dbObjects || dbObjects.size() == 0) { - return; - } - try { - long time = System.currentTimeMillis(); - checkFilter(); - final CountDownLatch countDownLatch = new CountDownLatch(masterCommitSolrServerCount); - for (int j = 0; j < this.masterCommitSolrServers.length; j++) { - final int jndex = j; - final Map> shardedObjectMap = shard(dbObjects, this.masterCommitSolrServers[j].length); - for (int i = 0; i < this.masterCommitSolrServers[j].length; i++) { - final int index = i; - this.solrAddExecutorService.execute(new Runnable() { - public void run() { - try { - List sharedObjects = shardedObjectMap.get(index); - if (null == sharedObjects || sharedObjects.size() == 0) { - return; - } - final List documents = new ArrayList(sharedObjects.size()); - for (BSONObject obj : sharedObjects) { - SolrInputDocument document = toSolrInputDocument(obj, solrSchemas[jndex]); - if (!document.isEmpty()) { - document.addField(SolrSchema.VERTION, Core.VERSION_DATE_FORMAT.format(new Date())); - System.out.println("document:" + document.toString()); - documents.add(document); - } else if (LOGGER.isDebugEnabled()) { - LOGGER.debug("solr input document is empty:" + document); - } - } - addToSolr0(masterCommitSolrServers[jndex][index], documents, 0); - } catch (Exception e) { - if (LOGGER.isErrorEnabled()) { - LOGGER.error("[" + name + "] error save to solr: ", e); - } - } finally { - countDownLatch.countDown(); - } - } - }); - } - } - countDownLatch.await(); - time = System.currentTimeMillis() - time; - if (LOGGER.isInfoEnabled()) { - LOGGER.info("[" + this.name + "] add {} documents to solr, {}ms, {}/s.", new Object[] { dbObjects.size(), time, - dbObjects.size() * 1000 / (time) }); - } - } catch (Exception e) { - if (LOGGER.isErrorEnabled()) { - LOGGER.error("[" + this.name + "] error save to solr: ", e); - } - } - } - - @SuppressWarnings("unchecked") - private SolrInputDocument toSolrInputDocument(BSONObject doc, SolrSchema solrSchema) { - Collection indexes = solrSchema.getIndexes(); - SolrInputDocument document = new SolrInputDocument(); - Map boostsMap = null; - if (doc.get("$boosts") != null) { - if (doc.get("$boosts") instanceof Map) { - boostsMap = (Map) doc.get("$boosts"); - if (boostsMap.containsKey("$doc_boost")) { - document.setDocumentBoost(((Number) boostsMap.get("$doc_boost")).floatValue()); - } - } - } - - for (Index index : indexes) { - String key = index.getName(); - if (doc.containsField(key)) { - Object value = doc.get(key); - SolrInputField field = new SolrInputField(key); - if (value instanceof ObjectId) { - field.setValue(value.toString(), 1.0f); - } else { - if (boostsMap != null && boostsMap.containsKey(key)) { - field.setValue(value, ((Number) boostsMap.get(key)).floatValue()); - } else { - field.setValue(value, 1.0f); - } - } - document.put(key, field); - } - } - return document; - } - - private Map> shard(final List dbObjects, int shards) { - if (CollectionUtils.isEmpty(dbObjects)) { - return Collections.emptyMap(); - } - Map> ret = new HashMap>(); - for (BSONObject object : dbObjects) { - int hashMod = Math.abs(object.get("_id").hashCode()) % shards; - List modObjects = ret.get(hashMod); - if (null == modObjects) { - modObjects = new ArrayList(); - ret.put(hashMod, modObjects); - } - modObjects.add(object); - } - return ret; - } - - private void checkFilter() { - // 5分钟一次 - if (System.currentTimeMillis() - lastFilterTime < 300000) { - return; - } - try { - if (this.filtering.compareAndSet(false, true)) { - final long start = System.currentTimeMillis(); - for (int i = 0; i < masterCommitFilters.length; i++) { - if (StringUtils.isEmpty(masterCommitFilters[i])) { - continue; - } - for (HttpSolrServer server : masterCommitSolrServers[i]) { - try { - server.deleteByQuery(masterCommitFilters[i]); - } catch (SolrServerException e) { - if (LOGGER.isErrorEnabled()) { - LOGGER.error("[" + this.name + "] error execute filter delete: ", e); - } - } catch (IOException e) { - if (LOGGER.isErrorEnabled()) { - LOGGER.error("[" + this.name + "] error execute filter delete: ", e); - } - } - } - } - if (LOGGER.isInfoEnabled()) { - LOGGER.info("[" + this.name + "] execute filter delete, " + (System.currentTimeMillis() - start) / 1000 - + "s."); - } - lastFilterTime = System.currentTimeMillis(); - } - } catch (Exception e) { - if (LOGGER.isErrorEnabled()) { - LOGGER.error("[" + this.name + "] error execute filter delete: ", e); - } - } finally { - this.filtering.compareAndSet(true, false); - } - } - - public void deleteByQuery(String query) throws SolrException { - try { - for (HttpSolrServer[] servers : this.masterCommitSolrServers) { - for (HttpSolrServer server : servers) { - server.deleteByQuery(query); - } - } - } catch (SolrServerException e) { - throw new SolrException("[" + this.name + "] error execute delete[" + query + "]: ", e); - } catch (IOException e) { - throw new SolrException("[" + this.name + "] error execute delete[" + query + "]: ", e); - } - return; - } - - public synchronized void reload() throws ConfigurationException { - this.allSolrServers.clear(); - this.allSolrServers.putAll(initSolrServers(this.storeConfig, httpClient)); - String schema = this.storeConfig.getMasterSolrConfig().getHome() + "/" + name + "/conf/managed-schema"; - this.solrSchema.setSchemaFile(new File(schema)); - this.solrSchema.reload(); - } - - public void reloadCores() throws SolrException { - if (LOGGER.isInfoEnabled()) { - LOGGER.info("[" + this.name + "] reload core..."); - } - for (String id : this.allSolrServers.keySet()) { - HttpSolrServer solrServer = this.allSolrServers.get(id); - solrServer.reloadCore(); - } - if (LOGGER.isInfoEnabled()) { - LOGGER.info("[" + this.name + "] core reload success"); - } - } - - private boolean isShardedServer() { - return this.slaveSolrServers.length > 1; - } - - private HttpSolrServer selectRandomServer(HttpSolrServer[] shards) { - if (null == shards || shards.length == 0) { - throw new IllegalArgumentException("no shards"); - } - if (shards.length == 1) { - LOGGER.info("Choice:" + shards[0].getHost()); - return shards[0]; - } - HttpSolrServer server = shards[RandomUtils.nextInt(shards.length)]; - LOGGER.info("Choice:" + server.getHost()); - return server; - } - - private String buildShardsParam(HttpSolrServer[] solrServers) { - StringBuilder ret = new StringBuilder(); - for (int i = 0; i < solrServers.length; i++) { - ret.append(solrServers[i].getHost()).append('/').append(this.name); - if (i < solrServers.length - 1) { - ret.append(','); - } - } - LOGGER.info("slaveAddress:" + ret.toString()); - return ret.toString(); - } - - public QueryResponse query(SolrQuery solrQuery, boolean fromMaster, boolean bulk) throws SolrException { - if (isShardedServer() && !bulk && solrQuery.getStart() > MAX_NONE_BULK_SKIP) { - throw new SolrException("[" + this.name + "] skip of none bulk query must not greater than " + MAX_NONE_BULK_SKIP - + ", try set $bulk to true in the query !"); - } - return queryWithRetry(solrQuery, fromMaster, bulk, 0); - } - - private QueryResponse queryWithRetry(SolrQuery solrQuery, boolean fromMaster, boolean bulk, int retryCount) - throws SolrException { - // shards params - LOGGER.info("isShardedServer:"+isShardedServer()); - LOGGER.info("fromMaster:"+fromMaster); - if (isShardedServer()) { - if (fromMaster) { - solrQuery.set(ShardParams.SHARDS, buildShardsParam(this.masterSolrServers)); - } else { - solrQuery.set(ShardParams.SHARDS, buildShardsParam(this.slaveSolrServers)); - } - // bulk query - if (bulk) { - int numberShards = this.masterSolrServers.length; - solrQuery.set(ShardParams.SHARDS_START, solrQuery.getStart() / numberShards); - solrQuery.set(ShardParams.SHARDS_ROWS, Math.max(solrQuery.getRows() / numberShards, 1)); - solrQuery.setStart(0); - } - } - LOGGER.info("solrQuery:"+solrQuery.toString()); - QueryResponse queryResponse = null; - HttpSolrServer httpSolrServer = null; - try { - if (fromMaster) { - httpSolrServer = selectRandomServer(this.masterSolrServers); - try { - queryResponse = httpSolrServer.query(solrQuery); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } else { - httpSolrServer = selectRandomServer(this.slaveSolrServers); - try { - queryResponse = httpSolrServer.query(solrQuery); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - } catch (SolrServerException ex) { - if (ex.getCause() instanceof org.apache.solr.common.SolrException) { - org.apache.solr.common.SolrException e = (org.apache.solr.common.SolrException) ex.getCause(); - if (retryCount < SOLR_QUERY_MAX_RETRY_COUNT && ErrorCode.getErrorCode(e.code()) == ErrorCode.SERVER_ERROR) { - retryCount++; - if (LOGGER.isErrorEnabled()) { - LOGGER.error("#[" + this.name + "]# error query solr[" + httpSolrServer.getHost() + "], code:" - + ErrorCode.SERVER_ERROR + "(" + ErrorCode.SERVER_ERROR.code + "), retry for the " + retryCount - + "th time..."); - } - return queryWithRetry(solrQuery, fromMaster, bulk, retryCount); - } else { - throw new SolrException("=[" + this.name + "]= error query solr[" + httpSolrServer.getHost() + "]:", e); - } - } - throw new SolrException("*[" + this.name + "]* error query solr:", ex); - } - return queryResponse; - } - - public SolrSchema getSolrSchema() { - return solrSchema; - } - +public class SolrServerImpl implements SolrServer +{ + private static final Logger LOGGER; + private static final int SOLR_COMMIT_MAX_RETRY_COUNT = 3; + private static final int SOLR_QUERY_MAX_RETRY_COUNT = 3; + private static final int MAX_NONE_BULK_SKIP = 100000; + private final String name; + private final StoreConfig storeConfig; + private final HttpClient httpClient; + private int masterCommitSolrServerCount; + private final Map allSolrServers; + private String[] masterCommitFilters; + private HttpSolrServer[][] masterCommitSolrServers; + private HttpSolrServer[] masterSolrServers; + private HttpSolrServer[] slaveSolrServers; + private SolrSchema[] solrSchemas; + private final SolrSchema solrSchema; + private final Executor solrAddExecutorService; + private final AtomicBoolean filtering; + private long lastFilterTime; + + public SolrServerImpl(final StoreConfig storeConfig) throws ConfigurationException { + this.filtering = new AtomicBoolean(false); + this.lastFilterTime = 0L; + this.storeConfig = storeConfig; + this.name = storeConfig.getName(); + final PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); + cm.setMaxTotal(50); + this.httpClient = (HttpClient)new DefaultHttpClient(); + this.allSolrServers = this.initSolrServers(this.storeConfig, this.httpClient); + this.solrSchema = new SolrSchemaImpl(new File(storeConfig.getMasterSolrConfig().getHome() + "/" + this.name + "/conf/managed-schema")); + final int size = (this.masterCommitSolrServerCount + 1) * 2; + this.solrAddExecutorService = (Executor)new BlockingThreadPoolExecutor(this.name + "-SolrAdd", 1, size, size, 60000L); + } + + private CommonsHttpSolrServer createSolrServer(final String coreName, final SolrConfig solronfig) throws ConfigurationException { + try { + final CommonsHttpSolrServer solrServer = new CommonsHttpSolrServer(solronfig.getHost(), coreName, this.httpClient); + return solrServer; + } + catch (MalformedURLException e) { + throw new ConfigurationException("[" + this.name + "] error create solr server: ", e); + } + } + + private HttpSolrServer[] initSolrServer(final Map allSolrServers, final SolrConfig solrConfig) throws ConfigurationException { + SolrConfig[] shardsConfigs = solrConfig.getShardsConfigs(); + if (shardsConfigs.length == 0) { + shardsConfigs = new SolrConfig[] { solrConfig }; + } + final HttpSolrServer[] shards = new HttpSolrServer[shardsConfigs.length]; + for (int i = 0; i < shardsConfigs.length; ++i) { + final SolrConfig shardConfig = shardsConfigs[i]; + if (!allSolrServers.containsKey(shardConfig.getId())) { + final SolrConfig[] slaveConfigs = shardConfig.getSlaveConfigs(); + if (slaveConfigs.length == 0) { + allSolrServers.put(shardConfig.getId(), this.createSolrServer(this.name, shardConfig)); + } + else { + final CommonsHttpSolrServer[] slaveServers = new CommonsHttpSolrServer[slaveConfigs.length]; + for (int j = 0; j < slaveConfigs.length; ++j) { + final SolrConfig slaveConfig = slaveConfigs[j]; + if (!allSolrServers.containsKey(slaveConfig.getId())) { + allSolrServers.put(slaveConfig.getId(), this.createSolrServer(this.name, slaveConfig)); + } + slaveServers[j] = (CommonsHttpSolrServer)allSolrServers.get(slaveConfig.getId()); + } + allSolrServers.put(shardConfig.getId(), new LBHttpSolrServer((HttpSolrServer[])slaveServers)); + } + } + shards[i] = allSolrServers.get(shardConfig.getId()); + } + return shards; + } + + private synchronized Map initSolrServers(final StoreConfig storeConfig, final HttpClient httpClient) throws ConfigurationException { + final Map solrServers = new HashMap(); + final SolrConfig[] masterConfigs = storeConfig.getMasterSolrConfigs(); + this.masterCommitSolrServers = new HttpSolrServer[masterConfigs.length][]; + this.masterCommitFilters = new String[masterConfigs.length]; + this.solrSchemas = new SolrSchema[masterConfigs.length]; + this.masterCommitSolrServerCount = 0; + for (int i = 0; i < masterConfigs.length; ++i) { + this.masterCommitSolrServers[i] = this.initSolrServer(solrServers, masterConfigs[i]); + this.masterCommitFilters[i] = masterConfigs[i].getFilter(); + this.solrSchemas[i] = new SolrSchemaImpl(new File(masterConfigs[i].getHome() + "/" + this.name + "/conf/managed-schema")); + this.masterCommitSolrServerCount += this.masterCommitSolrServers[i].length; + } + this.masterSolrServers = this.initSolrServer(solrServers, storeConfig.getMasterSolrConfig()); + this.slaveSolrServers = this.initSolrServer(solrServers, storeConfig.getSlaveSolrConfig()); + return solrServers; + } + + private void addToSolr0(final HttpSolrServer solrServer, final List documents, int errorCount) { + try { + if (null != documents && !documents.isEmpty()) { + solrServer.add((Collection)documents); + solrServer.commit(); + } + } + catch (Exception e) { + if (errorCount <= 3) { + final long sleepTime = ++errorCount * errorCount; + if (SolrServerImpl.LOGGER.isInfoEnabled()) { + SolrServerImpl.LOGGER.info("[" + this.name + "] error commit to solr[" + solrServer.getHost() + "], sleep " + sleepTime + "s and retry(" + e.getMessage() + ")"); + } + try { + Thread.sleep(1000L * sleepTime); + } + catch (InterruptedException e2) { + SolrServerImpl.LOGGER.error("[" + this.name + "] error save to solr: ", (Throwable)e2); + } + this.addToSolr0(solrServer, documents, errorCount); + } + else if (SolrServerImpl.LOGGER.isErrorEnabled()) { + SolrServerImpl.LOGGER.error("[" + this.name + "] error save to solr[" + solrServer.getHost() + "] after " + 3 + " times retry: ", (Throwable)e); + } + } + } + + @Override + public void add(final List dbObjects) throws SolrException { + if (null == dbObjects || dbObjects.size() == 0) { + return; + } + try { + long time = System.currentTimeMillis(); + this.checkFilter(); + final CountDownLatch countDownLatch = new CountDownLatch(this.masterCommitSolrServerCount); + for (int j = 0; j < this.masterCommitSolrServers.length; ++j) { + final int jndex = j; + final Map> shardedObjectMap = this.shard(dbObjects, this.masterCommitSolrServers[j].length); + for (int i = 0; i < this.masterCommitSolrServers[j].length; ++i) { + final int index = i; + this.solrAddExecutorService.execute(new Runnable() { + @Override + public void run() { + try { + final List sharedObjects = shardedObjectMap.get(index); + if (null == sharedObjects || sharedObjects.size() == 0) { + return; + } + final List documents = new ArrayList(sharedObjects.size()); + for (final BSONObject obj : sharedObjects) { + final SolrInputDocument document = SolrServerImpl.this.toSolrInputDocument(obj, SolrServerImpl.this.solrSchemas[jndex]); + if (!document.isEmpty()) { + document.addField("_version", (Object)Core.VERSION_DATE_FORMAT.format(new Date())); + System.out.println("document:" + document.toString()); + documents.add(document); + } + else { + if (!SolrServerImpl.LOGGER.isDebugEnabled()) { + continue; + } + SolrServerImpl.LOGGER.debug("solr input document is empty:" + document); + } + } + SolrServerImpl.this.addToSolr0(SolrServerImpl.this.masterCommitSolrServers[jndex][index], documents, 0); + } + catch (Exception e) { + if (SolrServerImpl.LOGGER.isErrorEnabled()) { + SolrServerImpl.LOGGER.error("[" + SolrServerImpl.this.name + "] error save to solr: ", (Throwable)e); + } + } + finally { + countDownLatch.countDown(); + } + } + }); + } + } + countDownLatch.await(); + time = System.currentTimeMillis() - time; + if (SolrServerImpl.LOGGER.isInfoEnabled()) { + SolrServerImpl.LOGGER.info("[" + this.name + "] add {} documents to solr, {}ms, {}/s.", new Object[] { dbObjects.size(), time, dbObjects.size() * 1000 / time }); + } + } + catch (Exception e) { + if (SolrServerImpl.LOGGER.isErrorEnabled()) { + SolrServerImpl.LOGGER.error("[" + this.name + "] error save to solr: ", (Throwable)e); + } + } + } + + private SolrInputDocument toSolrInputDocument(final BSONObject doc, final SolrSchema solrSchema) { + final Collection indexes = solrSchema.getIndexes(); + final SolrInputDocument document = new SolrInputDocument(); + Map boostsMap = null; + if (doc.get("$boosts") != null && doc.get("$boosts") instanceof Map) { + boostsMap = (Map)doc.get("$boosts"); + if (boostsMap.containsKey("$doc_boost")) { + document.setDocumentBoost(boostsMap.get("$doc_boost").floatValue()); + } + } + for (final Index index : indexes) { + final String key = index.getName(); + if (doc.containsField(key)) { + final Object value = doc.get(key); + final SolrInputField field = new SolrInputField(key); + if (value instanceof ObjectId) { + field.setValue((Object)value.toString(), 1.0f); + } + else if (boostsMap != null && boostsMap.containsKey(key)) { + field.setValue(value, boostsMap.get(key).floatValue()); + } + else { + field.setValue(value, 1.0f); + } + document.put(key, field); + } + } + return document; + } + + private Map> shard(final List dbObjects, final int shards) { + if (CollectionUtils.isEmpty((Collection)dbObjects)) { + return Collections.>emptyMap(); + } + final Map> ret = new HashMap>(); + for (final BSONObject object : dbObjects) { + final int hashMod = Math.abs(object.get("_id").hashCode()) % shards; + List modObjects = ret.get(hashMod); + if (null == modObjects) { + modObjects = new ArrayList(); + ret.put(hashMod, modObjects); + } + modObjects.add(object); + } + return ret; + } + + private void checkFilter() { + if (System.currentTimeMillis() - this.lastFilterTime < 300000L) { + return; + } + try { + if (this.filtering.compareAndSet(false, true)) { + final long start = System.currentTimeMillis(); + for (int i = 0; i < this.masterCommitFilters.length; ++i) { + if (!StringUtils.isEmpty(this.masterCommitFilters[i])) { + for (final HttpSolrServer server : this.masterCommitSolrServers[i]) { + try { + server.deleteByQuery(this.masterCommitFilters[i]); + } + catch (SolrServerException e) { + if (SolrServerImpl.LOGGER.isErrorEnabled()) { + SolrServerImpl.LOGGER.error("[" + this.name + "] error execute filter delete: ", (Throwable)e); + } + } + catch (IOException e2) { + if (SolrServerImpl.LOGGER.isErrorEnabled()) { + SolrServerImpl.LOGGER.error("[" + this.name + "] error execute filter delete: ", (Throwable)e2); + } + } + } + } + } + if (SolrServerImpl.LOGGER.isInfoEnabled()) { + SolrServerImpl.LOGGER.info("[" + this.name + "] execute filter delete, " + (System.currentTimeMillis() - start) / 1000L + "s."); + } + this.lastFilterTime = System.currentTimeMillis(); + } + } + catch (Exception e3) { + if (SolrServerImpl.LOGGER.isErrorEnabled()) { + SolrServerImpl.LOGGER.error("[" + this.name + "] error execute filter delete: ", (Throwable)e3); + } + } + finally { + this.filtering.compareAndSet(true, false); + } + } + + @Override + public void deleteByQuery(final String query) throws SolrException { + try { + for (final HttpSolrServer[] array : this.masterCommitSolrServers) { + final HttpSolrServer[] servers = array; + for (final HttpSolrServer server : array) { + server.deleteByQuery(query); + } + } + } + catch (SolrServerException e) { + throw new SolrException("[" + this.name + "] error execute delete[" + query + "]: ", (Throwable)e); + } + catch (IOException e2) { + throw new SolrException("[" + this.name + "] error execute delete[" + query + "]: ", e2); + } + } + + @Override + public synchronized void reload() throws ConfigurationException { + this.allSolrServers.clear(); + this.allSolrServers.putAll(this.initSolrServers(this.storeConfig, this.httpClient)); + final String schema = this.storeConfig.getMasterSolrConfig().getHome() + "/" + this.name + "/conf/managed-schema"; + this.solrSchema.setSchemaFile(new File(schema)); + this.solrSchema.reload(); + } + + @Override + public void reloadCores() throws SolrException { + if (SolrServerImpl.LOGGER.isInfoEnabled()) { + SolrServerImpl.LOGGER.info("[" + this.name + "] reload core..."); + } + for (final String id : this.allSolrServers.keySet()) { + final HttpSolrServer solrServer = this.allSolrServers.get(id); + solrServer.reloadCore(); + } + if (SolrServerImpl.LOGGER.isInfoEnabled()) { + SolrServerImpl.LOGGER.info("[" + this.name + "] core reload success"); + } + } + + private boolean isShardedServer() { + return this.slaveSolrServers.length > 1; + } + + private HttpSolrServer selectRandomServer(final HttpSolrServer[] shards) { + if (null == shards || shards.length == 0) { + throw new IllegalArgumentException("no shards"); + } + if (shards.length == 1) { + SolrServerImpl.LOGGER.info("Choice:" + shards[0].getHost()); + return shards[0]; + } + final HttpSolrServer server = shards[RandomUtils.nextInt(shards.length)]; + SolrServerImpl.LOGGER.info("Choice:" + server.getHost()); + return server; + } + + private String buildShardsParam(final HttpSolrServer[] solrServers) { + final StringBuilder ret = new StringBuilder(); + for (int i = 0; i < solrServers.length; ++i) { + ret.append(solrServers[i].getHost()).append('/').append(this.name); + if (i < solrServers.length - 1) { + ret.append(','); + } + } + SolrServerImpl.LOGGER.info("slaveAddress:" + ret.toString()); + return ret.toString(); + } + + @Override + public QueryResponse query(final SolrQuery solrQuery, final boolean fromMaster, final boolean bulk) throws SolrException { + if (this.isShardedServer() && !bulk && solrQuery.getStart() > 100000) { + throw new SolrException("[" + this.name + "] skip of none bulk query must not greater than " + 100000 + ", try set $bulk to true in the query !"); + } + return this.queryWithRetry(solrQuery, fromMaster, bulk, 0); + } + + private QueryResponse queryWithRetry(final SolrQuery solrQuery, final boolean fromMaster, final boolean bulk, int retryCount) throws SolrException { + SolrServerImpl.LOGGER.info("isShardedServer:" + this.isShardedServer()); + SolrServerImpl.LOGGER.info("fromMaster:" + fromMaster); + if (this.isShardedServer()) { + if (fromMaster) { + solrQuery.set("shards", this.buildShardsParam(this.masterSolrServers)); + } + else { + solrQuery.set("shards", this.buildShardsParam(this.slaveSolrServers)); + } + if (bulk) { + final int numberShards = this.masterSolrServers.length; + solrQuery.set("shards.start", solrQuery.getStart() / numberShards); + solrQuery.set("shards.rows", Math.max(solrQuery.getRows() / numberShards, 1)); + solrQuery.setStart(0); + } + } + SolrServerImpl.LOGGER.info("solrQuery:" + solrQuery.toString()); + QueryResponse queryResponse = null; + HttpSolrServer httpSolrServer = null; + try { + if (fromMaster) { + httpSolrServer = this.selectRandomServer(this.masterSolrServers); + try { + queryResponse = httpSolrServer.query(solrQuery); + } + catch (IOException e) { + e.printStackTrace(); + } + } + else { + httpSolrServer = this.selectRandomServer(this.slaveSolrServers); + try { + queryResponse = httpSolrServer.query(solrQuery); + } + catch (IOException e) { + e.printStackTrace(); + } + } + solrQuery.add("__host", httpSolrServer.getHost()); + } + catch (SolrServerException ex) { + if (!(ex.getCause() instanceof org.apache.solr.common.SolrException)) { + throw new SolrException("*[" + this.name + "]* error query solr:", (Throwable)ex); + } + final org.apache.solr.common.SolrException e2 = (org.apache.solr.common.SolrException)ex.getCause(); + if (retryCount < 3 && org.apache.solr.common.SolrException.ErrorCode.getErrorCode(e2.code()) == org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR) { + ++retryCount; + if (SolrServerImpl.LOGGER.isErrorEnabled()) { + SolrServerImpl.LOGGER.error("#[" + this.name + "]# error query solr[" + httpSolrServer.getHost() + "], code:" + org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR + "(" + org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR.code + "), retry for the " + retryCount + "th time..."); + } + return this.queryWithRetry(solrQuery, fromMaster, bulk, retryCount); + } + throw new SolrException("=[" + this.name + "]= error query solr[" + httpSolrServer.getHost() + "]:", (Throwable)e2); + } + return queryResponse; + } + + @Override + public SolrSchema getSolrSchema() { + return this.solrSchema; + } + + static { + LOGGER = LoggerFactory.getLogger((Class)SolrServerImpl.class); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/BSONUtils.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/BSONUtils.java index 16e064815..5a1627761 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/BSONUtils.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/BSONUtils.java @@ -1,53 +1,34 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.utils; import java.io.IOException; -public class BSONUtils { - - public static long bitSet(long holder, int i, boolean set) { - if (i < 0 || i > 63) { - throw new IllegalArgumentException("i should from 0 inclusive and 64 exclusive"); - } - if (set) { - return holder | (1 << i); - } else { - int ii = 1 << i; - return (holder | ii) ^ ii; - } - } - - public static boolean isBitSet(long holder, int i) { - if (i < 0 || i > 63) { - throw new IllegalArgumentException("i should from 0 inclusive and 64 exclusive"); - } - return (holder & (1 << i)) > 0; - } - - public static int stringByteLength(String s) throws IOException { - byte[] bs = s.getBytes("UTF-8"); - return bs.length + 5; - } - - public static int cstringByteLength(String s) throws IOException { - byte[] bs = s.getBytes("UTF-8"); - return bs.length + 1; - } - +public class BSONUtils +{ + public static long bitSet(final long holder, final int i, final boolean set) { + if (i < 0 || i > 63) { + throw new IllegalArgumentException("i should from 0 inclusive and 64 exclusive"); + } + if (set) { + return holder | (long)(1 << i); + } + final int ii = 1 << i; + return (holder | (long)ii) ^ (long)ii; + } + + public static boolean isBitSet(final long holder, final int i) { + if (i < 0 || i > 63) { + throw new IllegalArgumentException("i should from 0 inclusive and 64 exclusive"); + } + return (holder & (long)(1 << i)) > 0L; + } + + public static int stringByteLength(final String s) throws IOException { + final byte[] bs = s.getBytes("UTF-8"); + return bs.length + 5; + } + + public static int cstringByteLength(final String s) throws IOException { + final byte[] bs = s.getBytes("UTF-8"); + return bs.length + 1; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/HexDump.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/HexDump.java index 3144e1ea1..dcf4e1dd0 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/HexDump.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/HexDump.java @@ -1,275 +1,106 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.utils; import java.io.PrintStream; -/** - * Hex dump class. - */ -public final class HexDump { - - public static final void dump(byte[] byt) { - dump(byt, byt.length, 0, System.out); - } - - /** - * Hex dump a byte array - * - * @param byt - * Byte array to dump - * @param len - * Length of data to dump - * @param offset - * Offset to start data dump - */ - public static final void dump(byte[] byt, int len, int offset) { - dump(byt, len, offset, System.out); - } - - /** - * Hex dump a byte array - * - * @param byt - * Byte array to dump - * @param len - * Length of data to dump - * @param offset - * Offset to start data dump - * @param stream - * Output stream to dump the output to. - */ - - public static final void dump(byte[] byt, int len, int offset, PrintStream stream) { - - // Create buffers for the ASCII and Hex output - - StringBuffer ascBuf = new StringBuffer(); - StringBuffer hexBuf = new StringBuffer(); - - // dump 16 byte blocks from the array until the length has been - // reached - - int dlen = 0; - int doff = offset; - String posStr = null; - - while (dlen < len) { - - // Reset the ASCII/Hex buffers - - ascBuf.setLength(0); - hexBuf.setLength(0); - - posStr = generatePositionString(doff); - - // dump a block of data, update the data offset - - doff = generateLine(byt, doff, ascBuf, hexBuf); - - // Output the current record - - stream.print(posStr); - stream.print(hexBuf.toString()); - stream.println(ascBuf.toString()); - - // Update the dump length - - dlen += 16; - } - } - - /** - * Generate a hex string for the specified string - * - * @param str - * String - * @return String - */ - public static final String hexString(String str) { - if (str != null) - return hexString(str.getBytes()); - return ""; - } - - /** - * Generate a hex string for the specified string - * - * @param str - * String - * @param gap - * String - * @return String - */ - public static final String hexString(String str, String gap) { - if (str != null) - return hexString(str.getBytes(), gap); - return ""; - } - - /** - * Generate a hex string for the specified bytes - * - * @param buf - * byte[] - * @return String - */ - public static final String hexString(byte[] buf) { - return hexString(buf, buf.length, null); - } - - /** - * Generate a hex string for the specified bytes - * - * @param buf - * byte[] - * @param gap - * String - * @return String - */ - public static final String hexString(byte[] buf, String gap) { - return hexString(buf, buf.length, gap); - } - - /** - * Generate a hex string for the specified bytes - * - * @param buf - * byte[] - * @param len - * int - * @param gap - * String - * @return String - */ - public static final String hexString(byte[] buf, int len, String gap) { - - // Check if the buffer is valid - - if (buf == null) - return ""; - - // Create a string buffer for the hex string - - int buflen = buf.length * 2; - if (gap != null) - buflen += buf.length * gap.length(); - - StringBuffer hex = new StringBuffer(buflen); - - // Convert the bytes to hex-ASCII - - for (int i = 0; i < len; i++) { - - // Get the current byte - - int curbyt = (int) (buf[i] & 0x00FF); - - // Output the hex string - - hex.append(Integer.toHexString((curbyt & 0xF0) >> 4)); - hex.append(Integer.toHexString(curbyt & 0x0F)); - - // Add the gap string, if specified - - if (gap != null && i < (len - 1)) - hex.append(gap); - } - - // Return the hex-ASCII string - - return hex.toString(); - } - - /** - * Generate a buffer position string - * - * @param off - * int - * @return String - */ - private static final String generatePositionString(int off) { - - // Create a buffer position string - - StringBuffer posStr = new StringBuffer("" + off + ": "); - while (posStr.length() < 10) - posStr.insert(0, " "); - - // Return the string - - return posStr.toString(); - } - - /** - * Output a single line of the hex dump to a debug device - * - * @param byt - * Byte array to dump - * @param off - * Offset to start data dump - * @param ascBuf - * Buffer for ASCII output - * @param hexBuf - * Buffer for Hex output - * @return New offset value - */ - - private static final int generateLine(byte[] byt, int off, StringBuffer ascBuf, StringBuffer hexBuf) { - - // Check if there is enough buffer space to dump 16 bytes - - int dumplen = byt.length - off; - if (dumplen > 16) - dumplen = 16; - - // dump a 16 byte block of data - - for (int i = 0; i < dumplen; i++) { - - // Get the current byte - - int curbyt = (int) (byt[off++] & 0x00FF); - - // Output the hex string - - hexBuf.append(Integer.toHexString((curbyt & 0xF0) >> 4)); - hexBuf.append(Integer.toHexString(curbyt & 0x0F)); - hexBuf.append(" "); - - // Output the character equivalent, if printable - - if (Character.isLetterOrDigit((char) curbyt) || Character.getType((char) curbyt) != Character.CONTROL) - ascBuf.append((char) curbyt); - else - ascBuf.append("."); - } - - for (int i = dumplen; i < 16; i++) { - hexBuf.append(" "); - } - - // Output the hex dump line - - hexBuf.append(" "); - - // Return the new data offset - - return off; - } +public final class HexDump +{ + public static final void dump(final byte[] byt) { + dump(byt, byt.length, 0, System.out); + } + + public static final void dump(final byte[] byt, final int len, final int offset) { + dump(byt, len, offset, System.out); + } + + public static final void dump(final byte[] byt, final int len, final int offset, final PrintStream stream) { + final StringBuffer ascBuf = new StringBuffer(); + final StringBuffer hexBuf = new StringBuffer(); + int dlen = 0; + int doff = offset; + String posStr = null; + while (dlen < len) { + ascBuf.setLength(0); + hexBuf.setLength(0); + posStr = generatePositionString(doff); + doff = generateLine(byt, doff, ascBuf, hexBuf); + stream.print(posStr); + stream.print(hexBuf.toString()); + stream.println(ascBuf.toString()); + dlen += 16; + } + } + + public static final String hexString(final String str) { + if (str != null) { + return hexString(str.getBytes()); + } + return ""; + } + + public static final String hexString(final String str, final String gap) { + if (str != null) { + return hexString(str.getBytes(), gap); + } + return ""; + } + + public static final String hexString(final byte[] buf) { + return hexString(buf, buf.length, null); + } + + public static final String hexString(final byte[] buf, final String gap) { + return hexString(buf, buf.length, gap); + } + + public static final String hexString(final byte[] buf, final int len, final String gap) { + if (buf == null) { + return ""; + } + int buflen = buf.length * 2; + if (gap != null) { + buflen += buf.length * gap.length(); + } + final StringBuffer hex = new StringBuffer(buflen); + for (int i = 0; i < len; ++i) { + final int curbyt = buf[i] & 0xFF; + hex.append(Integer.toHexString((curbyt & 0xF0) >> 4)); + hex.append(Integer.toHexString(curbyt & 0xF)); + if (gap != null && i < len - 1) { + hex.append(gap); + } + } + return hex.toString(); + } + + private static final String generatePositionString(final int off) { + final StringBuffer posStr = new StringBuffer("" + off + ": "); + while (posStr.length() < 10) { + posStr.insert(0, " "); + } + return posStr.toString(); + } + + private static final int generateLine(final byte[] byt, int off, final StringBuffer ascBuf, final StringBuffer hexBuf) { + int dumplen = byt.length - off; + if (dumplen > 16) { + dumplen = 16; + } + for (int i = 0; i < dumplen; ++i) { + final int curbyt = byt[off++] & 0xFF; + hexBuf.append(Integer.toHexString((curbyt & 0xF0) >> 4)); + hexBuf.append(Integer.toHexString(curbyt & 0xF)); + hexBuf.append(" "); + if (Character.isLetterOrDigit((char)curbyt) || Character.getType((char)curbyt) != 15) { + ascBuf.append((char)curbyt); + } + else { + ascBuf.append("."); + } + } + for (int i = dumplen; i < 16; ++i) { + hexBuf.append(" "); + } + hexBuf.append(" "); + return off; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/LoggerFactory.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/LoggerFactory.java index 8031389b2..41f60064f 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/LoggerFactory.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/LoggerFactory.java @@ -2,17 +2,17 @@ package com.fangjia.datastore.utils; import org.slf4j.Logger; -public class LoggerFactory { - - public static Logger getLogger(Class clazz) { - return org.slf4j.LoggerFactory.getLogger(clazz); - } - - public static Logger getLogger(String name) { - return org.slf4j.LoggerFactory.getLogger(name); - } - - public static Logger getPerformanceLogger(Class clazz) { - return org.slf4j.LoggerFactory.getLogger("performance." + clazz.getName()); - } +public class LoggerFactory +{ + public static Logger getLogger(final Class clazz) { + return org.slf4j.LoggerFactory.getLogger((Class)clazz); + } + + public static Logger getLogger(final String name) { + return org.slf4j.LoggerFactory.getLogger(name); + } + + public static Logger getPerformanceLogger(final Class clazz) { + return org.slf4j.LoggerFactory.getLogger("performance." + clazz.getName()); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/MongoUtils.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/MongoUtils.java index 1bc84fd61..e90fda2a0 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/MongoUtils.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/MongoUtils.java @@ -2,22 +2,23 @@ package com.fangjia.datastore.utils; import org.apache.commons.lang.StringUtils; -public class MongoUtils { - private MongoUtils() { - } - - public static String[] parseFullCollectionName(String fullCollectionName) { - if (StringUtils.isEmpty(fullCollectionName)) { - return null; - } - int i = fullCollectionName.indexOf('.'); - if (i <= 0 || i >= fullCollectionName.length() - 1) { - return null; - } - return new String[] { fullCollectionName.substring(0, i), fullCollectionName.substring(i + 1) }; - } - - public static void main(String[] args) { - System.out.println(parseFullCollectionName(".q")); - } +public class MongoUtils +{ + private MongoUtils() { + } + + public static String[] parseFullCollectionName(final String fullCollectionName) { + if (StringUtils.isEmpty(fullCollectionName)) { + return null; + } + final int i = fullCollectionName.indexOf(46); + if (i <= 0 || i >= fullCollectionName.length() - 1) { + return null; + } + return new String[] { fullCollectionName.substring(0, i), fullCollectionName.substring(i + 1) }; + } + + public static void main(final String[] args) { + System.out.println(parseFullCollectionName(".q")); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/XmlUtils.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/XmlUtils.java index 26246685b..72363e986 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/XmlUtils.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/XmlUtils.java @@ -2,45 +2,42 @@ package com.fangjia.datastore.utils; import org.dom4j.Node; -public class XmlUtils { - public static String getNodeValue(Node parentNode, String nodeName, String defaultValue) { - Node node = parentNode.selectSingleNode(nodeName); - String ret = defaultValue; - if (null == node) { - return ret; - } else { - ret = node.getText(); - if (null == ret) { - ret = defaultValue; - } - } - return ret; - } - - public static int getNodeValueAsInt(Node parentNode, String nodeName, int defaultValue) { - String value = getNodeValue(parentNode, nodeName, null); - if (null == value) { - return defaultValue; - } else { - return Integer.parseInt(value); - } - } - - public static long getNodeValueAsLong(Node parentNode, String nodeName, long defaultValue) { - String value = getNodeValue(parentNode, nodeName, null); - if (null == value) { - return defaultValue; - } else { - return Long.parseLong(value); - } - } - - public static boolean getNodeValueAsBoolean(Node parentNode, String nodeName, boolean defaultValue) { - String value = getNodeValue(parentNode, nodeName, null); - if (null == value) { - return defaultValue; - } else { - return Boolean.parseBoolean(value); - } - } +public class XmlUtils +{ + public static String getNodeValue(final Node parentNode, final String nodeName, final String defaultValue) { + final Node node = parentNode.selectSingleNode(nodeName); + String ret = defaultValue; + if (null == node) { + return ret; + } + ret = node.getText(); + if (null == ret) { + ret = defaultValue; + } + return ret; + } + + public static int getNodeValueAsInt(final Node parentNode, final String nodeName, final int defaultValue) { + final String value = getNodeValue(parentNode, nodeName, null); + if (null == value) { + return defaultValue; + } + return Integer.parseInt(value); + } + + public static long getNodeValueAsLong(final Node parentNode, final String nodeName, final long defaultValue) { + final String value = getNodeValue(parentNode, nodeName, null); + if (null == value) { + return defaultValue; + } + return Long.parseLong(value); + } + + public static boolean getNodeValueAsBoolean(final Node parentNode, final String nodeName, final boolean defaultValue) { + final String value = getNodeValue(parentNode, nodeName, null); + if (null == value) { + return defaultValue; + } + return Boolean.parseBoolean(value); + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index 272979fca..587b3888d 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -918,6 +918,19 @@ public class WeiXinEvaluationController { return jsonMmodel; } + public Object saveBhLoanAction(EvaluationModel model){ + JsonModel jsonMmodel = new JsonModel(); + if(StringUtils.isNotBlank(model.getEvaluationId())){ + FjtEvaluation fjtEvaluation = fjtEvaluationService.findEvaluationById(Long.valueOf(model.getEvaluationId())); + if(fjtEvaluation != null && fjtEvaluation.getLoan()!=model.getLoan()){ + fjtEvaluation.setLoan(model.getLoan()); + fjtEvaluation.setLastUpdated(new Date()); + fjtEvaluationService.updateEvaluation(fjtEvaluation); + } + } + return jsonMmodel; + } + //刷新审核价 public Object refreshAuditPriceAction(EvaluationModel model){ JsonModel jsonMmodel = new JsonModel(); @@ -1106,9 +1119,15 @@ public class WeiXinEvaluationController { public Object checkImageAction(EvaluationModel model){ JsonModel jsonModel = new JsonModel(); if (StringUtils.isNotBlank(model.getEvaluationId())){ - List datas = reportFileAttachService.findHouseImage(Integer.valueOf(model.getEvaluationId()), 3); + List datas3 = reportFileAttachService.findHouseImage(Integer.valueOf(model.getEvaluationId()), 3); Map data = new HashMap(); - data.put("count", datas.size()); + data.put("count_3", datas3.size()); + + FjtEvaluation evaluation = fjtEvaluationService.findEvaluationById(Long.valueOf(model.getEvaluationId())); + if(evaluation!=null && evaluation.getLoan()!=null && evaluation.getLoan()==1){ + List datas5 = reportFileAttachService.findHouseImage(Integer.valueOf(model.getEvaluationId()), 5); + data.put("count_5", datas5.size()); + } jsonModel.setJsonModel(data); } return jsonModel; @@ -1287,7 +1306,7 @@ public class WeiXinEvaluationController { FjtUsers fjtUser = fjtUsersService.findById(Long.valueOf(userId)); UsersDetail usersDetail = usersDetailService.findById(fjtUser.getId()); try { - String id = new ObjectId().toString(); + List uploadfiles = new ArrayList(); for (CommonsMultipartFile file : files) { if (file.getSize() == 0 ) { @@ -1319,6 +1338,8 @@ public class WeiXinEvaluationController { } catch (IOException e) { //处理异常信息 } + String id = new ObjectId().toString(); + reportFileAttach = new ReportFileAttach(); reportFileAttach.setEvaluationId(Integer.valueOf(model.getEvaluationId())); reportFileAttach.setQueryId(model.getQueryId()); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/weixin/EvaluationModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/weixin/EvaluationModel.java index 8de0977f8..20f093053 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/weixin/EvaluationModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/weixin/EvaluationModel.java @@ -45,6 +45,7 @@ public class EvaluationModel extends BaseModel{ private Map loanInfoMap; private Integer hopePrice; private String remark; + private Integer loan; // private String preCode;//渤海接口--楼盘编码 // private String preName;//渤海接口--楼盘名称 // private String countryCode;//渤海接口--区域编码 @@ -317,4 +318,12 @@ public class EvaluationModel extends BaseModel{ public void setAddress(String address) { this.address = address; } + + public Integer getLoan() { + return loan; + } + + public void setLoan(Integer loan) { + this.loan = loan; + } } diff --git a/projects/fangjia-open/src/main/webapp/static/js/report.js b/projects/fangjia-open/src/main/webapp/static/js/report.js index 92ef631b5..b61a6e2d1 100644 --- a/projects/fangjia-open/src/main/webapp/static/js/report.js +++ b/projects/fangjia-open/src/main/webapp/static/js/report.js @@ -190,6 +190,36 @@ $('#submitRemark').on('click', function () { } }); +$('#submitLoan').on('click', function () { + var opt = $("#submitLoan").text(); + if(opt == "淇敼"){ + $(this).parent().find("span").eq(1).hide() + $("input[name=loan]").removeClass("dn"); + $("#submitLoan").text("淇濆瓨"); + }else{ + var loan = $("input[name=loan]").val(); + if(loan == ""){ + $.toast("璇疯緭鍏ユ槸鍚︾画璐�"); + }else{ + $.ajax({ + url:"saveBhLoan", + data:{ + evaluationId:eId, + loan:loan + }, + dataType:"json", + success:function(data){ + $("#submitLoan").remove(); + $("input[name=loan]").remove(); + $("#refreshPrice0").show(); + location.href ="bhreport?evaluationId="+eId; + } + }) + } + } +}); + + $('#refreshPrice,#refreshPrice0').on('click', function () { $.ajax({ url:"refreshAuditPrice", @@ -301,21 +331,34 @@ $('.download_btn').on('click', function () { /* 涓嬭浇璇勪及鎶ュ憡 */ $('.formal_btn').on('click', function () { - $.ajax({ - url:"checkImage", - data:{ - evaluationId:eId - }, - dataType:"json", - success:function(data){ - if(data && data.count>0){ - clickFormalReport() - }else{ - $.alert('璇蜂笂浼犳埧灞嬬収鐗囷紒'); + if(isLoan!=null) { + $.ajax({ + url:"checkImage", + data:{ + evaluationId:eId + }, + dataType:"json", + success:function(data){ + if(isLoan==1){ + if(data.count_5==0){ + $.alert('璇蜂笂浼犱骇璋冩姤鍛婃垨鏉冪睄锛�'); + }else if(data.count_3==0){ + $.alert('璇蜂笂浼犳埧灞嬬収鐗囷紒'); + }else{ + clickFormalReport() + } + }else{ + if(data.count_3==0){ + $.alert('璇蜂笂浼犳埧灞嬬収鐗囷紒'); + }else{ + clickFormalReport() + } + } } - } - }) - + }) + }else{ + $.alert('璇烽�夋嫨鏄惁缁捶锛�'); + } }); function clickFormalReport(){ diff --git a/projects/fangjia-open/src/main/webapp/static/style/wx.css b/projects/fangjia-open/src/main/webapp/static/style/wx.css index 269a1a00e..0e85e8a93 100644 --- a/projects/fangjia-open/src/main/webapp/static/style/wx.css +++ b/projects/fangjia-open/src/main/webapp/static/style/wx.css @@ -134,7 +134,7 @@ input::-webkit-input-placeholder{ .inputTopic01{margin-bottom:-1px;} .inputTopic01 .topicTitle,.inputTopic02 .topicTitle{margin-right:0;border:none;line-height:2.5rem;} .inputTopic01 .topicTitle .input_price{width:20%;padding-left:5px;} -.inputTopic02 .topicTitle .input_lender{width:40%;padding-left:5px;} +.inputTopic02 .topicTitle .input_lender{width:39%;padding-left:5px;line-height: 28px;} .inputTopic02 .topicTitle .input_cardId{width:35%;padding-left:5px;} .submit_btn{width:90px;border-left:solid 1px #e7e7e7;float:right;padding-right:0;text-align:center;cursor:pointer;} .left01{float:left;margin-right:8%;} diff --git a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl index 165012ff6..f16637105 100644 --- a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl +++ b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl @@ -132,6 +132,18 @@ +
+ +
+ 娉細濡傚嚭鍏锋寮忔姤鍛婃椂锛岄渶涓婁紶浜ц皟鎶ュ憡鎴栨潈绫� +
+
+
@@ -139,7 +151,7 @@ <#if reportFileMap??>
宸蹭笂浼犳枃浠讹細 - <#if reportFileMap["1"]??>璇环鍗�(${reportFileMap["1"]?size}) + <#if reportFileMap["5"]??>浜ц皟鎶ュ憡鎴栨潈绫�(${reportFileMap["5"]?size}) <#if reportFileMap["2"]??> <#if reportFileMap["3"]??> <#if reportFileMap["4"]??> @@ -456,6 +468,10 @@ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/auto/CityInfoEmailSender.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/auto/CityInfoEmailSender.groovy index cb0a3f0f7..323ce415b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/auto/CityInfoEmailSender.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/auto/CityInfoEmailSender.groovy @@ -23,14 +23,7 @@ class CityInfoEmailSender extends BaseBatch{ private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files') def emails = [ - "wangsheng@fangjia.com" - ,"xuyejun@fangjia.com" - ,"tiki@fangjia.com" - ,"zhangzx@fangjia.com" - ,"panyingjiao@fangjia.com" - ,"yangqixing@fangjia.com" - ,"james.lin@fangjia.com" - ,"jhu@fangjia.com" + "gaojuan@fangjia.com" ] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy index d8437413e..aaf5bf416 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy @@ -16,6 +16,7 @@ import com.fangjia.script.util.ServerAreaMap import com.google.common.collect.MapDifference import com.google.common.collect.Maps import com.mongodb.* +import org.apache.commons.lang.StringUtils import org.bson.types.ObjectId /** @@ -47,7 +48,16 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch { } void process(Object args) { - def cities = ServerAreaMap.findAllCities() + String cityParam = System.getProperty("city"); + def cities = new ArrayList() + if(StringUtils.isNotBlank(cityParam)){ + cities = new ArrayList() + for(String c in cityParam.split(",")){ + cities.add(c); + } + }else{ + cities = ServerAreaMap.findAllCities() + } println cities Date weekend = getLastDayOfMonth() println weekend diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index f35522733..406d043a8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -383,7 +383,6 @@ class DealPriceDataProcess { def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}" sqlInstance236.executeUpdate(sql) } - } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy index 5354eafe8..21ddf507c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy @@ -48,13 +48,14 @@ class AnjukeAppCrawl { // task.run(); // task.run2() - task.addThirdPrice() +// task.addThirdPrice() // task.saveToEs() // task.saveToEs1() // task.groupByCity() // task.selectNoSave() // task.statCount() + task.exportFzDist() } def saveToEs1(){ def lists = getAllAppDatas() @@ -75,6 +76,24 @@ class AnjukeAppCrawl { }) } + def exportFzDist(){ + def q = new BasicDBObject() + q.put("city","福州") + def dbcursor = detail.find(q).batchSize(5000) + def index = 0 + def rets = [] + rets << ["city","region","name","_id","id","sale_num","city_short","source"] + for(def d : dbcursor){ + def count = 0 + if(d.propInfo.saleNum){ + count = d.propInfo.saleNum.toInteger() + } + rets << [d.base.cityName, d.base.areaName, d.base.name, d._id.toString(),d.base.id, count, "fz","安居客"] + } + + ExcelUtils.write("D:\\数据\\福州\\福州小区.xlsx", rets, "数据") + } + def selectNoSave(){ def esData = findAllEsData() def q = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy index 81056b29d..bd7567c8e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy @@ -25,14 +25,14 @@ class AnjukeHouseCrawl { System.setProperty("http.nonProxyHosts", "localhost"); WebDriver driver = new ChromeDriver() - def href = "https://sh.esf.fang.com/loupan/1210043376.htm" + def href = "https://fz.anjuke.com/prop/view/S2512363126645770" def doc = getDocument(href, driver) // def doc = ClaABuYun.getResponse(href) // def newUrl = href.substring(0, hrefs.indexOf("?")) - sendMq(href, doc, "district_detail_fang") + sendMq(href, doc, "sell_anjuke") -// for(def i=1;i<50;i++){ -// def url = "https://fz.anjuke.com/sale/guloue/p${i}/" +// for(def i=25;i<50;i++){ +// def url = "https://fz.anjuke.com/sale/jinana/p${i}/" // def doc = getDocument(url, driver) // def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc) // println "页数:"+i+" 记录数:"+eles.size() @@ -203,7 +203,18 @@ class AnjukeHouseCrawl { } if(!els){ - els = JQueryUtils.select(".village_info", doc) + els = JQueryUtils.select(".detail-content", doc) + } + if(!els){ + els = JQueryUtils.select(".content", doc) + } + + if(!els){ + els = JQueryUtils.select(".wrapper", doc) + } + + if(!els){ + els = JQueryUtils.select(".main-wrap", doc) } if(els && els.size()>0){ stop = true; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy index c9ca234ce..a465c984a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy @@ -23,14 +23,14 @@ class ZhugeFangDistCrawl { def run(){ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); - def url = "https://fz.esfxiaoqu.zhuge.com/pingtan/" + def url = "https://fz.esfxiaoqu.zhuge.com/changle/" WebDriver driver = new ChromeDriver() driver.manage().deleteAllCookies() getRegions(driver, url) } def getRegions(driver, url) { - for(def i=1;i<=4;i++){ + for(def i=1;i<=7;i++){ def newUrl = url+"page${i}/" println newUrl def doc = getDocument(newUrl, driver) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy new file mode 100644 index 000000000..0cd120438 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy @@ -0,0 +1,147 @@ +package com.fangjia.script.gj.paimai + +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class ExportPaimai { + + + def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl) + + static main(args) { + def task = new ExportPaimai(); + task.run() +// task.addressStandard() +// task.addPrice() +// task.stat() + } + + def stat(){ + def file = "D:\\数据\\重庆农商行\\法拍\\宿迁法拍数据.xlsx" + def datas = ExcelUtils.read(file, 0, 1) + def index1 = 0 + + def rates1 = [] + for(def d : datas){ + if(d['P'] && d['P']=='已成交'){ + if(d['M'] && d['M']=="住宅") +// if(d['K']>=150){ +// index1++ +// def sumTradePrice = d['S'] +// def marketPrice = d['Y'] +// rates1 << sumTradePrice/marketPrice +// } + if(d['S']>2000000){ + index1++ + def sumTradePrice = d['S'] + def marketPrice = d['Y'] + rates1 << sumTradePrice/marketPrice + } + } + } + def rate = rates1.sum()/rates1.size() + println index1 +" "+ rate + } + + def addPrice(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\宿迁法拍数据.xlsx",{d-> + if(!d['Y'] && d['Z']){ + d['Y'] = d['Z'] + } + }) + } + def addressStandard(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\重庆法拍数据.xlsx",{d-> + def formatAddress = AddressUtil.formatAddress(d['C'], d['A']) + if(formatAddress){ + d['F'] = formatAddress + } + }) + } + + def run(){ + def datas = findPaimaiData() + println "总量:"+datas.size() + def rows = [] + def header = ["标地物地址","省","城市","区域","地址","标题提取地址","楼栋号","单元","室号","楼盘名称","建筑面积","竣工日期","房屋用途","总楼层", + "拍卖状态","是否成交","拍卖阶段","拍卖方式","成交价","拍卖开始时间","拍卖结束时间","起拍价","保证金","加价幅度","评估价","市场价", + "变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数","URL"] + rows << header + def set = new HashSet() + def index = 0 + for(def d : datas){ + if(!set.contains(d.title)){ + index++ + println index + set << d.title + def row = [] + row << d.title + row << d.province + row << d.city + row << d.region + row << d.address + row << "" +// def formatAddress = AddressUtil.formatAddress(d.city, d.title) +// if(formatAddress){ +// row << formatAddress +// }else{ +// row << "" +// } + row << d.houseNum + row << d.unitNum + row << d.roomNum + row << d.matchName + row << d.area + row << d.buildYear + row << d.estateType + row << "" + row << "" + row << d.status + row << d.stage + row << "" + row << d.tradePrice + row << DateUtils.formatDateByPattern(d.startDate, "yyyy-MM-dd") + row << DateUtils.formatDateByPattern(d.finishDate, "yyyy-MM-dd") + row << d.startPrice + row << d.depositPrice + row << d.addPrice + row << d.evalatePrice + row << d.marketPrice + row << d.sellOffAdvancePay + row << d.pricePkWeekly + row << d.delayWeekly + row << d.preemptivePerson + row << d.enrollCount + row << d.lookCount + row << d.callCount + row << d.delayCount + row << d.url + + rows << row + } + } + ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\重庆法拍原数据.xlsx",rows,"数据") + } + + def findPaimaiData(){ + def pmquery = new BasicDBObject() + pmquery.put("city","重庆") + pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01"))) + pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-06-01"))) + pmquery.put("status","已成交") + def dbcursor = paimai43_COLL.find(pmquery).sort(new BasicDBObject("finishDate",-1)) + def rets = [] + for(def db : dbcursor){ +// if(db.estateType && db.estateType=="车库"){ +// continue +// } + rets << db + } + return rets + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy new file mode 100644 index 000000000..2f0d142d2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy @@ -0,0 +1,71 @@ +package com.fangjia.script.gj.paimai + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.AMQP + +import java.text.SimpleDateFormat + +DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ali_fapai_detail", "192.168.0.235", 27018) + +def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; +DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl) + + +//Date date=new Date(1550455200000); +//SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); +//String str = format.format(date); +//System.out.println(str); + +def pmquery = new BasicDBObject() +pmquery.put("city","宿迁") +pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01"))) +//pmquery.put("status","已成交") +//pmquery.put("estateType", null) +pmquery.put("url","https://sf-item.taobao.com/sf_item/665574733237.htm") + +def cur = paimai43_COLL.find(pmquery) +def urls = [] +for(def o : cur){ + urls << o.url +} +println "待解析:"+urls +def index = 0 +for(def url : urls){ + def q = new BasicDBObject() + q.put("url", url) + def dbcursor = collection.find(q) + index++ + println index + for(def d : dbcursor){ + + JSON json = new JSONObject() + json.put("project", "paimai_taobao") + json.put("url", d.url) + + JSON result = new JSONObject() + result.put("content", d.content) + def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content) + result.put("trades", bid_content) + result.put("condition", d.condition) + result.put("sub_count", d.sub_count) + def data_json = net.sf.json.JSONObject.fromObject(d.json) + result.put("data_json", data_json) + result.put("desc_content", d.desc_content) + result.put("item_notice_content", d.item_notice_content) + result.put("notice_detail_content", d.notice_detail_content) + + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + } +} + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy index 98110403e..3579922b1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy @@ -44,9 +44,7 @@ class PdsUseStatisticBatch extends BaseBatch{ private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000) def static emails = [ - "hubingliang@fangjia.com", - "james.lin@fangjia.com", - "panyingjiao@fangjia.com", + "duweiliang@fangjia.com", "yaojingmeng@fangjia.com", "xuqi@fangjia.com", diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/DaiQianGuZhiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/DaiQianGuZhiBatch.groovy index e1f6932eb..49db7393c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/DaiQianGuZhiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/DaiQianGuZhiBatch.groovy @@ -34,8 +34,7 @@ class DaiQianGuZhiBatch extends BaseBatch{ final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files') def static emails = [ "hubingliang@fangjia.com", - "james.lin@fangjia.com", - "panyingjiao@fangjia.com" + "james.lin@fangjia.com" ] static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuFenLeiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuFenLeiBatch.groovy index 6fdaf20e3..da74f4bd4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuFenLeiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuFenLeiBatch.groovy @@ -34,8 +34,7 @@ class KeHuFenLeiBatch extends BaseBatch{ final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000) def static emails = [ "hubingliang@fangjia.com", - "james.lin@fangjia.com", - "panyingjiao@fangjia.com" + "james.lin@fangjia.com" ] static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuPingJiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuPingJiBatch.groovy index 9aad05cf6..a2219d524 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuPingJiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuPingJiBatch.groovy @@ -33,9 +33,7 @@ class KeHuPingJiBatch extends BaseBatch{ private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files') def static emails = [ - "hubingliang@fangjia.com", - "james.lin@fangjia.com", - "panyingjiao@fangjia.com" + ] static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/oracle/ExportExceptionPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/oracle/ExportExceptionPrice.groovy index ce6a0b94b..9bb7c8c4b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/oracle/ExportExceptionPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/oracle/ExportExceptionPrice.groovy @@ -34,7 +34,7 @@ sqlInstance.eachRow(sql, { r -> if (year_month == '202205') { fiveMap.put(key, r['avg_price'] + "|" + r['calc_type']) } else if ((year_month == '202206')) { - fiveMap.put(key, r['avg_price'] + "|" + r['calc_type']) + sixMap.put(key, r['avg_price'] + "|" + r['calc_type']) } }) for (String key : sixMap.keySet()) { @@ -47,4 +47,4 @@ for (String key : sixMap.keySet()) { } } } -ExcelUtils.write("F:\\数据\\价格\\东莞上海价格偏差大.xlsx", result, "统计") \ No newline at end of file +ExcelUtils.write("D:\\数据\\价格\\东莞上海价格偏差大.xlsx", result, "统计") \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy new file mode 100644 index 000000000..f93b844d7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy @@ -0,0 +1,94 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.statistics.poi.AmapPoiDetail +import com.fangjia.script.temp.shenjian.poi.entity.Community +import com.fangjia.script.temp.shenjian.util.CommonDtoUtils +import com.fangjia.script.temp.shenjian.util.EsUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import org.apache.commons.collections.CollectionUtils +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.stream.Collectors + +/** + * 从ES获取POI + */ +class GetPoiFromCommunity { + + static final ExecutorService executorService = Executors.newFixedThreadPool(10) + static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail" + // 住宅类 + static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304"; + + static void main(String[] args) { + AtomicLong atomicLong = new AtomicLong(0) + Map map = new ConcurrentHashMap<>() + def rets = this.findCommunities() + CountDownLatch countDownLatch = new CountDownLatch(rets.size()) + for (def retTemp : rets) { + final def ret = retTemp + executorService.submit(new Runnable() { + @Override + void run() { + final String id = ret._id // 索引ID,并非小区ID + String city = ret.city + String region = ret.region + String seaweedName = ret.name + String address = ret.address + List pois = new ArrayList<>(); + List namePois = GaoDeUtils.searchPoi(city, seaweedName, region, typecodes); + List addressPois = GaoDeUtils.searchPoi(city, address, region, typecodes); + if (CollectionUtils.isNotEmpty(namePois)) { + pois.addAll(namePois) + } + if (CollectionUtils.isNotEmpty(addressPois)) { + pois.addAll(addressPois) + } + if (CollectionUtils.isNotEmpty(pois)) { + List detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois); + String jsonStr = JSONObject.toJSONString(detailList) + HttpUtils.postForJson(URL, jsonStr) + } + synchronized (map) { + map.put(id, id) + countDownLatch.countDown() + if (map.size() % 100 == 0) { + atomicLong.addAndGet(map.size()) + List ids = map.values().stream().collect(Collectors.toList()) + EsUtils.bulkUpdateFieldByIds("community", "crawl_flag", 0, ids) + map.clear() + println(atomicLong.intValue()) + } + } + } + }) + } + countDownLatch.await() + if (map.size() > 0) { + atomicLong.addAndGet(map.size()) + List ids = map.values().stream().collect(Collectors.toList()) + EsUtils.bulkUpdateFieldByIds("community", "crawl_flag", 0, ids) + map.clear() + println(atomicLong.intValue()) + } + System.exit(1) + } + + static def findCommunities(){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery() + .mustNot(QueryBuilders.termQuery("crawl_flag", 0)) + List communities = EsUtils.scrollQuery("community", queryBuilder, Community.class) + return communities + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy new file mode 100644 index 000000000..ba2bfaaea --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy @@ -0,0 +1,49 @@ +package com.fangjia.script.temp.shenjian.poi.entity; + +public class Community extends MyId { + private String id; + private String city; + private String region; + private String name; + private String address; + + String getId() { + return id + } + + void setId(String id) { + this.id = id + } + + String getCity() { + return city + } + + void setCity(String city) { + this.city = city + } + + String getRegion() { + return region + } + + void setRegion(String region) { + this.region = region + } + + String getName() { + return name + } + + void setName(String name) { + this.name = name + } + + String getAddress() { + return address + } + + void setAddress(String address) { + this.address = address + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/MyId.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/MyId.groovy new file mode 100644 index 000000000..53bd1b98b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/MyId.groovy @@ -0,0 +1,17 @@ +package com.fangjia.script.temp.shenjian.poi.entity + +class MyId { + + /** + * 索引id + */ + String _id; + + String get_id() { + return _id + } + + void set_id(String _id) { + this._id = _id + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy new file mode 100644 index 000000000..b158c9351 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy @@ -0,0 +1,116 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils +import com.fangjia.script.temp.shenjian.poi.entity.MyId +import org.elasticsearch.action.DocWriteRequest +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.search.* +import org.elasticsearch.action.update.UpdateRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.client.RestHighLevelClient +import org.elasticsearch.common.unit.TimeValue +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.search.Scroll +import org.elasticsearch.search.SearchHit +import org.elasticsearch.search.builder.SearchSourceBuilder +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * @author shenjian + * @since 2022/07/02 + */ +class EsUtils { + + private static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient() + private static final Logger logger = LoggerFactory.getLogger(EsUtils.class) + + /** + * 滚动方式查询大批量es数据 + * + * @param index 索引 + * @param boolQuery 查询语句 + * @param clazz 要返回的实体类型 + * @param + * @return + */ + public static List scrollQuery(String index, QueryBuilder boolQuery, Class clazz, def limitSize=300000) { + List result = new ArrayList<>(); + // 存活时间,当索引数据量特别大时,出现超时可能性大,此值适当调大 + Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30L)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(boolQuery); + // TEST 用!!!!!!!!!!!!!!!!!!!!!! + //searchSourceBuilder.size(1); + searchSourceBuilder.size(10000); + SearchRequest searchRequest = new SearchRequest().indices(index).scroll(scroll).source(searchSourceBuilder); + SearchResponse searchResponse = null; + try { + searchResponse = ES_CLIENT.search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("查询es数据异常:{}", e); + } + String scrollId = searchResponse.getScrollId(); + SearchHit[] searchHits = searchResponse.getHits().getHits(); + if (searchHits != null) { + for (SearchHit searchHit : searchHits) { + T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz); + t.set_id(searchHit.getId()) + result.add(t); + } + } + // 遍历搜索命中的数据,直到没有数据 + while (searchHits != null && searchHits.length > 0 && result.size() < limitSize) { + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + try { + searchResponse = ES_CLIENT.scroll(scrollRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("查询es数据异常:{}", e); + } + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + if (searchHits != null && searchHits.length > 0) { + for (SearchHit searchHit : searchHits) { + T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz); + t.set_id(searchHit.getId()) + result.add(t); + } + } + } + // clean scroll + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + ClearScrollResponse clearScrollResponse = null; + try { + clearScrollResponse = ES_CLIENT.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + } catch (IOException e) {} + boolean succeeded = clearScrollResponse.isSucceeded(); + if (succeeded) { + return result; + } + return null; + } + + + /** + * 批量更新es某个字段值 + * @param index 索引名 + * @param field 字段名 + * @param val 字段新值 + * @param ids 要更新的文档id + */ + public static void bulkUpdateFieldByIds(String index, String field, Object val, List ids) { + if (ids.size() > 1000) { + throw new RuntimeException("批量更新数量过多") + } + BulkRequest bulkRequest = new BulkRequest(); + for (String id : ids) { + Map doc = new HashMap<>(); + doc.put(field, val) + bulkRequest.add(new UpdateRequest(index, id).doc(doc)); + } + ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 6b880c5ea..af5f5fc8b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger class GaoDeUtils { // 新获取key - def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418"] + def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418", "10bcb421d2ee4e51ed9268d6dca58ad1"] def static AtomicInteger keyUsed = new AtomicInteger(0) def static codeNameMap = new HashMap(){{ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 4ac878217..383911899 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202204); - def count2 = findTotalCount(source, 202205); + def count1 = findTotalCount(source, 202205); + def count2 = findTotalCount(source, 202206); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 3c2bcde38..89e0e9483 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -132,7 +132,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ q.put("s_date", SHORT_DATE); def ret = [] try{ - def dbcursor = dataProcessColl.find(q).sort(new BasicDBObject("m_date", -1).append("type",1)) + def dbcursor = dataProcessColl.find(q).sort(new BasicDBObject("type",1).append("m_date", -1)) if(dbcursor){ for(def dbo : dbcursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy index 235253d60..51a920e7e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy @@ -1,6 +1,39 @@ package com.fangjia.script.wenyan +//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\国信达级联价格-上海.xlsx",{List item-> +// if(item["AB"]){ +// return +// } +// String region = null +// String name = item["F"] +// if(name.endsWith("小区")){ +// name = name.replaceAll("小区\$", "") +// } +// region = RegionData.formatRegion(item["B"], item["D"]) +// def results = [] +// DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(item["B"], region, name)).each { +// if(!results.contains(it)){ +// results.add(it) +// } +// } +// DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(item["B"], region).append("alias", name)).each { +// if(!results.contains(it)){ +// results.add(it) +// } +// } +// if(results.size() == 1){ +// item["Z"] = results[0]._id.toString() +// item["AA"] = results[0].region +// item["AB"] = results[0].name +// println(item) +// } +//}) + + + + import com.fangjia.core.ProvinceData +import com.fangjia.core.RegionData import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DbUtils @@ -76,7 +109,48 @@ import org.apache.commons.lang.StringUtils // }, new BasicDBObject("township", 1)) //} //println(count) +//def GXD_CCB = [:] +//def CCB_FJW = [:] +//ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\4月级联待优化小区_3个城市.xlsx",sheetIndex = 1).each {List item-> +// if(item["V"] == "级联"){ +// if(item["Y"] == "国信达"){ +// GXD_CCB[item['J']] = item["E"] +// }else if(item["Y"] == "房价网"){ +// CCB_FJW[item['E']] = item["J"] +// } +// } +//} +//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\4月级联待优化小区_3个城市.xlsx",{List item-> +// if(item["V"] == "级联" && item["Y"] == "国信达"){ +// def ccbId = GXD_CCB.get(item['J'], null) +// def fjwId = CCB_FJW.get(ccbId, null) +// if(fjwId != null){ +// item["AB"] = fjwId +//// BasicDBObject seaweed = DbUtils.seaweed.findById(fjwId) +//// List results = DbUtils.findAroundDistrict(seaweed) +//// long total = 0 +//// long count = 0 +//// for(BasicDBObject s : results){ +//// def avg = DbUtils.findDistrictAveragePrice(seaweed.city, seaweed.region, seaweed.name, Date.parse('yyyyMMdd', '20220619'), seaweed.estate_type2) +//// if(avg){ +//// total += avg.avg_price +//// count += 1 +//// } +//// } +//// if(count > 0){ +//// item['Z'] = (int)(total/count) +//// println(item) +//// } +// } +// } +//},sheetIndex = 1) -ElasticUtils.executeSqlQuery("select count(*), p_date from seawater where p_date > 20210501 group by p_date",{ - println(it) -}) \ No newline at end of file +//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\moban (3).xls",{List item-> +// item["B"] = RegionData.formatRegion(item["A"], item["B"]) +// println(item) +//}) +int count = 0 +DbUtils.seaweed.find(DbUtils.buildSeaweedQuery("东莞")).each {def a-> + count += 1 +} +println(count) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/TongCheng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/TongCheng.groovy new file mode 100644 index 000000000..a0e5faf71 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/TongCheng.groovy @@ -0,0 +1,151 @@ +package com.fangjia.script.wenyan + +import com.fangjia.core.util.Mongo3Utils +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import net.sf.json.JSONArray +import net.sf.json.JSONObject +import org.bson.Document + +//def data = ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\TC重新跑数据.xlsx") +//def addMap = [:] +//data.each {def list-> +// if(list["C"] == "有效"){ +// addMap[list["B"]] = list +// } +//} +//ExcelUtils.modify("D:\\Users\\wenyan\\Downloads\\data_gj20220616.xlsx",{List item-> +// if(addMap.containsKey(item["B"])){ +// List list = addMap.get(item["B"]) +// item["C"] = list["D"] +// item["D"] = list["E"] +// item["E"] = list["F"] +// item["S"] = "有效" +// println(item) +// } +//}) + +def DISTRICT_ANALY_DATA = Mongo3Utils.getDbCollection("fangjia_stat", "district_analy_data") +ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\data_gj20220616.xlsx", { List item -> +// if(item["S"] == null){ +// if(item["Q"] != null || item["R"] != null){ +// item["S"] = "有效" +// } +// } +// if(item["S"] == null){ +// def source = null +// JSONObject resp = OpenApiClient.addressMatch(item["A"], item["B"], "property") +// source = "小区库" +// if(resp.credit == 0){ +// resp = OpenApiClient.addressMatch(item["A"], item["B"], null) +// source = "小区库" +// } +// if(resp.credit == 0){ +// resp = OpenApiClient.addressMatch(item["A"], item["B"], null, true) +// source = "地图" +// } +// +// if(resp.credit > 0){ +// println(resp) +// item["C"] = resp.city +// item["D"] = resp.district +// item["E"] = resp.name +// item["F"] = source +// item["G"] = resp.lng +// item["H"] = resp.lat +// item["I"] = resp.propertyType +// } +// if(item["G"] != null && !["普通住宅", "商住", "别墅"].contains(item["I"])){ +// List seaweeds = DbUtils.findAroundDistrict(item["G"], item["H"], 1000 ,1, 2) +// if(seaweeds.size() > 0){ +// BasicDBObject s = seaweeds[0] +// item["C"] = s.city +// item["D"] = s.region +// item["E"] = s.name +// item["F"] = "周边" +// item["G"] = s.lng/10000000000d +// item["H"] = s.lat/10000000000d +// item["I"] = s.estate_type2 +// } +// } +// } + + + +// if(item["S"]|| item["T"]){ +// if(!item["i"]){ +// JSONObject result = GaoDeApiUtils.geo(item["A"], item["B"]) +// if(result != null && result.getInt("status") == 1){ +// JSONArray geocodes = result.getJSONArray("geocodes"); +// if(geocodes != null && geocodes.size() > 0){ +// JSONObject geocode = geocodes.getJSONObject(0); +// String location = geocode.getString("location") +// item["i"] = location.split(",")[0].toDouble() +// item["j"] = location.split(",")[1].toDouble() +// } +// } +// println(item) +// } +// } +// if(item["D"] == null && item["F"] != null){ +// String kw = item["F"] +// if(item["E"]){ +// kw = item["E"] + kw +// } +// JSONObject result = GaoDeApiUtils.geo(item["A"], kw) +// if(result != null && result.getInt("status") == 1){ +// JSONArray geocodes = result.getJSONArray("geocodes"); +// if(geocodes != null && geocodes.size() > 0){ +// JSONObject geocode = geocodes.getJSONObject(0); +// String location = geocode.getString("location") +// item["i"] = location.split(",")[0].toDouble() +// item["j"] = location.split(",")[1].toDouble() +// } +// } +// println(item) +// } +// if(item["E"] != null && ["普通住宅", "商住", "别墅"].contains(item["I"])){ +// Document doc = DISTRICT_ANALY_DATA.find(new Document("city", item["C"]).append("region", item["D"]).append("district_name", item["E"])).first() +// if(doc){ +// item["N"] =doc["rank"] +// } +// } +// if (item["S"] != "有效" || item["F"] != null) { +// return +// } +// JSONObject resp = OpenApiClient.addressMatch(item["A"], item["E"], "property", false, item["D"], null, "strict") +// if(resp.credit > 0){ +// println(resp) +// item["C"] = resp.city +// item["D"] = resp.district +// item["E"] = resp.name +// item["F"] = "小区库" +// item["G"] = resp.lng +// item["H"] = resp.lat +// item["I"] = resp.propertyType +// } +// println(resp) +// if (item["F"] == null) { +// String kw = item["E"] +// if (item["D"]) { +// kw = item["D"] + kw +// } +// JSONObject result = GaoDeApiUtils.geo(item["A"], kw) +// if (result != null && result.getInt("status") == 1) { +// JSONArray geocodes = result.getJSONArray("geocodes"); +// if (geocodes != null && geocodes.size() > 0) { +// JSONObject geocode = geocodes.getJSONObject(0); +// String location = geocode.getString("location") +// item["G"] = location.split(",")[0].toDouble() +// item["H"] = location.split(",")[1].toDouble() +// item["F"] = "反查经纬度" +// } +// } +// println(item) +// } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index 1297ca3ea..8bca701d7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20220501", "yyyyMMdd") -def endDate = DateUtil.parseDate("20220601", "yyyyMMdd") +def startDate = DateUtil.parseDate("20220601", "yyyyMMdd") +def endDate = DateUtil.parseDate("20220701", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/batch/ApiProductFieldStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/batch/ApiProductFieldStatBatch.groovy index 6f120465c..026ec703f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/batch/ApiProductFieldStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/batch/ApiProductFieldStatBatch.groovy @@ -62,7 +62,7 @@ class ApiProductFieldStatBatch extends BaseBatch { private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000) def emails = [ - "yinjihuan@fangjia.com","panyingjiao@fangjia.com","liyan@fangjia.com", "jiangxiaochun@fangjia.com" + "jiangxiaochun@fangjia.com" ] static def allDis = [] static DecimalFormat df = new DecimalFormat("0.00%") diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 15909a4c9..c7c6b08a6 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -499,6 +499,7 @@ public class AddressParserServiceImpl implements AddressParserService { addr.setAddressParserGoal(params.getAddressParserGoal()); addr.setProvince(params.getProvince()); addr.setDebugName(params.getDebugName()); + addr.setUseMap(params.isUseMap()); return AddressProcessors.process(addr); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java index ed723c1d5..02cee4b25 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java @@ -63,6 +63,7 @@ public class Address { private String suggestNamePinYin; private String polygonSeaweedId; private String geoRegion; + private boolean useMap; private List buildingNumbers; private PreciseLevel preciseLevel; private boolean pinYin; @@ -389,6 +390,14 @@ public class Address { isStandardAddress = standardAddress; } + public boolean isUseMap() { + return useMap; + } + + public void setUseMap(boolean useMap) { + this.useMap = useMap; + } + /** * 获得有效地址 * diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java index e3ef9a3da..bb33d7dbd 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java @@ -28,6 +28,9 @@ public class GeoProcessor extends AbstractAddressProcessor { if (address.hasCoordinate()) { return RepeatStatus.NEXT; } + if(!address.isUseMap()){ + return RepeatStatus.NEXT; + } String addr = address.getEffectiveAddress(); if(addr == null || addr.length() < 3){ addr = address.getOrgAddress(); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/index/SellTriggerServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/index/SellTriggerServiceImpl.java index efe0684cc..7e88147a0 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/index/SellTriggerServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/index/SellTriggerServiceImpl.java @@ -46,7 +46,7 @@ public class SellTriggerServiceImpl implements TriggerService { public List processBSONObjects(List bSONDocList) { List ret = new ArrayList(); // Map districtKnowbaseMap = indexHelperService.getFromKnowBase(bSONDocList); - Map> metroDistanceMap = indexHelperService.getMetroDistanceMap(bSONDocList); +// Map> metroDistanceMap = indexHelperService.getMetroDistanceMap(bSONDocList); Map districtMap = indexHelperService.getDistricts(bSONDocList); // Map districtStatMap = // indexHelperService.getDistrictStats(districtKnowbaseMap); @@ -70,12 +70,12 @@ public class SellTriggerServiceImpl implements TriggerService { for (BSONObject object : bSONDocList) { try { ServerArea city = ServerAreaData.getServerAreaByDisplayDefaultNull(String.valueOf(object.get("city"))); - DistrictKey districtKey = new DistrictKey((String) object.get("city"), (String) object.get("region"), - (String) object.get("district_name")); - District district = districtMap.get(districtKey); - if (district != null && district.getName() != null) { - districtKey = new DistrictKey(district.getCity(), district.getRegion(),district.getName()); - } +// DistrictKey districtKey = new DistrictKey((String) object.get("city"), (String) object.get("region"), +// (String) object.get("district_name")); +// District district = districtMap.get(districtKey); +// if (district != null && district.getName() != null) { +// districtKey = new DistrictKey(district.getCity(), district.getRegion(),district.getName()); +// } BSONObject newObj = new BasicBSONObject(); @@ -156,16 +156,16 @@ public class SellTriggerServiceImpl implements TriggerService { newObj.put("fitment", object.get("fitment")); newObj.put("contact_people", object.get("contact_people")); ScoreCalculatorInputData inputDate = new ScoreCalculatorInputData(); - if (district != null) { - newObj.put("region", district.getRegion()); - if (StringUtils.isNotBlank(district.getBlock())) { - newObj.put("block", district.getBlock()); - } - if (district.getName() != null) { - newObj.put("districtname_key", district.getName() + "_" + newObj.get("region") + "_" + object.get("city")); - newObj.put("district_name", district.getName()); - } - } +// if (district != null) { +// newObj.put("region", district.getRegion()); +// if (StringUtils.isNotBlank(district.getBlock())) { +// newObj.put("block", district.getBlock()); +// } +// if (district.getName() != null) { +// newObj.put("districtname_key", district.getName() + "_" + newObj.get("region") + "_" + object.get("city")); +// newObj.put("district_name", district.getName()); +// } +// } // newObj.put("if_school", 0); // if (districtSchoolMap != null && districtSchoolMap.containsKey(districtKey)) { @@ -248,16 +248,16 @@ public class SellTriggerServiceImpl implements TriggerService { newObj.put("visible", 1); } - // 地铁信息 - if (metroDistanceMap != null && metroDistanceMap.get(districtKey) != null) { - newObj.put("metro", metroDistanceMap.get(districtKey).get("metro")); - newObj.put("station", metroDistanceMap.get(districtKey).get("station")); - newObj.put("metro_info", metroDistanceMap.get(districtKey).get("metro_info")); - newObj.put("if_metro", metroDistanceMap.get(districtKey).get("if_metro")); - } else { - newObj.put("if_metro", 0); - } - +// // 地铁信息 +// if (metroDistanceMap != null && metroDistanceMap.get(districtKey) != null) { +// newObj.put("metro", metroDistanceMap.get(districtKey).get("metro")); +// newObj.put("station", metroDistanceMap.get(districtKey).get("station")); +// newObj.put("metro_info", metroDistanceMap.get(districtKey).get("metro_info")); +// newObj.put("if_metro", metroDistanceMap.get(districtKey).get("if_metro")); +// } else { +// newObj.put("if_metro", 0); +// } +// //2018-07-31新增字段 alert by jenis newObj.put("estate_type", object.get("estate_type"));//物业类型 newObj.put("house_number", object.get("house_number"));//房屋编号