Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
@@ -1847,6 +1847,9 @@ public class DataUtil {
|
||||
//
|
||||
// System.out.println(FourUpSixInto("258.393", 2));
|
||||
|
||||
String href = encodePicUrl("62afd81d303c3f827833bb03", "o", "upload");
|
||||
String link = "http://static.fangjia.com/stc/fs/"+href;
|
||||
System.out.println(link);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -105,6 +105,7 @@ public class FjtEvaluation implements SqlEntity{
|
||||
private String certificateAddress; //产证地址
|
||||
|
||||
private Date downPdfDate;//下载报告时间
|
||||
private Integer loan ;//是否续贷 1:是 0:否
|
||||
|
||||
@GeneratedValue(strategy = GenerationType.AUTO)
|
||||
@Id
|
||||
@@ -773,6 +774,15 @@ public class FjtEvaluation implements SqlEntity{
|
||||
this.downPdfDate = downPdfDate;
|
||||
}
|
||||
|
||||
@Column(name = "loan")
|
||||
public Integer getLoan() {
|
||||
return loan;
|
||||
}
|
||||
|
||||
public void setLoan(Integer loan) {
|
||||
this.loan = loan;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "FjtEvaluation{" +
|
||||
|
||||
@@ -11,7 +11,7 @@ public class ReportFileAttach implements SqlEntity {
|
||||
private Integer userId;
|
||||
private Integer evaluationId;
|
||||
private String queryId;
|
||||
private Integer fileClass;
|
||||
private Integer fileClass;//1:询价单 2:产证 3:房屋照片 4:其它附件 5:产调报告或权籍
|
||||
private String fileName;
|
||||
private String fileId;
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
<addClasspath>true</addClasspath>
|
||||
<classpathPrefix>lib/</classpathPrefix>
|
||||
<mainClass>com.fangjia.datastore.DataStoreServer</mainClass>
|
||||
</manifest>
|
||||
</manifest>
|
||||
</archive>
|
||||
</configuration>
|
||||
</plugin>
|
||||
@@ -94,8 +94,8 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mongodb</groupId>
|
||||
<artifactId>mongo-java-driver</artifactId>
|
||||
<version>2.6.5</version>
|
||||
<artifactId>mongodb-driver-legacy</artifactId>
|
||||
<version>3.9.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
|
||||
+18
-19
@@ -1,22 +1,21 @@
|
||||
package com.fangjia.datastore;
|
||||
|
||||
public class ConfigurationException extends Exception {
|
||||
|
||||
private static final long serialVersionUID = 6570405263317311822L;
|
||||
|
||||
public ConfigurationException() {
|
||||
}
|
||||
|
||||
public ConfigurationException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public ConfigurationException(Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
|
||||
public ConfigurationException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
|
||||
public class ConfigurationException extends Exception
|
||||
{
|
||||
private static final long serialVersionUID = 6570405263317311822L;
|
||||
|
||||
public ConfigurationException() {
|
||||
}
|
||||
|
||||
public ConfigurationException(final String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public ConfigurationException(final Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
|
||||
public ConfigurationException(final String message, final Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,80 +1,83 @@
|
||||
package com.fangjia.datastore;
|
||||
|
||||
import com.fangjia.datastore.mongo.protocol.QueryMessage;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import com.fangjia.datastore.mongo.protocol.QueryMessage;
|
||||
|
||||
public class Cursor {
|
||||
private static final AtomicLong ID_GENERATOR = new AtomicLong(1);
|
||||
private long id;
|
||||
private int limit;
|
||||
private int skip;
|
||||
private long totalCount;
|
||||
private QueryMessage queryMessage;
|
||||
|
||||
public Cursor(int skip, int limit, QueryMessage queryMessage, long totalCount) {
|
||||
id = ID_GENERATOR.getAndIncrement();
|
||||
this.skip = skip;
|
||||
this.limit = limit;
|
||||
this.totalCount = totalCount;
|
||||
this.queryMessage = queryMessage;
|
||||
}
|
||||
|
||||
public long getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public int getLimit() {
|
||||
return limit;
|
||||
}
|
||||
|
||||
public void setLimit(int limit) {
|
||||
this.limit = limit;
|
||||
}
|
||||
|
||||
public int nextSkip() {
|
||||
skip += limit;
|
||||
return skip;
|
||||
}
|
||||
|
||||
public void setSkip(int skip) {
|
||||
this.skip = skip;
|
||||
}
|
||||
|
||||
public boolean hasMore() {
|
||||
return this.totalCount > this.skip;
|
||||
}
|
||||
|
||||
public long getTotalCount() {
|
||||
return totalCount;
|
||||
}
|
||||
|
||||
public void setTotalCount(long totalCount) {
|
||||
this.totalCount = totalCount;
|
||||
}
|
||||
|
||||
public QueryMessage getQueryMessage() {
|
||||
return queryMessage;
|
||||
}
|
||||
|
||||
public void setQueryMessage(QueryMessage queryMessage) {
|
||||
this.queryMessage = queryMessage;
|
||||
}
|
||||
|
||||
public int getSkip() {
|
||||
return skip;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("id:").append(id);
|
||||
builder.append(", limit:").append(limit);
|
||||
builder.append(", skip:").append(skip);
|
||||
builder.append(", totalCount:").append(totalCount);
|
||||
builder.append(", queryMessage:").append(queryMessage);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
public class Cursor
|
||||
{
|
||||
private static final AtomicLong ID_GENERATOR;
|
||||
private long id;
|
||||
private int limit;
|
||||
private int skip;
|
||||
private long totalCount;
|
||||
private QueryMessage queryMessage;
|
||||
|
||||
public Cursor(final int skip, final int limit, final QueryMessage queryMessage, final long totalCount) {
|
||||
this.id = Cursor.ID_GENERATOR.getAndIncrement();
|
||||
this.skip = skip;
|
||||
this.limit = limit;
|
||||
this.totalCount = totalCount;
|
||||
this.queryMessage = queryMessage;
|
||||
}
|
||||
|
||||
public long getId() {
|
||||
return this.id;
|
||||
}
|
||||
|
||||
public int getLimit() {
|
||||
return this.limit;
|
||||
}
|
||||
|
||||
public void setLimit(final int limit) {
|
||||
this.limit = limit;
|
||||
}
|
||||
|
||||
public int nextSkip() {
|
||||
return this.skip += this.limit;
|
||||
}
|
||||
|
||||
public void setSkip(final int skip) {
|
||||
this.skip = skip;
|
||||
}
|
||||
|
||||
public boolean hasMore() {
|
||||
return this.totalCount > this.skip;
|
||||
}
|
||||
|
||||
public long getTotalCount() {
|
||||
return this.totalCount;
|
||||
}
|
||||
|
||||
public void setTotalCount(final long totalCount) {
|
||||
this.totalCount = totalCount;
|
||||
}
|
||||
|
||||
public QueryMessage getQueryMessage() {
|
||||
return this.queryMessage;
|
||||
}
|
||||
|
||||
public void setQueryMessage(final QueryMessage queryMessage) {
|
||||
this.queryMessage = queryMessage;
|
||||
}
|
||||
|
||||
public int getSkip() {
|
||||
return this.skip;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("id:").append(this.id);
|
||||
builder.append(", limit:").append(this.limit);
|
||||
builder.append(", skip:").append(this.skip);
|
||||
builder.append(", totalCount:").append(this.totalCount);
|
||||
builder.append(", queryMessage:").append(this.queryMessage);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
static {
|
||||
ID_GENERATOR = new AtomicLong(1L);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,44 +1,35 @@
|
||||
package com.fangjia.datastore;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
import org.bson.BSONObject;
|
||||
|
||||
import com.fangjia.datastore.config.Reloadable;
|
||||
import com.fangjia.datastore.config.StoreConfig;
|
||||
import com.fangjia.datastore.solr.Index;
|
||||
import java.util.Collection;
|
||||
import com.fangjia.datastore.config.StoreConfig;
|
||||
import com.fangjia.datastore.config.Reloadable;
|
||||
|
||||
public interface DataStore extends Reloadable {
|
||||
|
||||
StoreConfig getStoreConfig();
|
||||
|
||||
Mongo getMongo();
|
||||
|
||||
Object getAttribute(String key);
|
||||
|
||||
Object setAttribute(String key, Object attr);
|
||||
|
||||
Collection<Index> getIndexes() throws DataStoreException;
|
||||
|
||||
/**
|
||||
* °´²éѯÌõ¼þÖØ½¨Ë÷Òý
|
||||
*
|
||||
* @param query
|
||||
* @throws DataStoreException
|
||||
*/
|
||||
BSONObject reIndex(BSONObject query) throws DataStoreException;
|
||||
|
||||
void ensureIndex(Collection<Index> indexes) throws DataStoreException;
|
||||
|
||||
QueryResponse find(BSONObject query, BSONObject fields, BSONObject sort, BSONObject facet, int skip, int limit, boolean bulk,
|
||||
boolean fromMaster) throws DataStoreException;
|
||||
|
||||
int count(BSONObject query, boolean fromMaster) throws DataStoreException;
|
||||
|
||||
void remove(BSONObject query) throws DataStoreException;
|
||||
|
||||
void update(BSONObject query, BSONObject data,String ip) throws DataStoreException;
|
||||
|
||||
void save(List<? extends BSONObject> docList) throws DataStoreException;
|
||||
public interface DataStore extends Reloadable
|
||||
{
|
||||
StoreConfig getStoreConfig();
|
||||
|
||||
Mongo getMongo();
|
||||
|
||||
Object getAttribute(final String p0);
|
||||
|
||||
Object setAttribute(final String p0, final Object p1);
|
||||
|
||||
Collection<Index> getIndexes() throws DataStoreException;
|
||||
|
||||
BSONObject reIndex(final BSONObject p0) throws DataStoreException;
|
||||
|
||||
void ensureIndex(final Collection<Index> p0) throws DataStoreException;
|
||||
|
||||
QueryResponse find(final BSONObject p0, final BSONObject p1, final BSONObject p2, final BSONObject p3, final int p4, final int p5, final boolean p6, final boolean p7) throws DataStoreException;
|
||||
|
||||
int count(final BSONObject p0, final boolean p1) throws DataStoreException;
|
||||
|
||||
void remove(final BSONObject p0) throws DataStoreException;
|
||||
|
||||
int update(final BSONObject p0, final BSONObject p1) throws DataStoreException;
|
||||
|
||||
void save(final List<? extends BSONObject> p0) throws DataStoreException;
|
||||
}
|
||||
|
||||
+11
-11
@@ -3,15 +3,15 @@ package com.fangjia.datastore;
|
||||
import com.fangjia.datastore.config.DataStoreConfig;
|
||||
import com.fangjia.datastore.config.Reloadable;
|
||||
|
||||
public interface DataStoreContext extends Reloadable {
|
||||
|
||||
DataStoreConfig getDataStoreConfig();
|
||||
|
||||
DataStore getDataStore(String storeName) throws DataStoreException;
|
||||
|
||||
Cursor getCursor(long cursorId);
|
||||
|
||||
void addCursor(Cursor cursor);
|
||||
|
||||
Cursor removeCursor(long cursorId);
|
||||
public interface DataStoreContext extends Reloadable
|
||||
{
|
||||
DataStoreConfig getDataStoreConfig();
|
||||
|
||||
DataStore getDataStore(final String p0) throws DataStoreException;
|
||||
|
||||
Cursor getCursor(final long p0);
|
||||
|
||||
void addCursor(final Cursor p0);
|
||||
|
||||
Cursor removeCursor(final long p0);
|
||||
}
|
||||
|
||||
+55
-50
@@ -1,56 +1,61 @@
|
||||
package com.fangjia.datastore;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import com.fangjia.datastore.config.DataStoreConfig;
|
||||
import com.fangjia.datastore.config.StoreConfig;
|
||||
import java.util.Iterator;
|
||||
import java.util.Set;
|
||||
import java.util.HashMap;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import com.google.common.collect.MapMaker;
|
||||
import java.util.Map;
|
||||
import com.fangjia.datastore.config.DataStoreConfig;
|
||||
|
||||
public class DataStoreContextImpl implements DataStoreContext {
|
||||
|
||||
private final DataStoreConfig config;
|
||||
private final Map<Long, Cursor> cursorMap;
|
||||
private final Map<String, DataStore> stores;
|
||||
|
||||
public DataStoreContextImpl(DataStoreConfig config) throws ConfigurationException {
|
||||
this.config = config;
|
||||
cursorMap = new MapMaker().concurrencyLevel(50).expiration(config.getServerConfig().getCursorExpirationMinutes(),
|
||||
TimeUnit.MINUTES).makeMap();
|
||||
Set<String> storeNames = config.getStoreNames();
|
||||
stores = new HashMap<String, DataStore>(storeNames.size());
|
||||
for (String storeName : storeNames) {
|
||||
StoreConfig storeConfig = this.config.getStoreConfig(storeName);
|
||||
stores.put(storeName, new DataStoreImpl(storeConfig, new Mongo(storeConfig.getMongoConfig())));
|
||||
}
|
||||
}
|
||||
|
||||
public DataStoreConfig getDataStoreConfig() {
|
||||
return config;
|
||||
}
|
||||
|
||||
public DataStore getDataStore(String storeName) throws DataStoreException {
|
||||
if (!stores.containsKey(storeName)) {
|
||||
throw new DataStoreException("unknown store name: \"" + storeName + "\"");
|
||||
}
|
||||
return stores.get(storeName);
|
||||
}
|
||||
|
||||
public void reload() throws ConfigurationException {
|
||||
config.reload();
|
||||
}
|
||||
|
||||
public Cursor getCursor(long cursorId) {
|
||||
return this.cursorMap.get(cursorId);
|
||||
}
|
||||
|
||||
public void addCursor(Cursor cursor) {
|
||||
this.cursorMap.put(cursor.getId(), cursor);
|
||||
}
|
||||
|
||||
public Cursor removeCursor(long cursorId) {
|
||||
return this.cursorMap.remove(cursorId);
|
||||
}
|
||||
public class DataStoreContextImpl implements DataStoreContext
|
||||
{
|
||||
private final DataStoreConfig config;
|
||||
private final Map<Long, Cursor> cursorMap;
|
||||
private final Map<String, DataStore> stores;
|
||||
|
||||
public DataStoreContextImpl(final DataStoreConfig config) throws ConfigurationException {
|
||||
this.config = config;
|
||||
this.cursorMap = new MapMaker().concurrencyLevel(50).expiration((long)config.getServerConfig().getCursorExpirationMinutes(), TimeUnit.MINUTES).makeMap();
|
||||
final Set<String> storeNames = config.getStoreNames();
|
||||
this.stores = new HashMap<String, DataStore>(storeNames.size());
|
||||
for (final String storeName : storeNames) {
|
||||
final StoreConfig storeConfig = this.config.getStoreConfig(storeName);
|
||||
this.stores.put(storeName, new DataStoreImpl(storeConfig, new Mongo(storeConfig.getMongoConfig())));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public DataStoreConfig getDataStoreConfig() {
|
||||
return this.config;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DataStore getDataStore(final String storeName) throws DataStoreException {
|
||||
if (!this.stores.containsKey(storeName)) {
|
||||
throw new DataStoreException("unknown store name: \"" + storeName + "\"");
|
||||
}
|
||||
return this.stores.get(storeName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reload() throws ConfigurationException {
|
||||
this.config.reload();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Cursor getCursor(final long cursorId) {
|
||||
return this.cursorMap.get(cursorId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addCursor(final Cursor cursor) {
|
||||
this.cursorMap.put(cursor.getId(), cursor);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Cursor removeCursor(final long cursorId) {
|
||||
return this.cursorMap.remove(cursorId);
|
||||
}
|
||||
}
|
||||
|
||||
+18
-20
@@ -1,23 +1,21 @@
|
||||
package com.fangjia.datastore;
|
||||
|
||||
public class DataStoreException extends Exception {
|
||||
|
||||
private static final long serialVersionUID = -5492973985910379215L;
|
||||
|
||||
public DataStoreException() {
|
||||
super();
|
||||
}
|
||||
|
||||
public DataStoreException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
|
||||
public DataStoreException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public DataStoreException(Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
|
||||
public class DataStoreException extends Exception
|
||||
{
|
||||
private static final long serialVersionUID = -5492973985910379215L;
|
||||
|
||||
public DataStoreException() {
|
||||
}
|
||||
|
||||
public DataStoreException(final String message, final Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
|
||||
public DataStoreException(final String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public DataStoreException(final Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
}
|
||||
|
||||
+627
-692
File diff suppressed because it is too large
Load Diff
+167
-182
@@ -1,193 +1,178 @@
|
||||
package com.fangjia.datastore;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
|
||||
import org.apache.mina.core.buffer.IoBuffer;
|
||||
import org.apache.mina.core.service.IoHandlerAdapter;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.apache.mina.core.service.IoService;
|
||||
import org.apache.mina.core.service.IoServiceListener;
|
||||
import org.apache.mina.core.service.SimpleIoProcessorPool;
|
||||
import org.apache.mina.core.session.IdleStatus;
|
||||
import org.apache.mina.core.session.IoSession;
|
||||
import org.apache.mina.filter.codec.ProtocolCodecException;
|
||||
import org.apache.mina.filter.codec.ProtocolCodecFilter;
|
||||
import org.apache.mina.filter.logging.LogLevel;
|
||||
import org.apache.mina.filter.logging.LoggingFilter;
|
||||
import org.apache.mina.transport.socket.SocketSessionConfig;
|
||||
import org.apache.mina.transport.socket.nio.NioProcessor;
|
||||
import org.apache.mina.transport.socket.nio.NioSession;
|
||||
import org.apache.mina.transport.socket.nio.NioSocketAcceptor;
|
||||
import java.util.Arrays;
|
||||
import org.bson.BSONObject;
|
||||
import org.bson.BasicBSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.fangjia.datastore.config.DataStoreConfig;
|
||||
import com.fangjia.datastore.handler.DeleteMessageHandler;
|
||||
import com.fangjia.datastore.handler.GetMoreMessageHandler;
|
||||
import com.fangjia.datastore.handler.InsertMessageHandler;
|
||||
import com.fangjia.datastore.handler.KillCursorsMessageHandler;
|
||||
import com.fangjia.datastore.handler.MessageHandler;
|
||||
import com.fangjia.datastore.handler.QueryMessageHandler;
|
||||
import com.fangjia.datastore.mongo.protocol.ReplyMessage;
|
||||
import org.apache.mina.filter.codec.ProtocolCodecException;
|
||||
import java.io.IOException;
|
||||
import org.apache.mina.core.session.IoSession;
|
||||
import org.apache.mina.transport.socket.SocketSessionConfig;
|
||||
import java.net.SocketAddress;
|
||||
import java.net.InetSocketAddress;
|
||||
import org.apache.mina.core.session.IdleStatus;
|
||||
import org.apache.mina.core.service.IoHandler;
|
||||
import com.fangjia.datastore.handler.UpdateMessageHandler;
|
||||
import com.fangjia.datastore.mongo.codec.BSONIoBufferAllocator;
|
||||
import com.fangjia.datastore.mongo.protocol.UpdateMessage;
|
||||
import com.fangjia.datastore.handler.DeleteMessageHandler;
|
||||
import com.fangjia.datastore.mongo.protocol.DeleteMessage;
|
||||
import com.fangjia.datastore.handler.KillCursorsMessageHandler;
|
||||
import com.fangjia.datastore.mongo.protocol.KillCursorsMessage;
|
||||
import com.fangjia.datastore.handler.GetMoreMessageHandler;
|
||||
import com.fangjia.datastore.mongo.protocol.GetMoreMessage;
|
||||
import com.fangjia.datastore.handler.QueryMessageHandler;
|
||||
import com.fangjia.datastore.mongo.protocol.QueryMessage;
|
||||
import com.fangjia.datastore.handler.InsertMessageHandler;
|
||||
import com.fangjia.datastore.mongo.protocol.InsertMessage;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import org.apache.mina.filter.logging.LogLevel;
|
||||
import org.apache.mina.filter.logging.LoggingFilter;
|
||||
import org.apache.mina.core.filterchain.IoFilter;
|
||||
import org.apache.mina.filter.codec.ProtocolDecoder;
|
||||
import org.apache.mina.filter.codec.ProtocolEncoder;
|
||||
import org.apache.mina.filter.codec.ProtocolCodecFilter;
|
||||
import com.fangjia.datastore.mongo.codec.MongoProtocolDecoder;
|
||||
import com.fangjia.datastore.mongo.codec.MongoProtocolEncoder;
|
||||
import com.fangjia.datastore.mongo.protocol.DeleteMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.GetMoreMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.InsertMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.KillCursorsMessage;
|
||||
import org.apache.mina.core.service.IoProcessor;
|
||||
import java.util.concurrent.Executor;
|
||||
import org.apache.mina.core.service.SimpleIoProcessorPool;
|
||||
import org.apache.mina.transport.socket.nio.NioProcessor;
|
||||
import java.util.concurrent.Executors;
|
||||
import org.apache.mina.core.buffer.IoBufferAllocator;
|
||||
import org.apache.mina.core.buffer.IoBuffer;
|
||||
import com.fangjia.datastore.mongo.codec.BSONIoBufferAllocator;
|
||||
import com.fangjia.datastore.config.DataStoreConfig;
|
||||
import java.util.HashMap;
|
||||
import com.fangjia.datastore.handler.MessageHandler;
|
||||
import java.util.Map;
|
||||
import com.fangjia.datastore.mongo.protocol.MongoMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.QueryMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.ReplyMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.UpdateMessage;
|
||||
|
||||
public class DataStoreServer extends IoHandlerAdapter implements IoServiceListener {
|
||||
|
||||
private final NioSocketAcceptor acceptor;
|
||||
private final DataStoreContext context;
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(DataStoreServer.class);
|
||||
private static ThreadLocal<MongoMessage> CURRENT_MESSAGE = new ThreadLocal<MongoMessage>();
|
||||
private final Map<Class<? extends MongoMessage>, MessageHandler<? extends MongoMessage>> handlerMap = new HashMap<Class<? extends MongoMessage>, MessageHandler<? extends MongoMessage>>();
|
||||
|
||||
public DataStoreServer(String[] args) throws Exception {
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("starting datastore...");
|
||||
}
|
||||
|
||||
DataStoreConfig config = new DataStoreConfig(System.getProperty("datastore.home", "d:\\goojiaconfig\\datastore"));
|
||||
config.setReadOnly(Boolean.getBoolean("datastore.readOnly"));
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info(config.toString());
|
||||
}
|
||||
context = new DataStoreContextImpl(config);
|
||||
|
||||
IoBuffer.setAllocator(new BSONIoBufferAllocator());
|
||||
|
||||
final ExecutorService executorService = Executors.newCachedThreadPool();
|
||||
acceptor = new NioSocketAcceptor(executorService, new SimpleIoProcessorPool<NioSession>(NioProcessor.class, config
|
||||
.getServerConfig().getMaxConnections()));
|
||||
|
||||
acceptor.getFilterChain().addLast("codec",
|
||||
new ProtocolCodecFilter(new MongoProtocolEncoder(), new MongoProtocolDecoder()));
|
||||
acceptor.getFilterChain().addLast("performance",
|
||||
new PerformanceLogFilter(config.getServerConfig().getPerformanceLogFilterMaxTime()));
|
||||
if (!config.getServerConfig().isDisableLogging()) {
|
||||
LoggingFilter loggingFilter = new LoggingFilter(DataStoreServer.class);
|
||||
loggingFilter.setExceptionCaughtLogLevel(LogLevel.NONE);
|
||||
acceptor.getFilterChain().addLast("logger", loggingFilter);
|
||||
}
|
||||
|
||||
Runtime.getRuntime().addShutdownHook(new Thread() {
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
LOGGER.info("shutting down...");
|
||||
acceptor.unbind();
|
||||
executorService.shutdown();
|
||||
LOGGER.info("shut down");
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
// register message handlers
|
||||
register(InsertMessage.class, new InsertMessageHandler(context));
|
||||
register(QueryMessage.class, new QueryMessageHandler(context));
|
||||
register(GetMoreMessage.class, new GetMoreMessageHandler(context));
|
||||
register(KillCursorsMessage.class, new KillCursorsMessageHandler(context));
|
||||
register(DeleteMessage.class, new DeleteMessageHandler(context));
|
||||
register(UpdateMessage.class, new UpdateMessageHandler(context));
|
||||
|
||||
acceptor.setHandler(this);
|
||||
acceptor.addListener(this);
|
||||
acceptor.setReuseAddress(true);
|
||||
SocketSessionConfig sessionConfig = acceptor.getSessionConfig();
|
||||
sessionConfig.setTcpNoDelay(true);
|
||||
sessionConfig.setKeepAlive(true);
|
||||
sessionConfig.setReuseAddress(true);
|
||||
sessionConfig.setReadBufferSize(config.getServerConfig().getMinaReadBufferSize());
|
||||
sessionConfig.setIdleTime(IdleStatus.BOTH_IDLE, 300);
|
||||
acceptor.bind(new InetSocketAddress(config.getServerConfig().getPort()));
|
||||
}
|
||||
|
||||
private void register(Class<? extends MongoMessage> messageClass, MessageHandler<? extends MongoMessage> handler) {
|
||||
this.handlerMap.put(messageClass, handler);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param args
|
||||
* @throws Exception
|
||||
* @throws
|
||||
*/
|
||||
public static void main(String[] args) throws Exception {
|
||||
new DataStoreServer(args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exceptionCaught(IoSession session, Throwable cause) throws Exception {
|
||||
if (cause instanceof IOException && null != cause.getMessage() && cause.getMessage().contains("Connection reset by peer")) {
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("connection closed on session: {}", session);
|
||||
}
|
||||
} else if (cause instanceof ProtocolCodecException) {
|
||||
// ProtocolDecoderException or ProtocolEncoderException
|
||||
MongoMessage message = CURRENT_MESSAGE.get();
|
||||
if (null != message) {
|
||||
if (message instanceof QueryMessage) {
|
||||
QueryMessage queryMessage = (QueryMessage) message;
|
||||
ReplyMessage reply = new ReplyMessage();
|
||||
reply.setNumberReturned(1);
|
||||
reply.setResponseFlag(8);
|
||||
reply.getMessageHeader().setResponseTo(queryMessage.getMessageHeader().getRequestID());
|
||||
BSONObject msg = new BasicBSONObject("$err", cause.getMessage());
|
||||
reply.setDocuments(Arrays.asList(msg));
|
||||
session.write(reply);
|
||||
}
|
||||
}
|
||||
} else if (LOGGER.isErrorEnabled()) {
|
||||
LOGGER.error("exception caught on session:{" + session + "}", cause);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void messageReceived(IoSession session, Object message) throws Exception {
|
||||
if (message instanceof MongoMessage) {
|
||||
MongoMessage mongoMessage = (MongoMessage) message;
|
||||
CURRENT_MESSAGE.set(mongoMessage);
|
||||
MessageHandler<? extends MongoMessage> handler = this.handlerMap.get(mongoMessage.getClass());
|
||||
if (null != handler) {
|
||||
handler.messageReceived(session, mongoMessage);
|
||||
} else {
|
||||
throw new RuntimeException("unknown message received:" + message);
|
||||
}
|
||||
} else {
|
||||
throw new RuntimeException("unknown message received:" + message);
|
||||
}
|
||||
}
|
||||
|
||||
public void serviceActivated(IoService service) throws Exception {
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("datastore started");
|
||||
LOGGER.info("listening at " + this.context.getDataStoreConfig().getServerConfig().getPort());
|
||||
}
|
||||
}
|
||||
|
||||
public void serviceIdle(IoService service, IdleStatus idleStatus) throws Exception {
|
||||
// empty
|
||||
}
|
||||
|
||||
public void serviceDeactivated(IoService service) throws Exception {
|
||||
// empty
|
||||
}
|
||||
|
||||
public void sessionDestroyed(IoSession session) throws Exception {
|
||||
// empty
|
||||
}
|
||||
import org.slf4j.Logger;
|
||||
import org.apache.mina.transport.socket.nio.NioSocketAcceptor;
|
||||
import org.apache.mina.core.service.IoServiceListener;
|
||||
import org.apache.mina.core.service.IoHandlerAdapter;
|
||||
|
||||
public class DataStoreServer extends IoHandlerAdapter implements IoServiceListener
|
||||
{
|
||||
private final NioSocketAcceptor acceptor;
|
||||
private final DataStoreContext context;
|
||||
private static final Logger LOGGER;
|
||||
private static ThreadLocal<MongoMessage> CURRENT_MESSAGE;
|
||||
private final Map<Class<? extends MongoMessage>, MessageHandler<? extends MongoMessage>> handlerMap;
|
||||
|
||||
public DataStoreServer(final String[] args) throws Exception {
|
||||
this.handlerMap = new HashMap<Class<? extends MongoMessage>, MessageHandler<? extends MongoMessage>>();
|
||||
if (DataStoreServer.LOGGER.isInfoEnabled()) {
|
||||
DataStoreServer.LOGGER.info("starting datastore...");
|
||||
}
|
||||
final DataStoreConfig config = new DataStoreConfig(System.getProperty("datastore.home", "D:\\goojiaconfig\\seaweedsolr6"));
|
||||
config.setReadOnly(Boolean.getBoolean("datastore.readOnly"));
|
||||
if (DataStoreServer.LOGGER.isInfoEnabled()) {
|
||||
DataStoreServer.LOGGER.info(config.toString());
|
||||
}
|
||||
this.context = new DataStoreContextImpl(config);
|
||||
IoBuffer.setAllocator(new BSONIoBufferAllocator());
|
||||
final ExecutorService executorService = Executors.newCachedThreadPool();
|
||||
this.acceptor = new NioSocketAcceptor(executorService, new SimpleIoProcessorPool(NioProcessor.class, config.getServerConfig().getMaxConnections()));
|
||||
this.acceptor.getFilterChain().addLast("codec", new ProtocolCodecFilter(new MongoProtocolEncoder(), new MongoProtocolDecoder()));
|
||||
this.acceptor.getFilterChain().addLast("performance", new PerformanceLogFilter(config.getServerConfig().getPerformanceLogFilterMaxTime()));
|
||||
if (!config.getServerConfig().isDisableLogging()) {
|
||||
final LoggingFilter loggingFilter = new LoggingFilter(DataStoreServer.class);
|
||||
loggingFilter.setExceptionCaughtLogLevel(LogLevel.NONE);
|
||||
this.acceptor.getFilterChain().addLast("logger", loggingFilter);
|
||||
}
|
||||
Runtime.getRuntime().addShutdownHook(new Thread() {
|
||||
@Override
|
||||
public void run() {
|
||||
DataStoreServer.LOGGER.info("shutting down...");
|
||||
DataStoreServer.this.acceptor.unbind();
|
||||
executorService.shutdown();
|
||||
DataStoreServer.LOGGER.info("shut down");
|
||||
}
|
||||
});
|
||||
this.register(InsertMessage.class, new InsertMessageHandler(this.context));
|
||||
this.register(QueryMessage.class, new QueryMessageHandler(this.context));
|
||||
this.register(GetMoreMessage.class, new GetMoreMessageHandler(this.context));
|
||||
this.register(KillCursorsMessage.class, new KillCursorsMessageHandler(this.context));
|
||||
this.register(DeleteMessage.class, new DeleteMessageHandler(this.context));
|
||||
this.register(UpdateMessage.class, new UpdateMessageHandler(this.context));
|
||||
this.acceptor.setHandler(this);
|
||||
this.acceptor.addListener(this);
|
||||
this.acceptor.setReuseAddress(true);
|
||||
final SocketSessionConfig sessionConfig = this.acceptor.getSessionConfig();
|
||||
sessionConfig.setTcpNoDelay(true);
|
||||
sessionConfig.setKeepAlive(true);
|
||||
sessionConfig.setReuseAddress(true);
|
||||
sessionConfig.setReadBufferSize(config.getServerConfig().getMinaReadBufferSize());
|
||||
sessionConfig.setIdleTime(IdleStatus.BOTH_IDLE, 300);
|
||||
this.acceptor.bind(new InetSocketAddress(config.getServerConfig().getPort()));
|
||||
}
|
||||
|
||||
private void register(final Class<? extends MongoMessage> messageClass, final MessageHandler<? extends MongoMessage> handler) {
|
||||
this.handlerMap.put(messageClass, handler);
|
||||
}
|
||||
|
||||
public static void main(final String[] args) throws Exception {
|
||||
new DataStoreServer(args);
|
||||
}
|
||||
|
||||
public void exceptionCaught(final IoSession session, final Throwable cause) throws Exception {
|
||||
if (cause instanceof IOException && null != cause.getMessage() && cause.getMessage().contains("Connection reset by peer")) {
|
||||
if (DataStoreServer.LOGGER.isInfoEnabled()) {
|
||||
DataStoreServer.LOGGER.info("connection closed on session: {}", (Object)session);
|
||||
}
|
||||
}
|
||||
else if (cause instanceof ProtocolCodecException) {
|
||||
final MongoMessage message = DataStoreServer.CURRENT_MESSAGE.get();
|
||||
if (null != message && message instanceof QueryMessage) {
|
||||
final QueryMessage queryMessage = (QueryMessage)message;
|
||||
final ReplyMessage reply = new ReplyMessage();
|
||||
reply.setNumberReturned(1);
|
||||
reply.setResponseFlag(8);
|
||||
reply.getMessageHeader().setResponseTo(queryMessage.getMessageHeader().getRequestID());
|
||||
final BSONObject msg = (BSONObject)new BasicBSONObject("$err", cause.getMessage());
|
||||
reply.setDocuments(Arrays.<BSONObject>asList(msg));
|
||||
session.write((Object)reply);
|
||||
}
|
||||
}
|
||||
else if (DataStoreServer.LOGGER.isErrorEnabled()) {
|
||||
DataStoreServer.LOGGER.error("exception caught on session:{" + session + "}", cause);
|
||||
}
|
||||
}
|
||||
|
||||
public void messageReceived(final IoSession session, final Object message) throws Exception {
|
||||
if (!(message instanceof MongoMessage)) {
|
||||
throw new RuntimeException("unknown message received:" + message);
|
||||
}
|
||||
final MongoMessage mongoMessage = (MongoMessage)message;
|
||||
DataStoreServer.CURRENT_MESSAGE.set(mongoMessage);
|
||||
final MessageHandler<? extends MongoMessage> handler = this.handlerMap.get(mongoMessage.getClass());
|
||||
if (null != handler) {
|
||||
handler.messageReceived(session, mongoMessage);
|
||||
return;
|
||||
}
|
||||
throw new RuntimeException("unknown message received:" + message);
|
||||
}
|
||||
|
||||
public void serviceActivated(final IoService service) throws Exception {
|
||||
if (DataStoreServer.LOGGER.isInfoEnabled()) {
|
||||
DataStoreServer.LOGGER.info("datastore started");
|
||||
DataStoreServer.LOGGER.info("listening at " + this.context.getDataStoreConfig().getServerConfig().getPort());
|
||||
}
|
||||
}
|
||||
|
||||
public void serviceIdle(final IoService service, final IdleStatus idleStatus) throws Exception {
|
||||
}
|
||||
|
||||
public void serviceDeactivated(final IoService service) throws Exception {
|
||||
}
|
||||
|
||||
public void sessionDestroyed(final IoSession session) throws Exception {
|
||||
}
|
||||
|
||||
static {
|
||||
LOGGER = LoggerFactory.getLogger((Class)DataStoreServer.class);
|
||||
DataStoreServer.CURRENT_MESSAGE = new ThreadLocal<MongoMessage>();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,64 +1,22 @@
|
||||
package com.fangjia.datastore;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.bson.BSONObject;
|
||||
import java.util.Set;
|
||||
import org.bson.types.ObjectId;
|
||||
|
||||
import com.mongodb.DBObject;
|
||||
import java.util.List;
|
||||
|
||||
public interface FileStore {
|
||||
public final static String DEFAULT_CHARSET = "UTF-8";
|
||||
|
||||
/**
|
||||
*
|
||||
* @param fileList
|
||||
* 仅包含大字段的文档列表
|
||||
* @param dbObjList
|
||||
* 已经保存到数据库的主对象列表
|
||||
* @throws IllegalArgumentException
|
||||
* 大字段数据不支持默认编码时抛出
|
||||
*/
|
||||
void save(List<DBObject> fileList, List<DBObject> dbObjList) throws DataStoreException;
|
||||
|
||||
/**
|
||||
* 删除指定的大字段内容
|
||||
*
|
||||
* @param objIdList
|
||||
* 删除的主对象的id列表
|
||||
* @param removeFields
|
||||
* 需要删除的大字段名集合
|
||||
*/
|
||||
public void remove(List<ObjectId> objIdList, Set<String> removeFields);
|
||||
|
||||
/**
|
||||
* 删除所有大字段内容
|
||||
*
|
||||
* @param objIdList
|
||||
*/
|
||||
public void remove(List<ObjectId> objIdList);
|
||||
|
||||
/**
|
||||
* @param objIdList
|
||||
* 待更新的主对象id列表
|
||||
* @param data
|
||||
* 更新文档
|
||||
* @throws UnsupportedEncodingException
|
||||
* 大字段数据不支持默认编码时抛出
|
||||
*/
|
||||
public void update(List<ObjectId> objIdList, BSONObject data) throws DataStoreException;
|
||||
|
||||
/**
|
||||
* @param docList
|
||||
* 查询的结果文档
|
||||
* @param fields
|
||||
* 查询字段对象
|
||||
* @return
|
||||
* @throws IOException
|
||||
* 读取大字段数据异常时、大字段数据不支持默认编码时抛出
|
||||
*/
|
||||
public List<DBObject> find(List<DBObject> docList, BSONObject fields) throws DataStoreException;
|
||||
public interface FileStore
|
||||
{
|
||||
public static final String DEFAULT_CHARSET = "UTF-8";
|
||||
|
||||
void save(final List<DBObject> p0, final List<DBObject> p1) throws DataStoreException;
|
||||
|
||||
void remove(final List<ObjectId> p0, final Set<String> p1);
|
||||
|
||||
void remove(final List<ObjectId> p0);
|
||||
|
||||
void update(final List<ObjectId> p0, final BSONObject p1) throws DataStoreException;
|
||||
|
||||
List<DBObject> find(final List<DBObject> p0, final BSONObject p1) throws DataStoreException;
|
||||
}
|
||||
|
||||
+219
-239
@@ -1,249 +1,229 @@
|
||||
package com.fangjia.datastore;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import com.mongodb.gridfs.GridFSDBFile;
|
||||
import java.util.HashSet;
|
||||
import com.mongodb.BasicDBObject;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Set;
|
||||
import java.io.OutputStream;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import com.mongodb.gridfs.GridFSFile;
|
||||
import java.util.Date;
|
||||
import java.io.IOException;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import com.fangjia.datastore.config.FieldTypeEnum;
|
||||
import org.bson.BSONObject;
|
||||
import org.bson.types.ObjectId;
|
||||
|
||||
import com.fangjia.datastore.config.Field;
|
||||
import com.fangjia.datastore.config.FieldTypeEnum;
|
||||
import com.mongodb.BasicDBObject;
|
||||
import com.mongodb.DBObject;
|
||||
import java.util.List;
|
||||
import java.util.Iterator;
|
||||
import java.util.HashMap;
|
||||
import java.util.Collection;
|
||||
import com.fangjia.datastore.config.Field;
|
||||
import java.util.Map;
|
||||
import com.mongodb.gridfs.GridFS;
|
||||
import com.mongodb.gridfs.GridFSDBFile;
|
||||
import com.mongodb.gridfs.GridFSFile;
|
||||
|
||||
public class FileStoreImpl implements FileStore {
|
||||
|
||||
private GridFS gridFS;
|
||||
private Map<String, Field> bigFieldSet;
|
||||
|
||||
public FileStoreImpl(GridFS gridFS, Collection<Field> fields) {
|
||||
this.gridFS = gridFS;
|
||||
|
||||
// 把大字段的字段名放入set
|
||||
bigFieldSet = new HashMap<String, Field>();
|
||||
for (Field field : fields) {
|
||||
if (field.isBigField()) {
|
||||
bigFieldSet.put(field.getName(), field);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void save(List<DBObject> fileList, List<DBObject> dbObjList) {
|
||||
if (null == fileList || null == dbObjList || fileList.size() != dbObjList.size())
|
||||
throw new IllegalArgumentException("empty list or unequal length list is not allowed!");
|
||||
|
||||
Iterator<DBObject> dbObjIt = dbObjList.iterator();
|
||||
for (Iterator<DBObject> fileIt = fileList.iterator(); fileIt.hasNext();) {
|
||||
// 取得已保存主对象的id
|
||||
String id = ((ObjectId) dbObjIt.next().get("_id")).toString();
|
||||
|
||||
BSONObject file = fileIt.next();
|
||||
for (String fieldName : file.keySet()) {
|
||||
byte[] bytes = null;
|
||||
|
||||
Field field = bigFieldSet.get(fieldName);
|
||||
if (null == field) {
|
||||
continue;
|
||||
}
|
||||
FieldTypeEnum fieldType = field.getType();
|
||||
if (FieldTypeEnum.TEXT == fieldType) {
|
||||
String value = (String) file.get(fieldName);
|
||||
if (!StringUtils.isBlank(value)) {
|
||||
try {
|
||||
bytes = gzip(value);
|
||||
} catch (IOException e) {
|
||||
throw new IllegalArgumentException("unsupported encoding value!");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (null != bytes && bytes.length > 0) {
|
||||
GridFSFile mongofile = gridFS.createFile(bytes);
|
||||
mongofile.put("uploadDate", new Date());
|
||||
mongofile.put("filename", id);
|
||||
mongofile.put("contentType", fieldName);
|
||||
mongofile.save();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] gzip(String string) throws IOException {
|
||||
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||
GZIPOutputStream gzipOutputStream = new GZIPOutputStream(outputStream);
|
||||
gzipOutputStream.write(string.getBytes(DEFAULT_CHARSET));
|
||||
return outputStream.toByteArray();
|
||||
}
|
||||
|
||||
public void remove(List<ObjectId> objIdList) {
|
||||
remove(objIdList, null);
|
||||
}
|
||||
|
||||
public void remove(List<ObjectId> objIdList, Set<String> removeFields) {
|
||||
// 删除表的所有大字段
|
||||
if (null == removeFields)
|
||||
removeFields = bigFieldSet.keySet();
|
||||
|
||||
// 若到删除的字段中没有大字段,直接返回
|
||||
if (removeFields.size() == 0)
|
||||
return;
|
||||
|
||||
List<String> idList = new ArrayList<String>(objIdList.size());
|
||||
for (ObjectId objId : objIdList) {
|
||||
idList.add(objId.toString());
|
||||
}
|
||||
|
||||
// 删除大字段关联的GridFS
|
||||
DBObject fileSeletor = new BasicDBObject();
|
||||
fileSeletor.put("filename", new BasicDBObject("$in", idList));
|
||||
fileSeletor.put("contentType", new BasicDBObject("$in", removeFields));
|
||||
gridFS.remove(fileSeletor);
|
||||
}
|
||||
|
||||
public void update(List<ObjectId> objIdList, BSONObject data) throws DataStoreException {
|
||||
// 待删除的字段
|
||||
BSONObject unset = (BSONObject) data.get("$unset");
|
||||
if (null != unset) {
|
||||
// 待删除的大字段set
|
||||
Set<String> removeSet = new HashSet<String>();
|
||||
for (String field : unset.keySet()) {
|
||||
if (bigFieldSet.containsKey(field))
|
||||
removeSet.add(field);
|
||||
}
|
||||
|
||||
if (removeSet.size() > 0)
|
||||
remove(objIdList, removeSet);
|
||||
}
|
||||
|
||||
// 更新数据
|
||||
BSONObject set = (BSONObject) data.get("$set");
|
||||
if (null == set)
|
||||
return;
|
||||
|
||||
Map<String, Object> fileMap = new HashMap<String, Object>();
|
||||
for (String field : set.keySet()) {
|
||||
if (bigFieldSet.containsKey(field)) {
|
||||
fileMap.put(field, set.get(field));
|
||||
}
|
||||
}
|
||||
|
||||
// 更新数据中没有大字段
|
||||
if (fileMap.size() == 0)
|
||||
return;
|
||||
|
||||
// 删除大字段关联的GridFS
|
||||
remove(objIdList, fileMap.keySet());
|
||||
|
||||
// 将新的内容存入GridFS
|
||||
for (ObjectId objId : objIdList) {
|
||||
String id = objId.toString();
|
||||
for (String fieldName : fileMap.keySet()) {
|
||||
byte[] bytes = null;
|
||||
|
||||
Field field = bigFieldSet.get(fieldName);
|
||||
if (null == field) {
|
||||
continue;
|
||||
}
|
||||
FieldTypeEnum fieldType = field.getType();
|
||||
if (FieldTypeEnum.TEXT == fieldType) {
|
||||
String value = (String) fileMap.get(fieldName);
|
||||
if (!StringUtils.isBlank(value)) {
|
||||
try {
|
||||
bytes = gzip(value);
|
||||
} catch (IOException e) {
|
||||
throw new IllegalArgumentException("unsupported encoding value!");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (null != bytes && bytes.length > 0) {
|
||||
GridFSFile mongofile = gridFS.createFile(bytes);
|
||||
mongofile.put("uploadDate", new Date());
|
||||
mongofile.put("filename", id);
|
||||
mongofile.put("contentType", fieldName);
|
||||
mongofile.save();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public List<DBObject> find(List<DBObject> docList, BSONObject fields) throws DataStoreException {
|
||||
// 判断查询的字段中有没有大字段
|
||||
Set<String> queryBigField = new HashSet<String>();
|
||||
if (null == fields) {
|
||||
queryBigField = bigFieldSet.keySet();
|
||||
} else {
|
||||
for (String field : fields.keySet()) {
|
||||
if (bigFieldSet.containsKey(field)) {
|
||||
queryBigField.add(field);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 没有大字段,则返回原结果文档
|
||||
if (queryBigField.size() == 0)
|
||||
return docList;
|
||||
|
||||
Map<String, DBObject> docMap = new HashMap<String, DBObject>(docList.size());
|
||||
List<String> idList = new ArrayList<String>(docList.size());
|
||||
for (DBObject dbObj : docList) {
|
||||
ObjectId objId = (ObjectId) dbObj.get("_id");
|
||||
if (null != objId) {
|
||||
String id = objId.toString();
|
||||
idList.add(id);
|
||||
docMap.put(id, dbObj);
|
||||
}
|
||||
}
|
||||
|
||||
DBObject fileSeletor = new BasicDBObject();
|
||||
fileSeletor.put("filename", new BasicDBObject("$in", idList));
|
||||
fileSeletor.put("contentType", new BasicDBObject("$in", queryBigField));
|
||||
List<GridFSDBFile> fileList = gridFS.find(fileSeletor);
|
||||
|
||||
for (GridFSDBFile file : fileList) {
|
||||
// 对应主记录id
|
||||
String id = file.getFilename();
|
||||
// 对应主记录大字段名
|
||||
String fieldName = file.getContentType();
|
||||
|
||||
DBObject obj = docMap.get(id);
|
||||
if (obj.keySet().contains(fieldName)) {
|
||||
|
||||
try {
|
||||
ByteArrayOutputStream data = new ByteArrayOutputStream();
|
||||
GZIPOutputStream out = new GZIPOutputStream(data);
|
||||
IOUtils.copy(file.getInputStream(), out);
|
||||
|
||||
Field field = bigFieldSet.get(fieldName);
|
||||
if (null == field) {
|
||||
continue;
|
||||
}
|
||||
FieldTypeEnum fieldType = field.getType();
|
||||
if (FieldTypeEnum.TEXT == fieldType) {
|
||||
obj.put(fieldName, new String(data.toByteArray(), DEFAULT_CHARSET));
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new DataStoreException("error extract text field[" + fieldName + "]:", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return docList;
|
||||
}
|
||||
public class FileStoreImpl implements FileStore
|
||||
{
|
||||
private GridFS gridFS;
|
||||
private Map<String, Field> bigFieldSet;
|
||||
|
||||
public FileStoreImpl(final GridFS gridFS, final Collection<Field> fields) {
|
||||
this.gridFS = gridFS;
|
||||
this.bigFieldSet = new HashMap<String, Field>();
|
||||
for (final Field field : fields) {
|
||||
if (field.isBigField()) {
|
||||
this.bigFieldSet.put(field.getName(), field);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void save(final List<DBObject> fileList, final List<DBObject> dbObjList) {
|
||||
if (null == fileList || null == dbObjList || fileList.size() != dbObjList.size()) {
|
||||
throw new IllegalArgumentException("empty list or unequal length list is not allowed!");
|
||||
}
|
||||
final Iterator<DBObject> dbObjIt = dbObjList.iterator();
|
||||
final Iterator<DBObject> fileIt = fileList.iterator();
|
||||
while (fileIt.hasNext()) {
|
||||
final String id = ((ObjectId)dbObjIt.next().get("_id")).toString();
|
||||
final BSONObject file = (BSONObject)fileIt.next();
|
||||
for (final String fieldName : file.keySet()) {
|
||||
byte[] bytes = null;
|
||||
final Field field = this.bigFieldSet.get(fieldName);
|
||||
if (null == field) {
|
||||
continue;
|
||||
}
|
||||
final FieldTypeEnum fieldType = field.getType();
|
||||
if (FieldTypeEnum.TEXT == fieldType) {
|
||||
final String value = (String)file.get(fieldName);
|
||||
if (!StringUtils.isBlank(value)) {
|
||||
try {
|
||||
bytes = this.gzip(value);
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new IllegalArgumentException("unsupported encoding value!");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (null == bytes || bytes.length <= 0) {
|
||||
continue;
|
||||
}
|
||||
final GridFSFile mongofile = (GridFSFile)this.gridFS.createFile(bytes);
|
||||
mongofile.put("uploadDate", (Object)new Date());
|
||||
mongofile.put("filename", (Object)id);
|
||||
mongofile.put("contentType", (Object)fieldName);
|
||||
mongofile.save();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] gzip(final String string) throws IOException {
|
||||
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||
final GZIPOutputStream gzipOutputStream = new GZIPOutputStream(outputStream);
|
||||
gzipOutputStream.write(string.getBytes("UTF-8"));
|
||||
return outputStream.toByteArray();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove(final List<ObjectId> objIdList) {
|
||||
this.remove(objIdList, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove(final List<ObjectId> objIdList, Set<String> removeFields) {
|
||||
if (null == removeFields) {
|
||||
removeFields = this.bigFieldSet.keySet();
|
||||
}
|
||||
if (removeFields.size() == 0) {
|
||||
return;
|
||||
}
|
||||
final List<String> idList = new ArrayList<String>(objIdList.size());
|
||||
for (final ObjectId objId : objIdList) {
|
||||
idList.add(objId.toString());
|
||||
}
|
||||
final DBObject fileSeletor = (DBObject)new BasicDBObject();
|
||||
fileSeletor.put("filename", (Object)new BasicDBObject("$in", (Object)idList));
|
||||
fileSeletor.put("contentType", (Object)new BasicDBObject("$in", (Object)removeFields));
|
||||
this.gridFS.remove(fileSeletor);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void update(final List<ObjectId> objIdList, final BSONObject data) throws DataStoreException {
|
||||
final BSONObject unset = (BSONObject)data.get("$unset");
|
||||
if (null != unset) {
|
||||
final Set<String> removeSet = new HashSet<String>();
|
||||
for (final String field : unset.keySet()) {
|
||||
if (this.bigFieldSet.containsKey(field)) {
|
||||
removeSet.add(field);
|
||||
}
|
||||
}
|
||||
if (removeSet.size() > 0) {
|
||||
this.remove(objIdList, removeSet);
|
||||
}
|
||||
}
|
||||
final BSONObject set = (BSONObject)data.get("$set");
|
||||
if (null == set) {
|
||||
return;
|
||||
}
|
||||
final Map<String, Object> fileMap = new HashMap<String, Object>();
|
||||
for (final String field2 : set.keySet()) {
|
||||
if (this.bigFieldSet.containsKey(field2)) {
|
||||
fileMap.put(field2, set.get(field2));
|
||||
}
|
||||
}
|
||||
if (fileMap.size() == 0) {
|
||||
return;
|
||||
}
|
||||
this.remove(objIdList, fileMap.keySet());
|
||||
for (final ObjectId objId : objIdList) {
|
||||
final String id = objId.toString();
|
||||
for (final String fieldName : fileMap.keySet()) {
|
||||
byte[] bytes = null;
|
||||
final Field field3 = this.bigFieldSet.get(fieldName);
|
||||
if (null == field3) {
|
||||
continue;
|
||||
}
|
||||
final FieldTypeEnum fieldType = field3.getType();
|
||||
if (FieldTypeEnum.TEXT == fieldType) {
|
||||
final String value = (String)fileMap.get(fieldName);
|
||||
if (!StringUtils.isBlank(value)) {
|
||||
try {
|
||||
bytes = this.gzip(value);
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new IllegalArgumentException("unsupported encoding value!");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (null == bytes || bytes.length <= 0) {
|
||||
continue;
|
||||
}
|
||||
final GridFSFile mongofile = (GridFSFile)this.gridFS.createFile(bytes);
|
||||
mongofile.put("uploadDate", (Object)new Date());
|
||||
mongofile.put("filename", (Object)id);
|
||||
mongofile.put("contentType", (Object)fieldName);
|
||||
mongofile.save();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DBObject> find(final List<DBObject> docList, final BSONObject fields) throws DataStoreException {
|
||||
Set<String> queryBigField = new HashSet<String>();
|
||||
if (null == fields) {
|
||||
queryBigField = this.bigFieldSet.keySet();
|
||||
}
|
||||
else {
|
||||
for (final String field : fields.keySet()) {
|
||||
if (this.bigFieldSet.containsKey(field)) {
|
||||
queryBigField.add(field);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (queryBigField.size() == 0) {
|
||||
return docList;
|
||||
}
|
||||
final Map<String, DBObject> docMap = new HashMap<String, DBObject>(docList.size());
|
||||
final List<String> idList = new ArrayList<String>(docList.size());
|
||||
for (final DBObject dbObj : docList) {
|
||||
final ObjectId objId = (ObjectId)dbObj.get("_id");
|
||||
if (null != objId) {
|
||||
final String id = objId.toString();
|
||||
idList.add(id);
|
||||
docMap.put(id, dbObj);
|
||||
}
|
||||
}
|
||||
final DBObject fileSeletor = (DBObject)new BasicDBObject();
|
||||
fileSeletor.put("filename", (Object)new BasicDBObject("$in", (Object)idList));
|
||||
fileSeletor.put("contentType", (Object)new BasicDBObject("$in", (Object)queryBigField));
|
||||
final List<GridFSDBFile> fileList = (List<GridFSDBFile>)this.gridFS.find(fileSeletor);
|
||||
for (final GridFSDBFile file : fileList) {
|
||||
final String id2 = file.getFilename();
|
||||
final String fieldName = file.getContentType();
|
||||
final DBObject obj = docMap.get(id2);
|
||||
if (obj.keySet().contains(fieldName)) {
|
||||
try {
|
||||
final ByteArrayOutputStream data = new ByteArrayOutputStream();
|
||||
final GZIPOutputStream out = new GZIPOutputStream(data);
|
||||
IOUtils.copy(file.getInputStream(), (OutputStream)out);
|
||||
final Field field2 = this.bigFieldSet.get(fieldName);
|
||||
if (null == field2) {
|
||||
continue;
|
||||
}
|
||||
final FieldTypeEnum fieldType = field2.getType();
|
||||
if (FieldTypeEnum.TEXT != fieldType) {
|
||||
continue;
|
||||
}
|
||||
obj.put(fieldName, (Object)new String(data.toByteArray(), "UTF-8"));
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new DataStoreException("error extract text field[" + fieldName + "]:", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
return docList;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,46 +1,46 @@
|
||||
package com.fangjia.datastore;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.fangjia.datastore.config.MongoConfig;
|
||||
import com.mongodb.DB;
|
||||
import com.mongodb.MongoException;
|
||||
import com.mongodb.WriteConcern;
|
||||
import com.mongodb.MongoException;
|
||||
import java.util.List;
|
||||
import com.mongodb.DB;
|
||||
import com.fangjia.datastore.config.MongoConfig;
|
||||
|
||||
public class Mongo {
|
||||
|
||||
private final MongoConfig mongoConfig;
|
||||
|
||||
public Mongo(MongoConfig mongoConfig) {
|
||||
this.mongoConfig = mongoConfig;
|
||||
}
|
||||
|
||||
private com.mongodb.Mongo getMongo() {
|
||||
try {
|
||||
return mongoConfig.getMongo();
|
||||
} catch (ConfigurationException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public DB getDB(String dbname) {
|
||||
return getMongo().getDB(dbname);
|
||||
}
|
||||
|
||||
public List<String> getDatabaseNames() throws MongoException {
|
||||
return getMongo().getDatabaseNames();
|
||||
}
|
||||
|
||||
public void setWriteConcern(WriteConcern concern) {
|
||||
getMongo().setWriteConcern(concern);
|
||||
}
|
||||
|
||||
public WriteConcern getWriteConcern() {
|
||||
return getMongo().getWriteConcern();
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
return getMongo().toString();
|
||||
}
|
||||
|
||||
public class Mongo
|
||||
{
|
||||
private final MongoConfig mongoConfig;
|
||||
|
||||
public Mongo(final MongoConfig mongoConfig) {
|
||||
this.mongoConfig = mongoConfig;
|
||||
}
|
||||
|
||||
private com.mongodb.Mongo getMongo() {
|
||||
try {
|
||||
return this.mongoConfig.getMongo();
|
||||
}
|
||||
catch (ConfigurationException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public DB getDB(final String dbname) {
|
||||
return this.getMongo().getDB(dbname);
|
||||
}
|
||||
|
||||
public List<String> getDatabaseNames() throws MongoException {
|
||||
return (List<String>)this.getMongo().getDatabaseNames();
|
||||
}
|
||||
|
||||
public void setWriteConcern(final WriteConcern concern) {
|
||||
this.getMongo().setWriteConcern(concern);
|
||||
}
|
||||
|
||||
public WriteConcern getWriteConcern() {
|
||||
return this.getMongo().getWriteConcern();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return this.getMongo().toString();
|
||||
}
|
||||
}
|
||||
|
||||
+114
-109
@@ -1,114 +1,119 @@
|
||||
package com.fangjia.datastore;
|
||||
|
||||
import org.apache.mina.core.filterchain.IoFilterAdapter;
|
||||
import org.apache.mina.core.session.IdleStatus;
|
||||
import org.apache.mina.core.session.IoSession;
|
||||
import org.apache.mina.core.write.WriteRequest;
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import com.fangjia.datastore.mongo.protocol.MongoMessage;
|
||||
import com.fangjia.datastore.utils.LoggerFactory;
|
||||
import org.apache.mina.core.session.IdleStatus;
|
||||
import org.apache.mina.core.write.WriteRequest;
|
||||
import org.apache.mina.core.filterchain.IoFilter;
|
||||
import com.fangjia.datastore.mongo.protocol.MongoMessage;
|
||||
import org.apache.mina.core.session.IoSession;
|
||||
import org.slf4j.Logger;
|
||||
import org.apache.mina.core.filterchain.IoFilterAdapter;
|
||||
|
||||
public class PerformanceLogFilter extends IoFilterAdapter {
|
||||
private final long maxTime;
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getPerformanceLogger(PerformanceLogFilter.class);
|
||||
|
||||
public PerformanceLogFilter(long maxTime) {
|
||||
this.maxTime = maxTime;
|
||||
}
|
||||
|
||||
private void logPerformace(long time, String methodName, IoSession session, Object message) {
|
||||
Object[] params;
|
||||
if (message instanceof MongoMessage) {
|
||||
MongoMessage mongoMessage = (MongoMessage) message;
|
||||
params = new Object[] { time, methodName, session, mongoMessage.toPrettyString() };
|
||||
} else {
|
||||
params = new Object[] { time, methodName, session, message };
|
||||
}
|
||||
if (time < maxTime * 3) {
|
||||
LOGGER.warn("time={}, methodName={}, session={}, message={}", params);
|
||||
} else {
|
||||
LOGGER.error("time={}, methodName={}, session={}, message={}", params);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void messageReceived(NextFilter nextFilter, IoSession session, Object message) throws Exception {
|
||||
long time = System.currentTimeMillis();
|
||||
try {
|
||||
nextFilter.messageReceived(session, message);
|
||||
} finally {
|
||||
time = System.currentTimeMillis() - time;
|
||||
if (time >= maxTime) {
|
||||
logPerformace(time, "messageReceived", session, message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void messageSent(NextFilter nextFilter, IoSession session, WriteRequest writeRequest) throws Exception {
|
||||
long time = System.currentTimeMillis();
|
||||
try {
|
||||
nextFilter.messageSent(session, writeRequest);
|
||||
} finally {
|
||||
time = System.currentTimeMillis() - time;
|
||||
if (time >= maxTime) {
|
||||
logPerformace(time, "messageSent", session, writeRequest);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void sessionCreated(NextFilter nextFilter, IoSession session) throws Exception {
|
||||
long time = System.currentTimeMillis();
|
||||
try {
|
||||
nextFilter.sessionCreated(session);
|
||||
} finally {
|
||||
time = System.currentTimeMillis() - time;
|
||||
if (time >= maxTime) {
|
||||
logPerformace(time, "sessionCreated", session, null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void sessionOpened(NextFilter nextFilter, IoSession session) throws Exception {
|
||||
long time = System.currentTimeMillis();
|
||||
try {
|
||||
nextFilter.sessionOpened(session);
|
||||
} finally {
|
||||
time = System.currentTimeMillis() - time;
|
||||
if (time >= maxTime) {
|
||||
logPerformace(time, "sessionOpened", session, null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void sessionIdle(NextFilter nextFilter, IoSession session, IdleStatus status) throws Exception {
|
||||
long time = System.currentTimeMillis();
|
||||
try {
|
||||
nextFilter.sessionIdle(session, status);
|
||||
} finally {
|
||||
time = System.currentTimeMillis() - time;
|
||||
if (time >= maxTime) {
|
||||
logPerformace(time, "sessionIdle", session, status);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void sessionClosed(NextFilter nextFilter, IoSession session) throws Exception {
|
||||
long time = System.currentTimeMillis();
|
||||
try {
|
||||
nextFilter.sessionClosed(session);
|
||||
} finally {
|
||||
time = System.currentTimeMillis() - time;
|
||||
if (time >= maxTime) {
|
||||
logPerformace(time, "sessionClosed", session, null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class PerformanceLogFilter extends IoFilterAdapter
|
||||
{
|
||||
private final long maxTime;
|
||||
private static final Logger LOGGER;
|
||||
|
||||
public PerformanceLogFilter(final long maxTime) {
|
||||
this.maxTime = maxTime;
|
||||
}
|
||||
|
||||
private void logPerformace(final long time, final String methodName, final IoSession session, final Object message) {
|
||||
Object[] params;
|
||||
if (message instanceof MongoMessage) {
|
||||
final MongoMessage mongoMessage = (MongoMessage)message;
|
||||
params = new Object[] { time, methodName, session, mongoMessage.toPrettyString() };
|
||||
}
|
||||
else {
|
||||
params = new Object[] { time, methodName, session, message };
|
||||
}
|
||||
if (time < this.maxTime * 3L) {
|
||||
PerformanceLogFilter.LOGGER.warn("time={}, methodName={}, session={}, message={}", params);
|
||||
}
|
||||
else {
|
||||
PerformanceLogFilter.LOGGER.error("time={}, methodName={}, session={}, message={}", params);
|
||||
}
|
||||
}
|
||||
|
||||
public void messageReceived(final NextFilter nextFilter, final IoSession session, final Object message) throws Exception {
|
||||
long time = System.currentTimeMillis();
|
||||
try {
|
||||
nextFilter.messageReceived(session, message);
|
||||
}
|
||||
finally {
|
||||
time = System.currentTimeMillis() - time;
|
||||
if (time >= this.maxTime) {
|
||||
this.logPerformace(time, "messageReceived", session, message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void messageSent(final NextFilter nextFilter, final IoSession session, final WriteRequest writeRequest) throws Exception {
|
||||
long time = System.currentTimeMillis();
|
||||
try {
|
||||
nextFilter.messageSent(session, writeRequest);
|
||||
}
|
||||
finally {
|
||||
time = System.currentTimeMillis() - time;
|
||||
if (time >= this.maxTime) {
|
||||
this.logPerformace(time, "messageSent", session, writeRequest);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void sessionCreated(final NextFilter nextFilter, final IoSession session) throws Exception {
|
||||
long time = System.currentTimeMillis();
|
||||
try {
|
||||
nextFilter.sessionCreated(session);
|
||||
}
|
||||
finally {
|
||||
time = System.currentTimeMillis() - time;
|
||||
if (time >= this.maxTime) {
|
||||
this.logPerformace(time, "sessionCreated", session, null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void sessionOpened(final NextFilter nextFilter, final IoSession session) throws Exception {
|
||||
long time = System.currentTimeMillis();
|
||||
try {
|
||||
nextFilter.sessionOpened(session);
|
||||
}
|
||||
finally {
|
||||
time = System.currentTimeMillis() - time;
|
||||
if (time >= this.maxTime) {
|
||||
this.logPerformace(time, "sessionOpened", session, null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void sessionIdle(final NextFilter nextFilter, final IoSession session, final IdleStatus status) throws Exception {
|
||||
long time = System.currentTimeMillis();
|
||||
try {
|
||||
nextFilter.sessionIdle(session, status);
|
||||
}
|
||||
finally {
|
||||
time = System.currentTimeMillis() - time;
|
||||
if (time >= this.maxTime) {
|
||||
this.logPerformace(time, "sessionIdle", session, status);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void sessionClosed(final NextFilter nextFilter, final IoSession session) throws Exception {
|
||||
long time = System.currentTimeMillis();
|
||||
try {
|
||||
nextFilter.sessionClosed(session);
|
||||
}
|
||||
finally {
|
||||
time = System.currentTimeMillis() - time;
|
||||
if (time >= this.maxTime) {
|
||||
this.logPerformace(time, "sessionClosed", session, null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static {
|
||||
LOGGER = LoggerFactory.getPerformanceLogger(PerformanceLogFilter.class);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,72 +1,80 @@
|
||||
package com.fangjia.datastore;
|
||||
|
||||
import java.util.Collections;
|
||||
import org.apache.solr.client.solrj.response.FacetField;
|
||||
import com.mongodb.DBObject;
|
||||
import org.bson.types.ObjectId;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.solr.client.solrj.response.FacetField;
|
||||
import org.bson.types.ObjectId;
|
||||
|
||||
import com.mongodb.DBObject;
|
||||
|
||||
public class QueryResponse {
|
||||
private List<ObjectId> idList = Collections.emptyList();
|
||||
private List<DBObject> docList = Collections.emptyList();
|
||||
private List<FacetField> facetFieldList = null;
|
||||
private int numFound;
|
||||
private int numReturned;
|
||||
private int numToSkip;
|
||||
public final static QueryResponse EMPTY_RESPONSE = new QueryResponse();
|
||||
|
||||
public List<ObjectId> getIdList() {
|
||||
return idList;
|
||||
}
|
||||
|
||||
public void setIdList(List<ObjectId> idList) {
|
||||
this.idList = idList;
|
||||
}
|
||||
|
||||
public boolean hasMore() {
|
||||
return this.numFound > numReturned + numToSkip;
|
||||
}
|
||||
|
||||
public int getNumFound() {
|
||||
return numFound;
|
||||
}
|
||||
|
||||
public void setNumFound(int numFound) {
|
||||
this.numFound = numFound;
|
||||
}
|
||||
|
||||
public int getNumReturned() {
|
||||
return numReturned;
|
||||
}
|
||||
|
||||
public void setNumReturned(int numReturned) {
|
||||
this.numReturned = numReturned;
|
||||
}
|
||||
|
||||
public int getNumToSkip() {
|
||||
return numToSkip;
|
||||
}
|
||||
|
||||
public void setNumToSkip(int numToSkip) {
|
||||
this.numToSkip = numToSkip;
|
||||
}
|
||||
|
||||
public List<FacetField> getFacetFieldList() {
|
||||
return facetFieldList;
|
||||
}
|
||||
|
||||
public void setFacetFieldList(List<FacetField> facetFieldList) {
|
||||
this.facetFieldList = facetFieldList;
|
||||
}
|
||||
|
||||
public List<DBObject> getDocList() {
|
||||
return docList;
|
||||
}
|
||||
|
||||
public void setDocList(List<DBObject> docList) {
|
||||
this.docList = docList;
|
||||
}
|
||||
|
||||
public class QueryResponse
|
||||
{
|
||||
private List<ObjectId> idList;
|
||||
private List<DBObject> docList;
|
||||
private List<FacetField> facetFieldList;
|
||||
private int numFound;
|
||||
private int numReturned;
|
||||
private int numToSkip;
|
||||
public static final QueryResponse EMPTY_RESPONSE;
|
||||
|
||||
public QueryResponse() {
|
||||
this.idList = Collections.<ObjectId>emptyList();
|
||||
this.docList = Collections.<DBObject>emptyList();
|
||||
this.facetFieldList = null;
|
||||
}
|
||||
|
||||
public List<ObjectId> getIdList() {
|
||||
return this.idList;
|
||||
}
|
||||
|
||||
public void setIdList(final List<ObjectId> idList) {
|
||||
this.idList = idList;
|
||||
}
|
||||
|
||||
public boolean hasMore() {
|
||||
return this.numFound > this.numReturned + this.numToSkip;
|
||||
}
|
||||
|
||||
public int getNumFound() {
|
||||
return this.numFound;
|
||||
}
|
||||
|
||||
public void setNumFound(final int numFound) {
|
||||
this.numFound = numFound;
|
||||
}
|
||||
|
||||
public int getNumReturned() {
|
||||
return this.numReturned;
|
||||
}
|
||||
|
||||
public void setNumReturned(final int numReturned) {
|
||||
this.numReturned = numReturned;
|
||||
}
|
||||
|
||||
public int getNumToSkip() {
|
||||
return this.numToSkip;
|
||||
}
|
||||
|
||||
public void setNumToSkip(final int numToSkip) {
|
||||
this.numToSkip = numToSkip;
|
||||
}
|
||||
|
||||
public List<FacetField> getFacetFieldList() {
|
||||
return this.facetFieldList;
|
||||
}
|
||||
|
||||
public void setFacetFieldList(final List<FacetField> facetFieldList) {
|
||||
this.facetFieldList = facetFieldList;
|
||||
}
|
||||
|
||||
public List<DBObject> getDocList() {
|
||||
return this.docList;
|
||||
}
|
||||
|
||||
public void setDocList(final List<DBObject> docList) {
|
||||
this.docList = docList;
|
||||
}
|
||||
|
||||
static {
|
||||
EMPTY_RESPONSE = new QueryResponse();
|
||||
}
|
||||
}
|
||||
|
||||
+18
-19
@@ -1,22 +1,21 @@
|
||||
package com.fangjia.datastore;
|
||||
|
||||
public class TriggerExecuteException extends Exception {
|
||||
|
||||
private static final long serialVersionUID = -1165411196743370433L;
|
||||
|
||||
public TriggerExecuteException() {
|
||||
}
|
||||
|
||||
public TriggerExecuteException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public TriggerExecuteException(Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
|
||||
public TriggerExecuteException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
|
||||
public class TriggerExecuteException extends Exception
|
||||
{
|
||||
private static final long serialVersionUID = -1165411196743370433L;
|
||||
|
||||
public TriggerExecuteException() {
|
||||
}
|
||||
|
||||
public TriggerExecuteException(final String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public TriggerExecuteException(final Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
|
||||
public TriggerExecuteException(final String message, final Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
|
||||
+150
-154
@@ -1,165 +1,161 @@
|
||||
package com.fangjia.datastore;
|
||||
|
||||
import java.io.IOException;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.apache.http.HttpRequest;
|
||||
import java.io.InputStream;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.http.HttpHost;
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.HttpStatus;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.client.params.AllClientPNames;
|
||||
import org.apache.http.conn.scheme.PlainSocketFactory;
|
||||
import org.apache.http.conn.scheme.Scheme;
|
||||
import org.apache.http.conn.scheme.SchemeRegistry;
|
||||
import com.mongodb.DBObject;
|
||||
import org.bson.BasicBSONDecoder;
|
||||
import org.apache.http.protocol.HttpContext;
|
||||
import org.apache.http.HttpHost;
|
||||
import org.apache.http.HttpEntity;
|
||||
import org.apache.http.entity.ByteArrayEntity;
|
||||
import org.sothis.core.util.GZIPUtils;
|
||||
import org.bson.BasicBSONEncoder;
|
||||
import com.mongodb.BasicDBObject;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import java.net.URL;
|
||||
import java.net.URISyntaxException;
|
||||
import java.io.IOException;
|
||||
import org.bson.BSONObject;
|
||||
import java.util.List;
|
||||
import com.fangjia.datastore.config.Trigger;
|
||||
import org.apache.http.params.HttpParams;
|
||||
import org.apache.http.conn.ClientConnectionManager;
|
||||
import org.apache.http.impl.client.DefaultHttpClient;
|
||||
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
|
||||
import org.apache.http.params.BasicHttpParams;
|
||||
import org.apache.http.params.CoreProtocolPNames;
|
||||
import org.apache.http.params.HttpParams;
|
||||
import org.apache.http.protocol.HttpContext;
|
||||
import org.bson.BSONEncoder;
|
||||
import org.bson.BSONObject;
|
||||
import org.bson.BasicBSONDecoder;
|
||||
import org.apache.http.conn.scheme.SchemeSocketFactory;
|
||||
import org.apache.http.conn.scheme.Scheme;
|
||||
import org.apache.http.conn.scheme.PlainSocketFactory;
|
||||
import org.apache.http.conn.scheme.SchemeRegistry;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.sothis.core.util.GZIPUtils;
|
||||
|
||||
import com.fangjia.datastore.config.Trigger;
|
||||
import com.mongodb.BasicDBObject;
|
||||
import com.mongodb.DBObject;
|
||||
|
||||
public class TriggerExecutor {
|
||||
|
||||
private final static int MAX_TOTAL = 200;
|
||||
private final static int MAX_FOR_ROUTE = 200;
|
||||
private final static String DEFAULT_CHARSET = "UTF-8";
|
||||
private final static int DEFAULT_PORT = 80;
|
||||
private final static int MAX_RETRY_COUNT = 10;
|
||||
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(TriggerExecutor.class);
|
||||
private final static TriggerExecutor TRIGGER_EXECUTOR = new TriggerExecutor();
|
||||
|
||||
private final HttpClient httpClient;
|
||||
|
||||
private TriggerExecutor() {
|
||||
if (LOGGER.isDebugEnabled()) {
|
||||
LOGGER.debug("init trigger executor");
|
||||
}
|
||||
SchemeRegistry schemeRegistry = new SchemeRegistry();
|
||||
schemeRegistry.register(new Scheme("http", DEFAULT_PORT, new PlainSocketFactory()));
|
||||
|
||||
HttpParams httpParams = new BasicHttpParams();
|
||||
httpParams.setParameter(CoreProtocolPNames.HTTP_CONTENT_CHARSET, DEFAULT_CHARSET);
|
||||
httpParams.setParameter(AllClientPNames.SO_TIMEOUT, 300000);
|
||||
httpParams.setParameter(AllClientPNames.CONNECTION_TIMEOUT, 60000);
|
||||
|
||||
ThreadSafeClientConnManager connManager = new ThreadSafeClientConnManager(schemeRegistry);
|
||||
connManager.setMaxTotal(MAX_TOTAL);
|
||||
connManager.setDefaultMaxPerRoute(MAX_FOR_ROUTE);
|
||||
|
||||
httpClient = new DefaultHttpClient(connManager, httpParams);
|
||||
if (LOGGER.isDebugEnabled()) {
|
||||
LOGGER.debug("trigger executor initialized[{}]", toString());
|
||||
}
|
||||
}
|
||||
|
||||
public static TriggerExecutor getInstance() {
|
||||
return TRIGGER_EXECUTOR;
|
||||
}
|
||||
|
||||
public List<BSONObject> execute(Trigger trigger, List<BSONObject> docList) throws TriggerExecuteException {
|
||||
try {
|
||||
List<BSONObject> ret = doExecute(trigger, docList);
|
||||
if (LOGGER.isDebugEnabled()) {
|
||||
LOGGER.debug("execute trigger: " + trigger + ", doc size:" + docList.size() + ", return size:" + ret.size());
|
||||
}
|
||||
return ret;
|
||||
} catch (IOException e) {
|
||||
throw new TriggerExecuteException("error execute trigger: " + trigger, e);
|
||||
} catch (URISyntaxException e) {
|
||||
throw new TriggerExecuteException("error execute trigger: " + trigger, e);
|
||||
}
|
||||
}
|
||||
|
||||
private List<BSONObject> doExecute(Trigger trigger, List<BSONObject> docList) throws IOException, URISyntaxException,
|
||||
TriggerExecuteException {
|
||||
URL url = new URL(trigger.getInterface());
|
||||
HttpPost httpPost = new HttpPost(url.toURI());
|
||||
DBObject object = new BasicDBObject();
|
||||
object.put("doc_list_size", docList.size());
|
||||
object.put("doc_list", docList);
|
||||
httpPost.setEntity(new ByteArrayEntity(GZIPUtils.makeAsByteArray(new BSONEncoder().encode(object))));
|
||||
HttpHost host = new HttpHost(url.getHost(), url.getPort() == -1 ? DEFAULT_PORT : url.getPort());
|
||||
HttpResponse httpResponse = executeWithRetry(host, httpPost, null, trigger);
|
||||
InputStream inputStream = httpResponse.getEntity().getContent();
|
||||
try {
|
||||
if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
|
||||
BSONObject returnObject = new BasicBSONDecoder().readObject(GZIPUtils.wrap(inputStream));
|
||||
if (null != returnObject) {
|
||||
List<BSONObject> ret = (List<BSONObject>) returnObject.get("doc_list");
|
||||
if (ret == null) {
|
||||
throw new TriggerExecuteException("trigger" + trigger + " return an null doc_list[" + ret + "]");
|
||||
}
|
||||
return ret;
|
||||
} else {
|
||||
throw new TriggerExecuteException("trigger" + trigger + " returns nothing at all !");
|
||||
}
|
||||
} else {
|
||||
throw new TriggerExecuteException("error execute trigger:" + trigger + ", get http code:"
|
||||
+ httpResponse.getStatusLine().getStatusCode());
|
||||
}
|
||||
} finally {
|
||||
if (null != inputStream) {
|
||||
inputStream.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private HttpResponse executeWithRetry(HttpHost host, HttpPost httpPost, HttpContext httpContext, Trigger trigger)
|
||||
throws IOException, TriggerExecuteException {
|
||||
int retryCounter = 0;
|
||||
HttpResponse httpResponse = null;
|
||||
while (true) {
|
||||
try {
|
||||
httpResponse = httpClient.execute(host, httpPost, (HttpContext) null);
|
||||
if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
|
||||
return httpResponse;
|
||||
} else {
|
||||
retryCounter++;
|
||||
sleepAndRetry(new IllegalStateException("invalid http status code: "
|
||||
+ httpResponse.getStatusLine().getStatusCode()), retryCounter, trigger);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
retryCounter++;
|
||||
sleepAndRetry(e, retryCounter, trigger);
|
||||
} finally {
|
||||
if (null != httpResponse && httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
|
||||
httpResponse.getEntity().getContent().close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void sleepAndRetry(Exception e, int retryCounter, Trigger trigger) throws TriggerExecuteException {
|
||||
if (retryCounter > MAX_RETRY_COUNT) {
|
||||
throw new TriggerExecuteException(
|
||||
"error execute trigger[" + trigger + "] after " + MAX_RETRY_COUNT + " times retry!", e);
|
||||
}
|
||||
if (LOGGER.isErrorEnabled()) {
|
||||
LOGGER.error("error execute trigger[" + trigger + "]:(" + e.getMessage() + "), retry for the " + retryCounter
|
||||
+ "th time after " + (retryCounter * 30) + " seconds...");
|
||||
}
|
||||
try {
|
||||
Thread.sleep(retryCounter * 30000);
|
||||
} catch (InterruptedException e1) {
|
||||
if (LOGGER.isWarnEnabled()) {
|
||||
LOGGER.warn("", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
public class TriggerExecutor
|
||||
{
|
||||
private static final int MAX_TOTAL = 200;
|
||||
private static final int MAX_FOR_ROUTE = 200;
|
||||
private static final String DEFAULT_CHARSET = "UTF-8";
|
||||
private static final int DEFAULT_PORT = 80;
|
||||
private static final int MAX_RETRY_COUNT = 10;
|
||||
private static final Logger LOGGER;
|
||||
private static final TriggerExecutor TRIGGER_EXECUTOR;
|
||||
private final HttpClient httpClient;
|
||||
|
||||
private TriggerExecutor() {
|
||||
if (TriggerExecutor.LOGGER.isDebugEnabled()) {
|
||||
TriggerExecutor.LOGGER.debug("init trigger executor");
|
||||
}
|
||||
final SchemeRegistry schemeRegistry = new SchemeRegistry();
|
||||
schemeRegistry.register(new Scheme("http", 80, (SchemeSocketFactory)new PlainSocketFactory()));
|
||||
final HttpParams httpParams = (HttpParams)new BasicHttpParams();
|
||||
httpParams.setParameter("http.protocol.content-charset", (Object)"UTF-8");
|
||||
httpParams.setParameter("http.socket.timeout", (Object)300000);
|
||||
httpParams.setParameter("http.connection.timeout", (Object)60000);
|
||||
final ThreadSafeClientConnManager connManager = new ThreadSafeClientConnManager(schemeRegistry);
|
||||
connManager.setMaxTotal(200);
|
||||
connManager.setDefaultMaxPerRoute(200);
|
||||
this.httpClient = (HttpClient)new DefaultHttpClient((ClientConnectionManager)connManager, httpParams);
|
||||
if (TriggerExecutor.LOGGER.isDebugEnabled()) {
|
||||
TriggerExecutor.LOGGER.debug("trigger executor initialized[{}]", (Object)this.toString());
|
||||
}
|
||||
}
|
||||
|
||||
public static TriggerExecutor getInstance() {
|
||||
return TriggerExecutor.TRIGGER_EXECUTOR;
|
||||
}
|
||||
|
||||
public List<BSONObject> execute(final Trigger trigger, final List<BSONObject> docList) throws TriggerExecuteException {
|
||||
try {
|
||||
final List<BSONObject> ret = this.doExecute(trigger, docList);
|
||||
if (TriggerExecutor.LOGGER.isDebugEnabled()) {
|
||||
TriggerExecutor.LOGGER.debug("execute trigger: " + trigger + ", doc size:" + docList.size() + ", return size:" + ret.size());
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new TriggerExecuteException("error execute trigger: " + trigger, e);
|
||||
}
|
||||
catch (URISyntaxException e2) {
|
||||
throw new TriggerExecuteException("error execute trigger: " + trigger, e2);
|
||||
}
|
||||
}
|
||||
|
||||
private List<BSONObject> doExecute(final Trigger trigger, final List<BSONObject> docList) throws IOException, URISyntaxException, TriggerExecuteException {
|
||||
final URL url = new URL(trigger.getInterface());
|
||||
final HttpPost httpPost = new HttpPost(url.toURI());
|
||||
final DBObject object = (DBObject)new BasicDBObject();
|
||||
object.put("doc_list_size", (Object)docList.size());
|
||||
object.put("doc_list", (Object)docList);
|
||||
httpPost.setEntity((HttpEntity)new ByteArrayEntity(GZIPUtils.makeAsByteArray(new BasicBSONEncoder().encode((BSONObject)object))));
|
||||
final HttpHost host = new HttpHost(url.getHost(), (url.getPort() == -1) ? 80 : url.getPort());
|
||||
final HttpResponse httpResponse = this.executeWithRetry(host, httpPost, null, trigger);
|
||||
final InputStream inputStream = httpResponse.getEntity().getContent();
|
||||
try {
|
||||
if (httpResponse.getStatusLine().getStatusCode() != 200) {
|
||||
throw new TriggerExecuteException("error execute trigger:" + trigger + ", get http code:" + httpResponse.getStatusLine().getStatusCode());
|
||||
}
|
||||
final BSONObject returnObject = new BasicBSONDecoder().readObject(GZIPUtils.wrap(inputStream));
|
||||
if (null == returnObject) {
|
||||
throw new TriggerExecuteException("trigger" + trigger + " returns nothing at all !");
|
||||
}
|
||||
final List<BSONObject> ret = (List<BSONObject>)returnObject.get("doc_list");
|
||||
if (ret == null) {
|
||||
throw new TriggerExecuteException("trigger" + trigger + " return an null doc_list[" + ret + "]");
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
finally {
|
||||
if (null != inputStream) {
|
||||
inputStream.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private HttpResponse executeWithRetry(final HttpHost host, final HttpPost httpPost, final HttpContext httpContext, final Trigger trigger) throws IOException, TriggerExecuteException {
|
||||
int retryCounter = 0;
|
||||
HttpResponse httpResponse = null;
|
||||
while (true) {
|
||||
try {
|
||||
httpResponse = this.httpClient.execute(host, (HttpRequest)httpPost, (HttpContext)null);
|
||||
if (httpResponse.getStatusLine().getStatusCode() == 200) {
|
||||
return httpResponse;
|
||||
}
|
||||
++retryCounter;
|
||||
this.sleepAndRetry(new IllegalStateException("invalid http status code: " + httpResponse.getStatusLine().getStatusCode()), retryCounter, trigger);
|
||||
}
|
||||
catch (Exception e) {
|
||||
++retryCounter;
|
||||
this.sleepAndRetry(e, retryCounter, trigger);
|
||||
}
|
||||
finally {
|
||||
if (null != httpResponse && httpResponse.getStatusLine().getStatusCode() != 200) {
|
||||
httpResponse.getEntity().getContent().close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void sleepAndRetry(final Exception e, final int retryCounter, final Trigger trigger) throws TriggerExecuteException {
|
||||
if (retryCounter > 10) {
|
||||
throw new TriggerExecuteException("error execute trigger[" + trigger + "] after " + 10 + " times retry!", e);
|
||||
}
|
||||
if (TriggerExecutor.LOGGER.isErrorEnabled()) {
|
||||
TriggerExecutor.LOGGER.error("error execute trigger[" + trigger + "]:(" + e.getMessage() + "), retry for the " + retryCounter + "th time after " + retryCounter * 30 + " seconds...");
|
||||
}
|
||||
try {
|
||||
Thread.sleep(retryCounter * 30000);
|
||||
}
|
||||
catch (InterruptedException e2) {
|
||||
if (TriggerExecutor.LOGGER.isWarnEnabled()) {
|
||||
TriggerExecutor.LOGGER.warn("", (Throwable)e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static {
|
||||
LOGGER = LoggerFactory.getLogger((Class)TriggerExecutor.class);
|
||||
TRIGGER_EXECUTOR = new TriggerExecutor();
|
||||
}
|
||||
}
|
||||
|
||||
+164
-162
@@ -1,170 +1,172 @@
|
||||
package com.fangjia.datastore.config;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.dom4j.Document;
|
||||
import org.dom4j.DocumentException;
|
||||
import java.io.File;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import org.dom4j.Element;
|
||||
import org.dom4j.Node;
|
||||
import org.dom4j.Document;
|
||||
import java.util.Collections;
|
||||
import java.util.Set;
|
||||
import com.fangjia.datastore.ConfigurationException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import org.dom4j.io.SAXReader;
|
||||
|
||||
import com.fangjia.datastore.ConfigurationException;
|
||||
|
||||
public class DataStoreConfig implements Reloadable {
|
||||
private final SAXReader reader = new SAXReader();
|
||||
private final String serverConfigFile;
|
||||
private final String storeConfigFile;
|
||||
private final ServerConfig serverConfig;
|
||||
private final Map<String, StoreConfig> storeConfigs;
|
||||
private final Map<String, SolrConfig> solrConfigs;
|
||||
private final Map<String, MongoConfig> mongoConfigs;
|
||||
private boolean readOnly = false;
|
||||
|
||||
public DataStoreConfig(String datastoreHome) throws ConfigurationException {
|
||||
this.serverConfigFile = datastoreHome + "/conf/server.xml";
|
||||
this.storeConfigFile = datastoreHome + "/conf/store.xml";
|
||||
this.serverConfig = new ServerConfig();
|
||||
this.storeConfigs = new HashMap<String, StoreConfig>();
|
||||
this.solrConfigs = new HashMap<String, SolrConfig>();
|
||||
this.mongoConfigs = new HashMap<String, MongoConfig>();
|
||||
load();
|
||||
}
|
||||
|
||||
public ServerConfig getServerConfig() {
|
||||
return serverConfig;
|
||||
}
|
||||
|
||||
public SolrConfig getSolrConfigById(String id) {
|
||||
return solrConfigs.get(id);
|
||||
}
|
||||
|
||||
public MongoConfig getMongoConfigById(String id) {
|
||||
return this.mongoConfigs.get(id);
|
||||
}
|
||||
|
||||
public StoreConfig getStoreConfig(String storeName) {
|
||||
return this.storeConfigs.get(storeName);
|
||||
}
|
||||
|
||||
public Set<String> getStoreNames() {
|
||||
return Collections.unmodifiableSet(this.storeConfigs.keySet());
|
||||
}
|
||||
|
||||
public void reload() throws ConfigurationException {
|
||||
load();
|
||||
}
|
||||
|
||||
private synchronized void load() throws ConfigurationException {
|
||||
Document document = read(this.serverConfigFile);
|
||||
serverConfig.reload(document.selectSingleNode("//datastore//server"));
|
||||
initSolrConfigs(document);
|
||||
initMongoConfigs(document);
|
||||
document = read(this.storeConfigFile);
|
||||
initStoreConfigs(document, solrConfigs, mongoConfigs);
|
||||
}
|
||||
|
||||
synchronized StoreConfig getOrCreateStoreConfig(String storeName) {
|
||||
StoreConfig ret = this.storeConfigs.get(storeName);
|
||||
if (null == ret) {
|
||||
ret = new StoreConfig(this, storeName);
|
||||
storeConfigs.put(storeName, ret);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
private synchronized SolrConfig getOrCreateSolrConfig(String solrId) {
|
||||
SolrConfig ret = this.solrConfigs.get(solrId);
|
||||
if (null == ret) {
|
||||
ret = new SolrConfig(this, solrId);
|
||||
solrConfigs.put(solrId, ret);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
private synchronized MongoConfig getOrCreateMongoConfig(String mongoId) throws ConfigurationException {
|
||||
MongoConfig ret = this.mongoConfigs.get(mongoId);
|
||||
if (null == ret) {
|
||||
ret = new MongoConfig(mongoId);
|
||||
mongoConfigs.put(mongoId, ret);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
private void initSolrConfigs(Document document) throws ConfigurationException {
|
||||
List<Node> solrNodes = document.selectNodes("//datastore//solr");
|
||||
if (null == solrNodes || solrNodes.isEmpty()) {
|
||||
throw new ConfigurationException("no solr is configured in " + serverConfigFile + " !");
|
||||
}
|
||||
Map<String, SolrConfig> newSolrConfigs = new HashMap<String, SolrConfig>();
|
||||
for (Node solrNode : solrNodes) {
|
||||
String id = ((Element) solrNode).attributeValue("id");
|
||||
SolrConfig config = getOrCreateSolrConfig(id);
|
||||
config.reload(solrNode);
|
||||
newSolrConfigs.put(id, config);
|
||||
}
|
||||
this.solrConfigs.clear();
|
||||
this.solrConfigs.putAll(newSolrConfigs);
|
||||
}
|
||||
|
||||
private void initMongoConfigs(Document document) throws ConfigurationException {
|
||||
List<Node> mongoNodes = document.selectNodes("//datastore//mongo");
|
||||
if (null == mongoNodes || mongoNodes.isEmpty()) {
|
||||
throw new ConfigurationException("no mongo is configured in " + serverConfigFile + " !");
|
||||
}
|
||||
Map<String, MongoConfig> newMongoConfigs = new HashMap<String, MongoConfig>();
|
||||
for (Node mongoNode : mongoNodes) {
|
||||
String id = ((Element) mongoNode).attributeValue("id");
|
||||
MongoConfig mongoConfig = getOrCreateMongoConfig(id);
|
||||
mongoConfig.reload(mongoNode);
|
||||
newMongoConfigs.put(id, mongoConfig);
|
||||
}
|
||||
this.mongoConfigs.clear();
|
||||
this.mongoConfigs.putAll(newMongoConfigs);
|
||||
}
|
||||
|
||||
private void initStoreConfigs(Document document, Map<String, SolrConfig> solrConfigMap,
|
||||
Map<String, MongoConfig> mongoConfigMap) throws ConfigurationException {
|
||||
List<Node> storeNodes = document.selectNodes("//datastore//store");
|
||||
if (null == storeNodes || storeNodes.isEmpty()) {
|
||||
throw new ConfigurationException("no store is configured in " + storeConfigFile + " !");
|
||||
}
|
||||
Map<String, StoreConfig> newStoreConfigs = new HashMap<String, StoreConfig>();
|
||||
for (Node storeNode : storeNodes) {
|
||||
String name = ((Element) storeNode).attributeValue("name");
|
||||
final StoreConfig storeConfig = getOrCreateStoreConfig(name);
|
||||
storeConfig.reload(storeNode);
|
||||
newStoreConfigs.put(name, storeConfig);
|
||||
}
|
||||
this.storeConfigs.clear();
|
||||
this.storeConfigs.putAll(newStoreConfigs);
|
||||
}
|
||||
|
||||
private Document read(String configFile) throws ConfigurationException {
|
||||
try {
|
||||
return reader.read(new File(configFile));
|
||||
} catch (DocumentException e) {
|
||||
throw new ConfigurationException("error read datastore config file: " + configFile, e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder ret = new StringBuilder();
|
||||
ret.append("server config file:").append(this.serverConfigFile).append(", store config file: ").append(
|
||||
this.storeConfigFile).append(", readOnly:").append(this.readOnly);
|
||||
return ret.toString();
|
||||
}
|
||||
|
||||
public boolean isReadOnly() {
|
||||
return readOnly;
|
||||
}
|
||||
|
||||
public void setReadOnly(boolean readOnly) {
|
||||
this.readOnly = readOnly;
|
||||
}
|
||||
public class DataStoreConfig implements Reloadable
|
||||
{
|
||||
private final SAXReader reader;
|
||||
private final String serverConfigFile;
|
||||
private final String storeConfigFile;
|
||||
private final ServerConfig serverConfig;
|
||||
private final Map<String, StoreConfig> storeConfigs;
|
||||
private final Map<String, SolrConfig> solrConfigs;
|
||||
private final Map<String, MongoConfig> mongoConfigs;
|
||||
private boolean readOnly;
|
||||
|
||||
public DataStoreConfig(final String datastoreHome) throws ConfigurationException {
|
||||
this.reader = new SAXReader();
|
||||
this.readOnly = false;
|
||||
this.serverConfigFile = datastoreHome + "/conf/server.xml";
|
||||
this.storeConfigFile = datastoreHome + "/conf/store.xml";
|
||||
this.serverConfig = new ServerConfig();
|
||||
this.storeConfigs = new HashMap<String, StoreConfig>();
|
||||
this.solrConfigs = new HashMap<String, SolrConfig>();
|
||||
this.mongoConfigs = new HashMap<String, MongoConfig>();
|
||||
this.load();
|
||||
}
|
||||
|
||||
public ServerConfig getServerConfig() {
|
||||
return this.serverConfig;
|
||||
}
|
||||
|
||||
public SolrConfig getSolrConfigById(final String id) {
|
||||
return this.solrConfigs.get(id);
|
||||
}
|
||||
|
||||
public MongoConfig getMongoConfigById(final String id) {
|
||||
return this.mongoConfigs.get(id);
|
||||
}
|
||||
|
||||
public StoreConfig getStoreConfig(final String storeName) {
|
||||
return this.storeConfigs.get(storeName);
|
||||
}
|
||||
|
||||
public Set<String> getStoreNames() {
|
||||
return Collections.<String>unmodifiableSet((Set<? extends String>)this.storeConfigs.keySet());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reload() throws ConfigurationException {
|
||||
this.load();
|
||||
}
|
||||
|
||||
private synchronized void load() throws ConfigurationException {
|
||||
Document document = this.read(this.serverConfigFile);
|
||||
this.serverConfig.reload(document.selectSingleNode("//datastore//server"));
|
||||
this.initSolrConfigs(document);
|
||||
this.initMongoConfigs(document);
|
||||
document = this.read(this.storeConfigFile);
|
||||
this.initStoreConfigs(document, this.solrConfigs, this.mongoConfigs);
|
||||
}
|
||||
|
||||
synchronized StoreConfig getOrCreateStoreConfig(final String storeName) {
|
||||
StoreConfig ret = this.storeConfigs.get(storeName);
|
||||
if (null == ret) {
|
||||
ret = new StoreConfig(this, storeName);
|
||||
this.storeConfigs.put(storeName, ret);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
private synchronized SolrConfig getOrCreateSolrConfig(final String solrId) {
|
||||
SolrConfig ret = this.solrConfigs.get(solrId);
|
||||
if (null == ret) {
|
||||
ret = new SolrConfig(this, solrId);
|
||||
this.solrConfigs.put(solrId, ret);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
private synchronized MongoConfig getOrCreateMongoConfig(final String mongoId) throws ConfigurationException {
|
||||
MongoConfig ret = this.mongoConfigs.get(mongoId);
|
||||
if (null == ret) {
|
||||
ret = new MongoConfig(mongoId);
|
||||
this.mongoConfigs.put(mongoId, ret);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
private void initSolrConfigs(final Document document) throws ConfigurationException {
|
||||
final List<Node> solrNodes = (List<Node>)document.selectNodes("//datastore//solr");
|
||||
if (null == solrNodes || solrNodes.isEmpty()) {
|
||||
throw new ConfigurationException("no solr is configured in " + this.serverConfigFile + " !");
|
||||
}
|
||||
final Map<String, SolrConfig> newSolrConfigs = new HashMap<String, SolrConfig>();
|
||||
for (final Node solrNode : solrNodes) {
|
||||
final String id = ((Element)solrNode).attributeValue("id");
|
||||
final SolrConfig config = this.getOrCreateSolrConfig(id);
|
||||
config.reload(solrNode);
|
||||
newSolrConfigs.put(id, config);
|
||||
}
|
||||
this.solrConfigs.clear();
|
||||
this.solrConfigs.putAll(newSolrConfigs);
|
||||
}
|
||||
|
||||
private void initMongoConfigs(final Document document) throws ConfigurationException {
|
||||
final List<Node> mongoNodes = (List<Node>)document.selectNodes("//datastore//mongo");
|
||||
if (null == mongoNodes || mongoNodes.isEmpty()) {
|
||||
throw new ConfigurationException("no mongo is configured in " + this.serverConfigFile + " !");
|
||||
}
|
||||
final Map<String, MongoConfig> newMongoConfigs = new HashMap<String, MongoConfig>();
|
||||
for (final Node mongoNode : mongoNodes) {
|
||||
final String id = ((Element)mongoNode).attributeValue("id");
|
||||
final MongoConfig mongoConfig = this.getOrCreateMongoConfig(id);
|
||||
mongoConfig.reload(mongoNode);
|
||||
newMongoConfigs.put(id, mongoConfig);
|
||||
}
|
||||
this.mongoConfigs.clear();
|
||||
this.mongoConfigs.putAll(newMongoConfigs);
|
||||
}
|
||||
|
||||
private void initStoreConfigs(final Document document, final Map<String, SolrConfig> solrConfigMap, final Map<String, MongoConfig> mongoConfigMap) throws ConfigurationException {
|
||||
final List<Node> storeNodes = (List<Node>)document.selectNodes("//datastore//store");
|
||||
if (null == storeNodes || storeNodes.isEmpty()) {
|
||||
throw new ConfigurationException("no store is configured in " + this.storeConfigFile + " !");
|
||||
}
|
||||
final Map<String, StoreConfig> newStoreConfigs = new HashMap<String, StoreConfig>();
|
||||
for (final Node storeNode : storeNodes) {
|
||||
final String name = ((Element)storeNode).attributeValue("name");
|
||||
final StoreConfig storeConfig = this.getOrCreateStoreConfig(name);
|
||||
storeConfig.reload(storeNode);
|
||||
newStoreConfigs.put(name, storeConfig);
|
||||
}
|
||||
this.storeConfigs.clear();
|
||||
this.storeConfigs.putAll(newStoreConfigs);
|
||||
}
|
||||
|
||||
private Document read(final String configFile) throws ConfigurationException {
|
||||
try {
|
||||
return this.reader.read(new File(configFile));
|
||||
}
|
||||
catch (DocumentException e) {
|
||||
throw new ConfigurationException("error read datastore config file: " + configFile, (Throwable)e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder ret = new StringBuilder();
|
||||
ret.append("server config file:").append(this.serverConfigFile).append(", store config file: ").append(this.storeConfigFile).append(", readOnly:").append(this.readOnly);
|
||||
return ret.toString();
|
||||
}
|
||||
|
||||
public boolean isReadOnly() {
|
||||
return this.readOnly;
|
||||
}
|
||||
|
||||
public void setReadOnly(final boolean readOnly) {
|
||||
this.readOnly = readOnly;
|
||||
}
|
||||
}
|
||||
|
||||
+111
-120
@@ -1,127 +1,118 @@
|
||||
package com.fangjia.datastore.config;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.commons.lang.builder.EqualsBuilder;
|
||||
import org.apache.commons.lang.builder.HashCodeBuilder;
|
||||
import org.bson.types.ObjectId;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.Date;
|
||||
import java.util.Map;
|
||||
import java.util.List;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
public class Field {
|
||||
private String name;
|
||||
private FieldTypeEnum type;
|
||||
private boolean required;
|
||||
|
||||
Field() {
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public FieldTypeEnum getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
void setName(String name) {
|
||||
if (StringUtils.isEmpty(name)) {
|
||||
throw new IllegalArgumentException("invalid field name:" + name);
|
||||
}
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
void setType(FieldTypeEnum type) {
|
||||
if (null == type) {
|
||||
throw new IllegalArgumentException("invalid type of field[" + name + "]:" + type);
|
||||
}
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public boolean isRequired() {
|
||||
return required;
|
||||
}
|
||||
|
||||
void setRequired(boolean required) {
|
||||
this.required = required;
|
||||
}
|
||||
|
||||
public boolean isBigField() {
|
||||
return this.type == FieldTypeEnum.TEXT;
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证一个值是否匹配其对应的类型
|
||||
*
|
||||
* @param value
|
||||
* @return 匹配则返回true,不匹配返回false
|
||||
*/
|
||||
public boolean validate(Object value) {
|
||||
if (this.required && value == null) {
|
||||
return false;
|
||||
}
|
||||
if (value == null) {
|
||||
return true;
|
||||
}
|
||||
Class<?> valueClass = value.getClass();
|
||||
switch (this.type) {
|
||||
case BOOLEAN:
|
||||
return valueClass == boolean.class || valueClass == Boolean.class;
|
||||
case BYTE:
|
||||
return valueClass == byte.class || valueClass == Byte.class;
|
||||
case INTEGER:
|
||||
return valueClass == int.class || valueClass == Integer.class;
|
||||
case LONG:
|
||||
return valueClass == long.class || valueClass == Long.class;
|
||||
case DOUBLE:
|
||||
return valueClass == double.class || valueClass == Double.class;
|
||||
case STRING:
|
||||
case TEXT:
|
||||
return valueClass == String.class;
|
||||
case LIST:
|
||||
return value instanceof List;
|
||||
case MAP:
|
||||
return value instanceof Map;
|
||||
case DATETIME:
|
||||
return value instanceof Date;
|
||||
case BINARY:
|
||||
return valueClass == byte[].class;
|
||||
case REGEXP:
|
||||
return valueClass == Pattern.class;
|
||||
case OBJECTID:
|
||||
return valueClass == ObjectId.class;
|
||||
default:
|
||||
throw new RuntimeException("un cased type:" + this);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return new HashCodeBuilder().append(this.name).append(this.type).append(this.required).toHashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (obj == this) {
|
||||
return true;
|
||||
}
|
||||
if (null == obj) {
|
||||
return false;
|
||||
}
|
||||
if (!(obj instanceof Field)) {
|
||||
return false;
|
||||
}
|
||||
Field other = (Field) obj;
|
||||
return new EqualsBuilder().append(this.name, other.name).append(this.type, other.type)
|
||||
.append(this.required, other.required).isEquals();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return new StringBuilder().append("{field:").append(name).append(", type:").append(type).append(", required:")
|
||||
.append(required).append("}").toString();
|
||||
}
|
||||
|
||||
public class Field
|
||||
{
|
||||
private String name;
|
||||
private FieldTypeEnum type;
|
||||
private boolean required;
|
||||
|
||||
Field() {
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
public FieldTypeEnum getType() {
|
||||
return this.type;
|
||||
}
|
||||
|
||||
void setName(final String name) {
|
||||
if (StringUtils.isEmpty(name)) {
|
||||
throw new IllegalArgumentException("invalid field name:" + name);
|
||||
}
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
void setType(final FieldTypeEnum type) {
|
||||
if (null == type) {
|
||||
throw new IllegalArgumentException("invalid type of field[" + this.name + "]:" + type);
|
||||
}
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public boolean isRequired() {
|
||||
return this.required;
|
||||
}
|
||||
|
||||
void setRequired(final boolean required) {
|
||||
this.required = required;
|
||||
}
|
||||
|
||||
public boolean isBigField() {
|
||||
return this.type == FieldTypeEnum.TEXT;
|
||||
}
|
||||
|
||||
public boolean validate(final Object value) {
|
||||
if (this.required && value == null) {
|
||||
return false;
|
||||
}
|
||||
if (value == null) {
|
||||
return true;
|
||||
}
|
||||
final Class<?> valueClass = value.getClass();
|
||||
switch (this.type) {
|
||||
case BOOLEAN:
|
||||
return valueClass == Boolean.TYPE || valueClass == Boolean.class;
|
||||
case BYTE:
|
||||
return valueClass == Byte.TYPE || valueClass == Byte.class;
|
||||
case INTEGER:
|
||||
return valueClass == Integer.TYPE || valueClass == Integer.class;
|
||||
case LONG:
|
||||
return valueClass == Long.TYPE || valueClass == Long.class;
|
||||
case DOUBLE:
|
||||
return valueClass == Double.TYPE || valueClass == Double.class;
|
||||
case STRING:
|
||||
case TEXT:
|
||||
return valueClass == String.class;
|
||||
case LIST:
|
||||
return value instanceof List;
|
||||
case MAP:
|
||||
return value instanceof Map;
|
||||
case DATETIME:
|
||||
return value instanceof Date;
|
||||
case BINARY:
|
||||
return valueClass == byte[].class;
|
||||
case REGEXP:
|
||||
return valueClass == Pattern.class;
|
||||
case OBJECTID:
|
||||
return valueClass == ObjectId.class;
|
||||
default:
|
||||
throw new RuntimeException("un cased type:" + this);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return new HashCodeBuilder().append((Object)this.name).append((Object)this.type).append(this.required).toHashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(final Object obj) {
|
||||
if (obj == this) {
|
||||
return true;
|
||||
}
|
||||
if (null == obj) {
|
||||
return false;
|
||||
}
|
||||
if (!(obj instanceof Field)) {
|
||||
return false;
|
||||
}
|
||||
final Field other = (Field)obj;
|
||||
return new EqualsBuilder().append((Object)this.name, (Object)other.name).append((Object)this.type, (Object)other.type).append(this.required, other.required).isEquals();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "{field:" + this.name + ", type:" + this.type + ", required:" + this.required + "}";
|
||||
}
|
||||
}
|
||||
|
||||
+28
-19
@@ -1,22 +1,31 @@
|
||||
package com.fangjia.datastore.config;
|
||||
|
||||
public enum FieldTypeEnum {
|
||||
BOOLEAN, BYTE, INTEGER, LONG, DOUBLE, STRING, TEXT, LIST, MAP, DATETIME, BINARY, REGEXP, OBJECTID;
|
||||
|
||||
private FieldTypeEnum() {
|
||||
}
|
||||
|
||||
public String getType() {
|
||||
return this.name().toLowerCase();
|
||||
}
|
||||
|
||||
public static FieldTypeEnum parse(String type) {
|
||||
for (FieldTypeEnum typeEnum : FieldTypeEnum.values()) {
|
||||
if (typeEnum.name().toLowerCase().equals(type)) {
|
||||
return typeEnum;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public enum FieldTypeEnum
|
||||
{
|
||||
BOOLEAN,
|
||||
BYTE,
|
||||
INTEGER,
|
||||
LONG,
|
||||
DOUBLE,
|
||||
STRING,
|
||||
TEXT,
|
||||
LIST,
|
||||
MAP,
|
||||
DATETIME,
|
||||
BINARY,
|
||||
REGEXP,
|
||||
OBJECTID;
|
||||
|
||||
public String getType() {
|
||||
return this.name().toLowerCase();
|
||||
}
|
||||
|
||||
public static FieldTypeEnum parse(final String type) {
|
||||
for (final FieldTypeEnum typeEnum : values()) {
|
||||
if (typeEnum.name().toLowerCase().equals(type)) {
|
||||
return typeEnum;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
+91
-80
@@ -2,87 +2,98 @@ package com.fangjia.datastore.config;
|
||||
|
||||
import org.apache.commons.lang.builder.EqualsBuilder;
|
||||
import org.apache.commons.lang.builder.HashCodeBuilder;
|
||||
import org.dom4j.Node;
|
||||
|
||||
import com.fangjia.datastore.ConfigurationException;
|
||||
import com.fangjia.datastore.utils.XmlUtils;
|
||||
import org.dom4j.Node;
|
||||
import com.mongodb.MongoURI;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import com.fangjia.datastore.ConfigurationException;
|
||||
import com.mongodb.Mongo;
|
||||
|
||||
public final class MongoConfig {
|
||||
|
||||
private String host;
|
||||
private int port;
|
||||
private final String id;
|
||||
private Mongo mongo;
|
||||
|
||||
MongoConfig(String id) throws ConfigurationException {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
private synchronized void init() throws ConfigurationException {
|
||||
if (null == mongo) {
|
||||
try {
|
||||
mongo = new Mongo(host, port);
|
||||
} catch (Exception e) {
|
||||
throw new ConfigurationException("error init mongo[" + host + ":" + port, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public Mongo getMongo() throws ConfigurationException {
|
||||
if (null == mongo) {
|
||||
init();
|
||||
}
|
||||
return mongo;
|
||||
}
|
||||
|
||||
synchronized void reload(Node mongoNode) throws ConfigurationException {
|
||||
String host = XmlUtils.getNodeValue(mongoNode, "host", "datastore.mongo.db.fangjia.com");
|
||||
int port = XmlUtils.getNodeValueAsInt(mongoNode, "port", 27017);
|
||||
if (host.equals(this.host) && this.port == port) {
|
||||
return;
|
||||
}
|
||||
this.host = host;
|
||||
this.port = port;
|
||||
if (null != mongo) {
|
||||
try {
|
||||
init();
|
||||
} catch (Exception e) {
|
||||
throw new ConfigurationException("error init mongo[" + host + ":" + port, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return new HashCodeBuilder().append(this.host).append(this.port).append(this.id).toHashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (obj == this) {
|
||||
return true;
|
||||
}
|
||||
if (null == obj) {
|
||||
return false;
|
||||
}
|
||||
if (!(obj instanceof MongoConfig)) {
|
||||
return false;
|
||||
}
|
||||
MongoConfig other = (MongoConfig) obj;
|
||||
return new EqualsBuilder().append(this.id, other.id).append(this.host, other.host).append(this.port, other.port)
|
||||
.isEquals();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder ret = new StringBuilder();
|
||||
ret.append('{');
|
||||
ret.append("id:").append(this.id);
|
||||
ret.append(", host:").append(this.host);
|
||||
ret.append(", port:").append(this.port);
|
||||
ret.append('}');
|
||||
return ret.toString();
|
||||
}
|
||||
|
||||
public final class MongoConfig
|
||||
{
|
||||
private String host;
|
||||
private String uri;
|
||||
private int port;
|
||||
private final String id;
|
||||
private Mongo mongo;
|
||||
|
||||
MongoConfig(final String id) throws ConfigurationException {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
private synchronized void init() throws ConfigurationException {
|
||||
if (null == this.mongo) {
|
||||
try {
|
||||
MongoURI mongoURI;
|
||||
if (StringUtils.isNotBlank(this.uri)) {
|
||||
mongoURI = new MongoURI(this.uri);
|
||||
}
|
||||
else {
|
||||
mongoURI = new MongoURI("mongodb://" + this.host + ":" + this.port);
|
||||
}
|
||||
this.mongo = new Mongo(mongoURI);
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new ConfigurationException("error init mongo[" + this.host + ":" + this.port, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public Mongo getMongo() throws ConfigurationException {
|
||||
if (null == this.mongo) {
|
||||
this.init();
|
||||
}
|
||||
return this.mongo;
|
||||
}
|
||||
|
||||
synchronized void reload(final Node mongoNode) throws ConfigurationException {
|
||||
final String host = XmlUtils.getNodeValue(mongoNode, "host", "datastore.mongo.db.fangjia.com");
|
||||
final int port = XmlUtils.getNodeValueAsInt(mongoNode, "port", 27017);
|
||||
final String uri = XmlUtils.getNodeValue(mongoNode, "uri", "");
|
||||
if (host.equals(this.host) && this.port == port) {
|
||||
return;
|
||||
}
|
||||
this.host = host;
|
||||
this.port = port;
|
||||
this.uri = uri;
|
||||
if (null != this.mongo) {
|
||||
try {
|
||||
this.init();
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new ConfigurationException("error init mongo[" + host + ":" + port, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return new HashCodeBuilder().append((Object)this.host).append(this.port).append((Object)this.id).toHashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(final Object obj) {
|
||||
if (obj == this) {
|
||||
return true;
|
||||
}
|
||||
if (null == obj) {
|
||||
return false;
|
||||
}
|
||||
if (!(obj instanceof MongoConfig)) {
|
||||
return false;
|
||||
}
|
||||
final MongoConfig other = (MongoConfig)obj;
|
||||
return new EqualsBuilder().append((Object)this.id, (Object)other.id).append((Object)this.host, (Object)other.host).append(this.port, other.port).isEquals();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder ret = new StringBuilder();
|
||||
ret.append('{');
|
||||
ret.append("id:").append(this.id);
|
||||
ret.append(", host:").append(this.host);
|
||||
ret.append(", port:").append(this.port);
|
||||
ret.append('}');
|
||||
return ret.toString();
|
||||
}
|
||||
}
|
||||
|
||||
+3
-2
@@ -2,6 +2,7 @@ package com.fangjia.datastore.config;
|
||||
|
||||
import com.fangjia.datastore.ConfigurationException;
|
||||
|
||||
public interface Reloadable {
|
||||
void reload() throws ConfigurationException;
|
||||
public interface Reloadable
|
||||
{
|
||||
void reload() throws ConfigurationException;
|
||||
}
|
||||
|
||||
+67
-69
@@ -1,74 +1,72 @@
|
||||
package com.fangjia.datastore.config;
|
||||
|
||||
import com.fangjia.datastore.utils.XmlUtils;
|
||||
import com.fangjia.datastore.ConfigurationException;
|
||||
import org.dom4j.Node;
|
||||
|
||||
import com.fangjia.datastore.ConfigurationException;
|
||||
import com.fangjia.datastore.utils.XmlUtils;
|
||||
|
||||
public final class ServerConfig {
|
||||
private int port;
|
||||
private boolean disableLogging;
|
||||
private long performanceLogFilterMaxTime;
|
||||
private int minaReadBufferSize;
|
||||
private int cursorExpirationMinutes;
|
||||
private int defaultCursorLimit;
|
||||
private int maxConnections;
|
||||
private int triggerBatchSize;
|
||||
private int reIndexBatchSize;
|
||||
|
||||
ServerConfig() {
|
||||
super();
|
||||
}
|
||||
|
||||
void reload(Node server) throws ConfigurationException {
|
||||
if (null == server) {
|
||||
throw new ConfigurationException("no server is configured!");
|
||||
}
|
||||
this.port = XmlUtils.getNodeValueAsInt(server, "port", 27017);
|
||||
this.disableLogging = XmlUtils.getNodeValueAsBoolean(server, "disableLogging", false);
|
||||
this.performanceLogFilterMaxTime = XmlUtils.getNodeValueAsLong(server, "performanceLogFilterMaxTime", 1000);
|
||||
this.minaReadBufferSize = XmlUtils.getNodeValueAsInt(server, "minaReadBufferSize", 20480);
|
||||
this.cursorExpirationMinutes = XmlUtils.getNodeValueAsInt(server, "cursorExpirationMinutes", 5);
|
||||
this.defaultCursorLimit = XmlUtils.getNodeValueAsInt(server, "defaultCursorLimit", 20);
|
||||
this.maxConnections = XmlUtils.getNodeValueAsInt(server, "maxConnections", 10);
|
||||
this.triggerBatchSize = XmlUtils.getNodeValueAsInt(server, "triggerBatchSize", 200);
|
||||
this.reIndexBatchSize = XmlUtils.getNodeValueAsInt(server, "reIndexBatchSize", 200);
|
||||
}
|
||||
|
||||
public int getPort() {
|
||||
return port;
|
||||
}
|
||||
|
||||
public boolean isDisableLogging() {
|
||||
return disableLogging;
|
||||
}
|
||||
|
||||
public long getPerformanceLogFilterMaxTime() {
|
||||
return performanceLogFilterMaxTime;
|
||||
}
|
||||
|
||||
public int getMinaReadBufferSize() {
|
||||
return minaReadBufferSize;
|
||||
}
|
||||
|
||||
public int getCursorExpirationMinutes() {
|
||||
return cursorExpirationMinutes;
|
||||
}
|
||||
|
||||
public int getDefaultCursorLimit() {
|
||||
return defaultCursorLimit;
|
||||
}
|
||||
|
||||
public int getMaxConnections() {
|
||||
return maxConnections;
|
||||
}
|
||||
|
||||
public int getTriggerBatchSize() {
|
||||
return triggerBatchSize;
|
||||
}
|
||||
|
||||
public int getReIndexBatchSize() {
|
||||
return reIndexBatchSize;
|
||||
}
|
||||
|
||||
public final class ServerConfig
|
||||
{
|
||||
private int port;
|
||||
private boolean disableLogging;
|
||||
private long performanceLogFilterMaxTime;
|
||||
private int minaReadBufferSize;
|
||||
private int cursorExpirationMinutes;
|
||||
private int defaultCursorLimit;
|
||||
private int maxConnections;
|
||||
private int triggerBatchSize;
|
||||
private int reIndexBatchSize;
|
||||
|
||||
ServerConfig() {
|
||||
}
|
||||
|
||||
void reload(final Node server) throws ConfigurationException {
|
||||
if (null == server) {
|
||||
throw new ConfigurationException("no server is configured!");
|
||||
}
|
||||
this.port = XmlUtils.getNodeValueAsInt(server, "port", 27017);
|
||||
this.disableLogging = XmlUtils.getNodeValueAsBoolean(server, "disableLogging", false);
|
||||
this.performanceLogFilterMaxTime = XmlUtils.getNodeValueAsLong(server, "performanceLogFilterMaxTime", 1000L);
|
||||
this.minaReadBufferSize = XmlUtils.getNodeValueAsInt(server, "minaReadBufferSize", 20480);
|
||||
this.cursorExpirationMinutes = XmlUtils.getNodeValueAsInt(server, "cursorExpirationMinutes", 5);
|
||||
this.defaultCursorLimit = XmlUtils.getNodeValueAsInt(server, "defaultCursorLimit", 20);
|
||||
this.maxConnections = XmlUtils.getNodeValueAsInt(server, "maxConnections", 10);
|
||||
this.triggerBatchSize = XmlUtils.getNodeValueAsInt(server, "triggerBatchSize", 200);
|
||||
this.reIndexBatchSize = XmlUtils.getNodeValueAsInt(server, "reIndexBatchSize", 200);
|
||||
}
|
||||
|
||||
public int getPort() {
|
||||
return this.port;
|
||||
}
|
||||
|
||||
public boolean isDisableLogging() {
|
||||
return this.disableLogging;
|
||||
}
|
||||
|
||||
public long getPerformanceLogFilterMaxTime() {
|
||||
return this.performanceLogFilterMaxTime;
|
||||
}
|
||||
|
||||
public int getMinaReadBufferSize() {
|
||||
return this.minaReadBufferSize;
|
||||
}
|
||||
|
||||
public int getCursorExpirationMinutes() {
|
||||
return this.cursorExpirationMinutes;
|
||||
}
|
||||
|
||||
public int getDefaultCursorLimit() {
|
||||
return this.defaultCursorLimit;
|
||||
}
|
||||
|
||||
public int getMaxConnections() {
|
||||
return this.maxConnections;
|
||||
}
|
||||
|
||||
public int getTriggerBatchSize() {
|
||||
return this.triggerBatchSize;
|
||||
}
|
||||
|
||||
public int getReIndexBatchSize() {
|
||||
return this.reIndexBatchSize;
|
||||
}
|
||||
}
|
||||
|
||||
+126
-126
@@ -1,132 +1,132 @@
|
||||
package com.fangjia.datastore.config;
|
||||
|
||||
import com.fangjia.datastore.ConfigurationException;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import com.fangjia.datastore.utils.XmlUtils;
|
||||
import org.dom4j.Node;
|
||||
|
||||
import com.fangjia.datastore.ConfigurationException;
|
||||
import com.fangjia.datastore.utils.XmlUtils;
|
||||
|
||||
public class SolrConfig {
|
||||
private final String id;
|
||||
private String host;
|
||||
private String home;
|
||||
private final DataStoreConfig dataStoreConfig;
|
||||
private String shards;
|
||||
private String slaves;
|
||||
private SolrConfig[] shardConfigs;
|
||||
private SolrConfig[] slaveConfigs;
|
||||
private String filter;
|
||||
|
||||
SolrConfig(DataStoreConfig dataStoreConfig, String id) {
|
||||
this.dataStoreConfig = dataStoreConfig;
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
synchronized void reload(Node solrNode) throws ConfigurationException {
|
||||
this.host = removeLastSlash(XmlUtils.getNodeValue(solrNode, "host", null));
|
||||
this.shards = XmlUtils.getNodeValue(solrNode, "shards", null);
|
||||
this.slaves = XmlUtils.getNodeValue(solrNode, "slaves", null);
|
||||
this.home = XmlUtils.getNodeValue(solrNode, "home", null);
|
||||
if (StringUtils.isNotEmpty(shards) && StringUtils.isEmpty(home)) {
|
||||
throw new ConfigurationException("[" + id + "] invalid home attribute:" + home);
|
||||
}
|
||||
if (StringUtils.isEmpty(shards) && StringUtils.isEmpty(slaves) && StringUtils.isEmpty(host)) {
|
||||
throw new ConfigurationException("[" + id + "] invalid host attribute:" + host);
|
||||
}
|
||||
this.filter = removeLastSlash(XmlUtils.getNodeValue(solrNode, "filter", null));
|
||||
}
|
||||
|
||||
private SolrConfig[] getConfigs(String ids) {
|
||||
SolrConfig[] ret;
|
||||
if (StringUtils.isBlank(ids)) {
|
||||
ret = new SolrConfig[0];
|
||||
} else {
|
||||
String[] idSplit = ids.split(",");
|
||||
ret = new SolrConfig[idSplit.length];
|
||||
for (int i = 0; i < idSplit.length; i++) {
|
||||
ret[i] = dataStoreConfig.getSolrConfigById(idSplit[i]);
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public SolrConfig[] getShardsConfigs() {
|
||||
if (null == shardConfigs) {
|
||||
shardConfigs = this.getConfigs(shards);
|
||||
}
|
||||
return shardConfigs;
|
||||
}
|
||||
|
||||
public SolrConfig[] getSlaveConfigs() {
|
||||
if (null == slaveConfigs) {
|
||||
slaveConfigs = this.getConfigs(slaves);
|
||||
}
|
||||
return slaveConfigs;
|
||||
}
|
||||
|
||||
private String removeLastSlash(String s) {
|
||||
if (StringUtils.isEmpty(s)) {
|
||||
return s;
|
||||
}
|
||||
if (s.charAt(s.length() - 1) == '/') {
|
||||
return s.substring(0, s.length() - 1);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
public String getShardsParam(String coreName) {
|
||||
SolrConfig[] shardConfigs = getShardsConfigs();
|
||||
if (shardConfigs.length == 0) {
|
||||
return "";
|
||||
}
|
||||
StringBuilder ret = new StringBuilder();
|
||||
for (int i = 0; i < shardConfigs.length; i++) {
|
||||
ret.append(shardConfigs[i].host).append('/').append(coreName);
|
||||
if (i < shardConfigs.length - 1) {
|
||||
ret.append(',');
|
||||
}
|
||||
}
|
||||
return ret.toString();
|
||||
}
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
public DataStoreConfig getDataStoreConfig() {
|
||||
return dataStoreConfig;
|
||||
}
|
||||
|
||||
public String getShards() {
|
||||
return shards;
|
||||
}
|
||||
|
||||
public String getFilter() {
|
||||
return filter;
|
||||
}
|
||||
|
||||
public String getHome() {
|
||||
return home;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder ret = new StringBuilder();
|
||||
ret.append('{');
|
||||
ret.append("id: ").append(this.id);
|
||||
ret.append(", host: ").append(this.host);
|
||||
ret.append(", shards: ").append(this.shards);
|
||||
ret.append(", slaves: ").append(this.slaves);
|
||||
ret.append('}');
|
||||
return ret.toString();
|
||||
}
|
||||
|
||||
public String getSlaves() {
|
||||
return slaves;
|
||||
}
|
||||
|
||||
public class SolrConfig
|
||||
{
|
||||
private final String id;
|
||||
private String host;
|
||||
private String home;
|
||||
private final DataStoreConfig dataStoreConfig;
|
||||
private String shards;
|
||||
private String slaves;
|
||||
private SolrConfig[] shardConfigs;
|
||||
private SolrConfig[] slaveConfigs;
|
||||
private String filter;
|
||||
|
||||
SolrConfig(final DataStoreConfig dataStoreConfig, final String id) {
|
||||
this.dataStoreConfig = dataStoreConfig;
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
synchronized void reload(final Node solrNode) throws ConfigurationException {
|
||||
this.host = this.removeLastSlash(XmlUtils.getNodeValue(solrNode, "host", null));
|
||||
this.shards = XmlUtils.getNodeValue(solrNode, "shards", null);
|
||||
this.slaves = XmlUtils.getNodeValue(solrNode, "slaves", null);
|
||||
this.home = XmlUtils.getNodeValue(solrNode, "home", null);
|
||||
if (StringUtils.isNotEmpty(this.shards) && StringUtils.isEmpty(this.home)) {
|
||||
throw new ConfigurationException("[" + this.id + "] invalid home attribute:" + this.home);
|
||||
}
|
||||
if (StringUtils.isEmpty(this.shards) && StringUtils.isEmpty(this.slaves) && StringUtils.isEmpty(this.host)) {
|
||||
throw new ConfigurationException("[" + this.id + "] invalid host attribute:" + this.host);
|
||||
}
|
||||
this.filter = this.removeLastSlash(XmlUtils.getNodeValue(solrNode, "filter", null));
|
||||
}
|
||||
|
||||
private SolrConfig[] getConfigs(final String ids) {
|
||||
SolrConfig[] ret;
|
||||
if (StringUtils.isBlank(ids)) {
|
||||
ret = new SolrConfig[0];
|
||||
}
|
||||
else {
|
||||
final String[] idSplit = ids.split(",");
|
||||
ret = new SolrConfig[idSplit.length];
|
||||
for (int i = 0; i < idSplit.length; ++i) {
|
||||
ret[i] = this.dataStoreConfig.getSolrConfigById(idSplit[i]);
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public SolrConfig[] getShardsConfigs() {
|
||||
if (null == this.shardConfigs) {
|
||||
this.shardConfigs = this.getConfigs(this.shards);
|
||||
}
|
||||
return this.shardConfigs;
|
||||
}
|
||||
|
||||
public SolrConfig[] getSlaveConfigs() {
|
||||
if (null == this.slaveConfigs) {
|
||||
this.slaveConfigs = this.getConfigs(this.slaves);
|
||||
}
|
||||
return this.slaveConfigs;
|
||||
}
|
||||
|
||||
private String removeLastSlash(final String s) {
|
||||
if (StringUtils.isEmpty(s)) {
|
||||
return s;
|
||||
}
|
||||
if (s.charAt(s.length() - 1) == '/') {
|
||||
return s.substring(0, s.length() - 1);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
public String getShardsParam(final String coreName) {
|
||||
final SolrConfig[] shardConfigs = this.getShardsConfigs();
|
||||
if (shardConfigs.length == 0) {
|
||||
return "";
|
||||
}
|
||||
final StringBuilder ret = new StringBuilder();
|
||||
for (int i = 0; i < shardConfigs.length; ++i) {
|
||||
ret.append(shardConfigs[i].host).append('/').append(coreName);
|
||||
if (i < shardConfigs.length - 1) {
|
||||
ret.append(',');
|
||||
}
|
||||
}
|
||||
return ret.toString();
|
||||
}
|
||||
|
||||
public String getId() {
|
||||
return this.id;
|
||||
}
|
||||
|
||||
public String getHost() {
|
||||
return this.host;
|
||||
}
|
||||
|
||||
public DataStoreConfig getDataStoreConfig() {
|
||||
return this.dataStoreConfig;
|
||||
}
|
||||
|
||||
public String getShards() {
|
||||
return this.shards;
|
||||
}
|
||||
|
||||
public String getFilter() {
|
||||
return this.filter;
|
||||
}
|
||||
|
||||
public String getHome() {
|
||||
return this.home;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder ret = new StringBuilder();
|
||||
ret.append('{');
|
||||
ret.append("id: ").append(this.id);
|
||||
ret.append(", host: ").append(this.host);
|
||||
ret.append(", shards: ").append(this.shards);
|
||||
ret.append(", slaves: ").append(this.slaves);
|
||||
ret.append('}');
|
||||
return ret.toString();
|
||||
}
|
||||
|
||||
public String getSlaves() {
|
||||
return this.slaves;
|
||||
}
|
||||
}
|
||||
|
||||
+200
-221
@@ -1,229 +1,208 @@
|
||||
package com.fangjia.datastore.config;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import java.util.Collection;
|
||||
import org.bson.types.ObjectId;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import com.fangjia.datastore.utils.XmlUtils;
|
||||
import com.fangjia.datastore.ConfigurationException;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.dom4j.Element;
|
||||
import org.dom4j.Node;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import com.fangjia.datastore.ConfigurationException;
|
||||
import com.fangjia.datastore.utils.XmlUtils;
|
||||
|
||||
public final class StoreConfig {
|
||||
|
||||
private final String name;
|
||||
private int solrMaxTotalConnections;
|
||||
private int solrMaxConnectionsPerHost;
|
||||
private SolrConfig[] masterSolrConfigs;
|
||||
private SolrConfig masterSolrConfig;
|
||||
private SolrConfig slaveSolrConfig;
|
||||
private MongoConfig mongoConfig;
|
||||
private boolean textFieldToGridFS;
|
||||
private final Trigger trigger;
|
||||
private final Map<String, Field> fields;
|
||||
private final DataStoreConfig dataStoreConfig;
|
||||
|
||||
StoreConfig(DataStoreConfig dataStoreConfig, String name) {
|
||||
this.dataStoreConfig = dataStoreConfig;
|
||||
fields = new HashMap<String, Field>();
|
||||
trigger = new Trigger();
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
synchronized void reload(Node storeNode) throws ConfigurationException {
|
||||
// init solr
|
||||
String solrMastersAttr = ((Element) storeNode).attributeValue("solrMasters");
|
||||
if (StringUtils.isEmpty(solrMastersAttr)) {
|
||||
throw new ConfigurationException("[" + name + "] invalid solrMasters attribute:" + solrMastersAttr);
|
||||
}
|
||||
String[] solrMasterIds = solrMastersAttr.split(",");
|
||||
if (solrMasterIds.length > 2) {
|
||||
throw new ConfigurationException("[" + name + "] too many solr server configured:" + solrMastersAttr);
|
||||
}
|
||||
masterSolrConfigs = new SolrConfig[solrMasterIds.length];
|
||||
for (int i = 0; i < solrMasterIds.length; i++) {
|
||||
masterSolrConfigs[i] = dataStoreConfig.getSolrConfigById(solrMasterIds[i]);
|
||||
if (null == masterSolrConfigs[i]) {
|
||||
throw new ConfigurationException("[" + name + "] unknown solr id: " + solrMasterIds[i]);
|
||||
}
|
||||
}
|
||||
|
||||
String solrSlaveAttr = System.getProperty("datastore." + name + ".solrSlave", ((Element) storeNode)
|
||||
.attributeValue("solrSlave"));
|
||||
if (StringUtils.isEmpty(solrSlaveAttr)) {
|
||||
throw new ConfigurationException("[" + name + "] invalid solrSlave attribute:" + solrSlaveAttr);
|
||||
}
|
||||
slaveSolrConfig = dataStoreConfig.getSolrConfigById(solrSlaveAttr);
|
||||
if (null == slaveSolrConfig) {
|
||||
throw new ConfigurationException("[" + name + "] unknown solr id: " + solrSlaveAttr);
|
||||
}
|
||||
|
||||
String solrMasterAttr = System.getProperty("datastore." + name + ".solrMaster", ((Element) storeNode)
|
||||
.attributeValue("solrMaster"));
|
||||
if (StringUtils.isEmpty(solrMasterAttr)) {
|
||||
throw new ConfigurationException("[" + name + "] invalid solrMaster attribute:" + solrMasterAttr);
|
||||
}
|
||||
masterSolrConfig = dataStoreConfig.getSolrConfigById(solrMasterAttr);
|
||||
if (null == masterSolrConfig) {
|
||||
throw new ConfigurationException("[" + name + "] unknown solr id: " + solrMasterAttr);
|
||||
}
|
||||
|
||||
if (slaveSolrConfig.getShardsConfigs().length != masterSolrConfig.getShardsConfigs().length) {
|
||||
throw new ConfigurationException("[" + name + "] master and slave shard count must equals !");
|
||||
}
|
||||
|
||||
// solr connections defaults
|
||||
this.solrMaxConnectionsPerHost = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxConnectionsPerHost", 300);
|
||||
if (slaveSolrConfig.getShardsConfigs().length > 0) {
|
||||
this.solrMaxTotalConnections = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxTotalConnections", slaveSolrConfig
|
||||
.getShardsConfigs().length
|
||||
* solrMaxConnectionsPerHost);
|
||||
} else {
|
||||
this.solrMaxTotalConnections = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxTotalConnections",
|
||||
solrMaxConnectionsPerHost);
|
||||
}
|
||||
|
||||
// init mongo
|
||||
String mongoId = ((Element) storeNode).attributeValue("mongo");
|
||||
MongoConfig mongoConfig = dataStoreConfig.getMongoConfigById(mongoId);
|
||||
if (null == mongoConfig) {
|
||||
throw new ConfigurationException("[" + name + "] unknown mongo id: " + mongoId);
|
||||
}
|
||||
this.mongoConfig = mongoConfig;
|
||||
|
||||
// init other config
|
||||
this.textFieldToGridFS = XmlUtils.getNodeValueAsBoolean(storeNode, "textFieldToGridFS", false);
|
||||
|
||||
// init fields
|
||||
Map<String, Field> newFields = new HashMap<String, Field>();
|
||||
List<Node> fieldNodes = storeNode.selectNodes("schema//field");
|
||||
for (Node fieldNode : fieldNodes) {
|
||||
Element element = (Element) fieldNode;
|
||||
String fieldName = element.attributeValue("name");
|
||||
Field field = getOrCreateField(fieldName);
|
||||
field.setName(fieldName);
|
||||
field.setRequired(Boolean.parseBoolean(element.attributeValue("required")));
|
||||
field.setType(FieldTypeEnum.parse(element.attributeValue("type")));
|
||||
newFields.put(fieldName, field);
|
||||
}
|
||||
this.fields.clear();
|
||||
this.fields.putAll(newFields);
|
||||
|
||||
// init triggers
|
||||
Element triggerNode = (Element) storeNode.selectSingleNode("trigger");
|
||||
if(triggerNode != null) {
|
||||
trigger.setInterface(triggerNode.attributeValue("interface"));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public Trigger getTrigger() {
|
||||
return this.trigger;
|
||||
}
|
||||
|
||||
public Field getField(String fieldName) {
|
||||
return this.fields.get(fieldName);
|
||||
}
|
||||
|
||||
public boolean containsField(String fieldName) {
|
||||
return this.fields.containsKey(fieldName);
|
||||
}
|
||||
|
||||
public boolean isFieldValueLegal(String fieldName, Object value) {
|
||||
if ("_id".equals(fieldName)) {
|
||||
return value instanceof ObjectId;
|
||||
}
|
||||
if (fields.containsKey(fieldName)) {
|
||||
Field field = fields.get(fieldName);
|
||||
return field.validate(value);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean isBigField(String fieldName) {
|
||||
return fields.get(fieldName).isBigField();
|
||||
}
|
||||
|
||||
public Collection<Field> getFields() {
|
||||
return Collections.unmodifiableCollection(this.fields.values());
|
||||
}
|
||||
|
||||
synchronized Field getOrCreateField(String fieldName) {
|
||||
if (StringUtils.isEmpty(fieldName)) {
|
||||
throw new IllegalArgumentException("fieldName can not be null or empty!");
|
||||
}
|
||||
Field ret = this.fields.get(fieldName);
|
||||
if (null == ret) {
|
||||
ret = new Field();
|
||||
ret.setName(fieldName);
|
||||
this.fields.put(fieldName, ret);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public MongoConfig getMongoConfig() {
|
||||
return mongoConfig;
|
||||
}
|
||||
|
||||
public DataStoreConfig getDataStoreConfig() {
|
||||
return dataStoreConfig;
|
||||
}
|
||||
|
||||
public boolean isTextFieldToGridFS() {
|
||||
return textFieldToGridFS;
|
||||
}
|
||||
|
||||
public SolrConfig getSlaveSolrConfig() {
|
||||
return slaveSolrConfig;
|
||||
}
|
||||
|
||||
public SolrConfig[] getMasterSolrConfigs() {
|
||||
return masterSolrConfigs;
|
||||
}
|
||||
|
||||
public SolrConfig getMasterSolrConfig() {
|
||||
return masterSolrConfig;
|
||||
}
|
||||
|
||||
public int getSolrMaxTotalConnections() {
|
||||
return solrMaxTotalConnections;
|
||||
}
|
||||
|
||||
public int getSolrMaxConnectionsPerHost() {
|
||||
return solrMaxConnectionsPerHost;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder ret = new StringBuilder();
|
||||
ret.append("\n{");
|
||||
ret.append("\n\tname: ").append(name);
|
||||
ret.append(",\n\tmasterSolrConfig: ").append(masterSolrConfig);
|
||||
ret.append(",\n\tslaveSolrConfig: ").append(this.slaveSolrConfig);
|
||||
ret.append(",\n\tmongoConfig: ").append(mongoConfig);
|
||||
ret.append(",\n\tsolrMaxTotalConnections: ").append(solrMaxTotalConnections);
|
||||
ret.append(",\n\tsolrMaxConnectionsPerHost: ").append(this.solrMaxConnectionsPerHost);
|
||||
ret.append(",\n\ttextFieldToGridFS: ").append(this.textFieldToGridFS);
|
||||
ret.append(",\n\ttrigger: ").append(trigger);
|
||||
ret.append(",\n\tmasterSolrConfigs: [\n");
|
||||
for (int i = 0; i < this.masterSolrConfigs.length; i++) {
|
||||
ret.append('\t').append('\t').append(this.masterSolrConfigs[i]);
|
||||
if (i < this.masterSolrConfigs.length - 1) {
|
||||
ret.append(',');
|
||||
}
|
||||
ret.append('\n');
|
||||
}
|
||||
ret.append("\t]\n}");
|
||||
return ret.toString();
|
||||
}
|
||||
|
||||
public final class StoreConfig
|
||||
{
|
||||
private final String name;
|
||||
private int solrMaxTotalConnections;
|
||||
private int solrMaxConnectionsPerHost;
|
||||
private SolrConfig[] masterSolrConfigs;
|
||||
private SolrConfig masterSolrConfig;
|
||||
private SolrConfig slaveSolrConfig;
|
||||
private MongoConfig mongoConfig;
|
||||
private boolean textFieldToGridFS;
|
||||
private final Trigger trigger;
|
||||
private final Map<String, Field> fields;
|
||||
private final DataStoreConfig dataStoreConfig;
|
||||
|
||||
StoreConfig(final DataStoreConfig dataStoreConfig, final String name) {
|
||||
this.dataStoreConfig = dataStoreConfig;
|
||||
this.fields = new HashMap<String, Field>();
|
||||
this.trigger = new Trigger();
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
synchronized void reload(final Node storeNode) throws ConfigurationException {
|
||||
final String solrMastersAttr = ((Element)storeNode).attributeValue("solrMasters");
|
||||
if (StringUtils.isEmpty(solrMastersAttr)) {
|
||||
throw new ConfigurationException("[" + this.name + "] invalid solrMasters attribute:" + solrMastersAttr);
|
||||
}
|
||||
final String[] solrMasterIds = solrMastersAttr.split(",");
|
||||
if (solrMasterIds.length > 2) {
|
||||
throw new ConfigurationException("[" + this.name + "] too many solr server configured:" + solrMastersAttr);
|
||||
}
|
||||
this.masterSolrConfigs = new SolrConfig[solrMasterIds.length];
|
||||
for (int i = 0; i < solrMasterIds.length; ++i) {
|
||||
this.masterSolrConfigs[i] = this.dataStoreConfig.getSolrConfigById(solrMasterIds[i]);
|
||||
if (null == this.masterSolrConfigs[i]) {
|
||||
throw new ConfigurationException("[" + this.name + "] unknown solr id: " + solrMasterIds[i]);
|
||||
}
|
||||
}
|
||||
final String solrSlaveAttr = System.getProperty("datastore." + this.name + ".solrSlave", ((Element)storeNode).attributeValue("solrSlave"));
|
||||
if (StringUtils.isEmpty(solrSlaveAttr)) {
|
||||
throw new ConfigurationException("[" + this.name + "] invalid solrSlave attribute:" + solrSlaveAttr);
|
||||
}
|
||||
this.slaveSolrConfig = this.dataStoreConfig.getSolrConfigById(solrSlaveAttr);
|
||||
if (null == this.slaveSolrConfig) {
|
||||
throw new ConfigurationException("[" + this.name + "] unknown solr id: " + solrSlaveAttr);
|
||||
}
|
||||
final String solrMasterAttr = System.getProperty("datastore." + this.name + ".solrMaster", ((Element)storeNode).attributeValue("solrMaster"));
|
||||
if (StringUtils.isEmpty(solrMasterAttr)) {
|
||||
throw new ConfigurationException("[" + this.name + "] invalid solrMaster attribute:" + solrMasterAttr);
|
||||
}
|
||||
this.masterSolrConfig = this.dataStoreConfig.getSolrConfigById(solrMasterAttr);
|
||||
if (null == this.masterSolrConfig) {
|
||||
throw new ConfigurationException("[" + this.name + "] unknown solr id: " + solrMasterAttr);
|
||||
}
|
||||
if (this.slaveSolrConfig.getShardsConfigs().length != this.masterSolrConfig.getShardsConfigs().length) {
|
||||
throw new ConfigurationException("[" + this.name + "] master and slave shard count must equals !");
|
||||
}
|
||||
this.solrMaxConnectionsPerHost = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxConnectionsPerHost", 300);
|
||||
if (this.slaveSolrConfig.getShardsConfigs().length > 0) {
|
||||
this.solrMaxTotalConnections = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxTotalConnections", this.slaveSolrConfig.getShardsConfigs().length * this.solrMaxConnectionsPerHost);
|
||||
}
|
||||
else {
|
||||
this.solrMaxTotalConnections = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxTotalConnections", this.solrMaxConnectionsPerHost);
|
||||
}
|
||||
final String mongoId = ((Element)storeNode).attributeValue("mongo");
|
||||
final MongoConfig mongoConfig = this.dataStoreConfig.getMongoConfigById(mongoId);
|
||||
if (null == mongoConfig) {
|
||||
throw new ConfigurationException("[" + this.name + "] unknown mongo id: " + mongoId);
|
||||
}
|
||||
this.mongoConfig = mongoConfig;
|
||||
this.textFieldToGridFS = XmlUtils.getNodeValueAsBoolean(storeNode, "textFieldToGridFS", false);
|
||||
final Map<String, Field> newFields = new HashMap<String, Field>();
|
||||
final List<Node> fieldNodes = (List<Node>)storeNode.selectNodes("schema//field");
|
||||
for (final Node fieldNode : fieldNodes) {
|
||||
final Element element = (Element)fieldNode;
|
||||
final String fieldName = element.attributeValue("name");
|
||||
final Field field = this.getOrCreateField(fieldName);
|
||||
field.setName(fieldName);
|
||||
field.setRequired(Boolean.parseBoolean(element.attributeValue("required")));
|
||||
field.setType(FieldTypeEnum.parse(element.attributeValue("type")));
|
||||
newFields.put(fieldName, field);
|
||||
}
|
||||
this.fields.clear();
|
||||
this.fields.putAll(newFields);
|
||||
final Element triggerNode = (Element)storeNode.selectSingleNode("trigger");
|
||||
if (triggerNode != null) {
|
||||
this.trigger.setInterface(triggerNode.attributeValue("interface"));
|
||||
}
|
||||
}
|
||||
|
||||
public Trigger getTrigger() {
|
||||
return this.trigger;
|
||||
}
|
||||
|
||||
public Field getField(final String fieldName) {
|
||||
return this.fields.get(fieldName);
|
||||
}
|
||||
|
||||
public boolean containsField(final String fieldName) {
|
||||
return this.fields.containsKey(fieldName);
|
||||
}
|
||||
|
||||
public boolean isFieldValueLegal(final String fieldName, final Object value) {
|
||||
if ("_id".equals(fieldName)) {
|
||||
return value instanceof ObjectId;
|
||||
}
|
||||
if (this.fields.containsKey(fieldName)) {
|
||||
final Field field = this.fields.get(fieldName);
|
||||
return field.validate(value);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean isBigField(final String fieldName) {
|
||||
return this.fields.get(fieldName).isBigField();
|
||||
}
|
||||
|
||||
public Collection<Field> getFields() {
|
||||
return Collections.<Field>unmodifiableCollection((Collection<? extends Field>)this.fields.values());
|
||||
}
|
||||
|
||||
synchronized Field getOrCreateField(final String fieldName) {
|
||||
if (StringUtils.isEmpty(fieldName)) {
|
||||
throw new IllegalArgumentException("fieldName can not be null or empty!");
|
||||
}
|
||||
Field ret = this.fields.get(fieldName);
|
||||
if (null == ret) {
|
||||
ret = new Field();
|
||||
ret.setName(fieldName);
|
||||
this.fields.put(fieldName, ret);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
public MongoConfig getMongoConfig() {
|
||||
return this.mongoConfig;
|
||||
}
|
||||
|
||||
public DataStoreConfig getDataStoreConfig() {
|
||||
return this.dataStoreConfig;
|
||||
}
|
||||
|
||||
public boolean isTextFieldToGridFS() {
|
||||
return this.textFieldToGridFS;
|
||||
}
|
||||
|
||||
public SolrConfig getSlaveSolrConfig() {
|
||||
return this.slaveSolrConfig;
|
||||
}
|
||||
|
||||
public SolrConfig[] getMasterSolrConfigs() {
|
||||
return this.masterSolrConfigs;
|
||||
}
|
||||
|
||||
public SolrConfig getMasterSolrConfig() {
|
||||
return this.masterSolrConfig;
|
||||
}
|
||||
|
||||
public int getSolrMaxTotalConnections() {
|
||||
return this.solrMaxTotalConnections;
|
||||
}
|
||||
|
||||
public int getSolrMaxConnectionsPerHost() {
|
||||
return this.solrMaxConnectionsPerHost;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder ret = new StringBuilder();
|
||||
ret.append("\n{");
|
||||
ret.append("\n\tname: ").append(this.name);
|
||||
ret.append(",\n\tmasterSolrConfig: ").append(this.masterSolrConfig);
|
||||
ret.append(",\n\tslaveSolrConfig: ").append(this.slaveSolrConfig);
|
||||
ret.append(",\n\tmongoConfig: ").append(this.mongoConfig);
|
||||
ret.append(",\n\tsolrMaxTotalConnections: ").append(this.solrMaxTotalConnections);
|
||||
ret.append(",\n\tsolrMaxConnectionsPerHost: ").append(this.solrMaxConnectionsPerHost);
|
||||
ret.append(",\n\ttextFieldToGridFS: ").append(this.textFieldToGridFS);
|
||||
ret.append(",\n\ttrigger: ").append(this.trigger);
|
||||
ret.append(",\n\tmasterSolrConfigs: [\n");
|
||||
for (int i = 0; i < this.masterSolrConfigs.length; ++i) {
|
||||
ret.append('\t').append('\t').append(this.masterSolrConfigs[i]);
|
||||
if (i < this.masterSolrConfigs.length - 1) {
|
||||
ret.append(',');
|
||||
}
|
||||
ret.append('\n');
|
||||
}
|
||||
ret.append("\t]\n}");
|
||||
return ret.toString();
|
||||
}
|
||||
}
|
||||
|
||||
+43
-43
@@ -1,49 +1,49 @@
|
||||
package com.fangjia.datastore.config;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.commons.lang.builder.EqualsBuilder;
|
||||
import org.apache.commons.lang.builder.HashCodeBuilder;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
public final class Trigger {
|
||||
private String interface_;
|
||||
|
||||
Trigger() {
|
||||
}
|
||||
|
||||
public String getInterface() {
|
||||
return interface_;
|
||||
}
|
||||
|
||||
void setInterface(String interface_) {
|
||||
if (StringUtils.isEmpty(interface_) || !interface_.startsWith("http://")) {
|
||||
throw new IllegalArgumentException("invalid trigger interface: " + interface_);
|
||||
}
|
||||
this.interface_ = interface_;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return new HashCodeBuilder().append(this.interface_).toHashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (null == obj) {
|
||||
return false;
|
||||
}
|
||||
if (this == obj) {
|
||||
return true;
|
||||
}
|
||||
if (!(obj instanceof Trigger)) {
|
||||
return false;
|
||||
}
|
||||
Trigger other = (Trigger) obj;
|
||||
return new EqualsBuilder().append(this.interface_, other.interface_).isEquals();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return new StringBuilder().append("{interface=").append(interface_).append("}").toString();
|
||||
}
|
||||
|
||||
public final class Trigger
|
||||
{
|
||||
private String interface_;
|
||||
|
||||
Trigger() {
|
||||
}
|
||||
|
||||
public String getInterface() {
|
||||
return this.interface_;
|
||||
}
|
||||
|
||||
void setInterface(final String interface_) {
|
||||
if (StringUtils.isEmpty(interface_) || !interface_.startsWith("http://")) {
|
||||
throw new IllegalArgumentException("invalid trigger interface: " + interface_);
|
||||
}
|
||||
this.interface_ = interface_;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return new HashCodeBuilder().append((Object)this.interface_).toHashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(final Object obj) {
|
||||
if (null == obj) {
|
||||
return false;
|
||||
}
|
||||
if (this == obj) {
|
||||
return true;
|
||||
}
|
||||
if (!(obj instanceof Trigger)) {
|
||||
return false;
|
||||
}
|
||||
final Trigger other = (Trigger)obj;
|
||||
return new EqualsBuilder().append((Object)this.interface_, (Object)other.interface_).isEquals();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "{interface=" + this.interface_ + "}";
|
||||
}
|
||||
}
|
||||
|
||||
+13
-12
@@ -1,19 +1,20 @@
|
||||
package com.fangjia.datastore.handler;
|
||||
|
||||
import com.fangjia.datastore.mongo.protocol.MongoMessage;
|
||||
import com.fangjia.datastore.DataStore;
|
||||
import com.fangjia.datastore.DataStoreContext;
|
||||
import com.fangjia.datastore.mongo.protocol.DeleteMessage;
|
||||
|
||||
public class DeleteMessageHandler extends MessageHandler<DeleteMessage> {
|
||||
|
||||
public DeleteMessageHandler(DataStoreContext dataStoreContext) {
|
||||
super(dataStoreContext);
|
||||
}
|
||||
|
||||
public void messageReceived(DeleteMessage message,String ip) throws Exception {
|
||||
assertNotReadOnly();
|
||||
DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName());
|
||||
dataStore.remove(message.getSelector());
|
||||
}
|
||||
|
||||
public class DeleteMessageHandler extends MessageHandler<DeleteMessage>
|
||||
{
|
||||
public DeleteMessageHandler(final DataStoreContext dataStoreContext) {
|
||||
super(dataStoreContext);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void messageReceived(final DeleteMessage message) throws Exception {
|
||||
this.assertNotReadOnly();
|
||||
final DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName());
|
||||
dataStore.remove(message.getSelector());
|
||||
}
|
||||
}
|
||||
|
||||
+52
-47
@@ -1,54 +1,59 @@
|
||||
package com.fangjia.datastore.handler;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.bson.BSONObject;
|
||||
|
||||
import com.fangjia.datastore.mongo.protocol.MongoMessage;
|
||||
import com.mongodb.DBObject;
|
||||
import com.fangjia.datastore.QueryResponse;
|
||||
import com.fangjia.datastore.mongo.protocol.QueryMessage;
|
||||
import com.fangjia.datastore.Cursor;
|
||||
import com.fangjia.datastore.DataStore;
|
||||
import com.fangjia.datastore.DataStoreContext;
|
||||
import com.fangjia.datastore.QueryResponse;
|
||||
import com.fangjia.datastore.mongo.protocol.GetMoreMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.QueryMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.ReplyMessage;
|
||||
import org.bson.BSONObject;
|
||||
import java.util.List;
|
||||
import com.fangjia.datastore.DataStoreContext;
|
||||
import com.fangjia.datastore.mongo.protocol.GetMoreMessage;
|
||||
|
||||
public class GetMoreMessageHandler extends MessageHandler<GetMoreMessage> {
|
||||
|
||||
public GetMoreMessageHandler(DataStoreContext dataStoreContext) {
|
||||
super(dataStoreContext);
|
||||
}
|
||||
|
||||
public void messageReceived(GetMoreMessage message) throws Exception {
|
||||
String fullCollectionName = message.getFullCollectionName();
|
||||
if(fullCollectionName.contains("$cmd")){
|
||||
ReplyMessage reply = prepareReplyMessage(message.getMessageHeader().getRequestID(), (List<? extends BSONObject>) null);
|
||||
reply.setStartingFrom(0);
|
||||
this.writeMessage(reply);
|
||||
this.dataStoreContext.removeCursor(message.getCursorID());
|
||||
return;
|
||||
}
|
||||
DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName());
|
||||
Cursor cursor = this.dataStoreContext.getCursor(message.getCursorID());
|
||||
if (null == cursor) {
|
||||
ReplyMessage reply = prepareReplyMessage(message.getMessageHeader().getRequestID(), (List<? extends BSONObject>) null);
|
||||
reply.setStartingFrom(0);
|
||||
this.writeMessage(reply);
|
||||
} else {
|
||||
QueryMessage queryMessage = cursor.getQueryMessage();
|
||||
QueryResponse response = dataStore.find(queryMessage.getRealQuery(), queryMessage.getReturnFieldSelector(),
|
||||
queryMessage.getOrderByQuery(), queryMessage.getFacetQuery(), cursor.nextSkip(),
|
||||
message.getNumberToReturn() - 1, queryMessage.isBulkQuery(), queryMessage.isFromMaster());
|
||||
ReplyMessage reply = prepareReplyMessage(message.getMessageHeader().getRequestID(), response.getDocList());
|
||||
reply.setStartingFrom(cursor.getSkip());
|
||||
if (response.hasMore()) {
|
||||
// cursor.setTotalCount(response.getNumFound());
|
||||
// cursor.setLimit((int) response.getNumReturned());
|
||||
// this.dataStoreContext.addCursor(cursor);
|
||||
// reply.setCursorID(cursor.getId());
|
||||
} else {
|
||||
this.dataStoreContext.removeCursor(cursor.getId());
|
||||
}
|
||||
this.writeMessage(reply);
|
||||
}
|
||||
}
|
||||
public class GetMoreMessageHandler extends MessageHandler<GetMoreMessage>
|
||||
{
|
||||
public GetMoreMessageHandler(final DataStoreContext dataStoreContext) {
|
||||
super(dataStoreContext);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void messageReceived(final GetMoreMessage message) throws Exception {
|
||||
final String fullCollectionName = message.getFullCollectionName();
|
||||
if (fullCollectionName.contains("$cmd")) {
|
||||
final ReplyMessage reply = this.prepareReplyMessage(message.getMessageHeader().getRequestID(), (List<? extends BSONObject>)null);
|
||||
reply.setStartingFrom(0);
|
||||
this.writeMessage(reply);
|
||||
this.dataStoreContext.removeCursor(message.getCursorID());
|
||||
return;
|
||||
}
|
||||
final DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName());
|
||||
final Cursor cursor = this.dataStoreContext.getCursor(message.getCursorID());
|
||||
if (null == cursor) {
|
||||
final ReplyMessage reply2 = this.prepareReplyMessage(message.getMessageHeader().getRequestID(), (List<? extends BSONObject>)null);
|
||||
reply2.setStartingFrom(0);
|
||||
this.writeMessage(reply2);
|
||||
}
|
||||
else {
|
||||
final QueryMessage queryMessage = cursor.getQueryMessage();
|
||||
final QueryResponse response = dataStore.find(queryMessage.getRealQuery(), queryMessage.getReturnFieldSelector(), queryMessage.getOrderByQuery(), queryMessage.getFacetQuery(), cursor.nextSkip(), this.useCursor() ? message.getNumberToReturn() : (message.getNumberToReturn() - 1), queryMessage.isBulkQuery(), queryMessage.isFromMaster());
|
||||
final List<DBObject> doc = response.getDocList();
|
||||
if (this.useCursor()) {
|
||||
doc.remove(0);
|
||||
}
|
||||
final ReplyMessage reply3 = this.prepareReplyMessage(message.getMessageHeader().getRequestID(), (List<? extends BSONObject>)doc);
|
||||
reply3.setStartingFrom(cursor.getSkip());
|
||||
if (response.hasMore() && this.useCursor()) {
|
||||
cursor.setTotalCount(response.getNumFound());
|
||||
cursor.setLimit(response.getNumReturned());
|
||||
this.dataStoreContext.addCursor(cursor);
|
||||
reply3.setCursorID(cursor.getId());
|
||||
}
|
||||
else {
|
||||
this.dataStoreContext.removeCursor(cursor.getId());
|
||||
}
|
||||
this.writeMessage(reply3);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+109
-107
@@ -1,115 +1,117 @@
|
||||
package com.fangjia.datastore.handler;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.fangjia.datastore.mongo.protocol.MongoMessage;
|
||||
import java.util.Iterator;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.bson.BSONObject;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import com.fangjia.datastore.solr.Index;
|
||||
import java.util.Collection;
|
||||
import com.fangjia.datastore.DataStore;
|
||||
import org.bson.BSONObject;
|
||||
import java.util.List;
|
||||
import com.fangjia.datastore.DataStoreContext;
|
||||
import com.fangjia.datastore.mongo.protocol.InsertMessage;
|
||||
import com.fangjia.datastore.solr.Index;
|
||||
import com.fangjia.datastore.solr.Index.IndexTypeEnum;
|
||||
|
||||
public class InsertMessageHandler extends MessageHandler<InsertMessage> {
|
||||
|
||||
public InsertMessageHandler(DataStoreContext dataStoreContext) {
|
||||
super(dataStoreContext);
|
||||
}
|
||||
|
||||
public void messageReceived(InsertMessage message,String ip) throws Exception {
|
||||
if (message.getFullCollectionName().endsWith("system.indexes")) {
|
||||
ensureIndex(message);
|
||||
} else {
|
||||
insert(message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* db.foo.insert()
|
||||
*
|
||||
* @param message
|
||||
* @throws Exception
|
||||
*/
|
||||
private void insert(InsertMessage message) throws Exception {
|
||||
assertNotReadOnly();
|
||||
DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName());
|
||||
dataStore.save(message.getDocuments());
|
||||
}
|
||||
|
||||
/**
|
||||
* db.foo.ensureIndex()
|
||||
*
|
||||
* @param message
|
||||
* @throws Exception
|
||||
*/
|
||||
private void ensureIndex(InsertMessage message) throws Exception {
|
||||
assertNotReadOnly();
|
||||
DataStore dataStore = this.dataStoreContext.getDataStore(String.valueOf(message.getDocuments().get(0).get("ns")));
|
||||
List<Index> indexList = addToIndexList(message.getDocuments());
|
||||
dataStore.ensureIndex(indexList);
|
||||
}
|
||||
|
||||
private List<Index> addToIndexList(List<BSONObject> docList) {
|
||||
if (null == docList || docList.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
List<Index> indexList = new ArrayList<Index>();
|
||||
for (BSONObject bson : docList) {
|
||||
BSONObject keys = (BSONObject) bson.get("key");
|
||||
if (null == keys) {
|
||||
continue;
|
||||
}
|
||||
for (String key : keys.keySet()) {
|
||||
Object value = keys.get(key);
|
||||
if (value instanceof String) {
|
||||
Index index = new Index(key, IndexTypeEnum.parse((String) value));
|
||||
indexList.add(index);
|
||||
} else if (value instanceof BSONObject) {
|
||||
// TODO:
|
||||
Index index = new Index();
|
||||
index.setName(key);
|
||||
for (String k : ((BSONObject) value).keySet()) {
|
||||
Object v = ((BSONObject) value).get(k);
|
||||
if (StringUtils.equalsIgnoreCase(k, "type")) {
|
||||
if (v instanceof String) {
|
||||
index.setType(IndexTypeEnum.parse((String) v));
|
||||
}
|
||||
} else if (StringUtils.equalsIgnoreCase(k, "stored")) {
|
||||
if (v instanceof Boolean) {
|
||||
index.setStored((Boolean) v);
|
||||
}
|
||||
} else if (StringUtils.equalsIgnoreCase(k, "indexed")) {
|
||||
if (v instanceof Boolean) {
|
||||
index.setIndexed((Boolean) v);
|
||||
}
|
||||
|
||||
} else if (StringUtils.equalsIgnoreCase(k, "multiValued")) {
|
||||
if (v instanceof Boolean) {
|
||||
index.setMultiValued((Boolean) v);
|
||||
}
|
||||
} else if (StringUtils.equalsIgnoreCase(k, "dynamic")) {
|
||||
if (v instanceof Boolean) {
|
||||
index.setDynamic((Boolean) v);
|
||||
}
|
||||
} else if (StringUtils.equalsIgnoreCase(k, "comments")) {
|
||||
if (v instanceof String) {
|
||||
index.setComments((String) v);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (index.getType() == null) {
|
||||
throw new IllegalArgumentException("illegal schema field: type");
|
||||
}
|
||||
indexList.add(index);
|
||||
} else {
|
||||
if (logger.isErrorEnabled()) {
|
||||
logger.error("invalid key value of ensureIndex:{}", value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return indexList;
|
||||
}
|
||||
public class InsertMessageHandler extends MessageHandler<InsertMessage>
|
||||
{
|
||||
public InsertMessageHandler(final DataStoreContext dataStoreContext) {
|
||||
super(dataStoreContext);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void messageReceived(final InsertMessage message) throws Exception {
|
||||
if (message.getFullCollectionName().endsWith("system.indexes")) {
|
||||
this.ensureIndex(message);
|
||||
}
|
||||
else {
|
||||
this.insert(message);
|
||||
}
|
||||
}
|
||||
|
||||
private void insert(final InsertMessage message) throws Exception {
|
||||
this.assertNotReadOnly();
|
||||
final DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName());
|
||||
dataStore.save(message.getDocuments());
|
||||
}
|
||||
|
||||
private void ensureIndex(final InsertMessage message) throws Exception {
|
||||
this.assertNotReadOnly();
|
||||
final DataStore dataStore = this.dataStoreContext.getDataStore(String.valueOf(message.getDocuments().get(0).get("ns")));
|
||||
final List<Index> indexList = this.addToIndexList(message.getDocuments());
|
||||
dataStore.ensureIndex(indexList);
|
||||
}
|
||||
|
||||
private List<Index> addToIndexList(final List<BSONObject> docList) {
|
||||
if (null == docList || docList.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
final List<Index> indexList = new ArrayList<Index>();
|
||||
for (final BSONObject bson : docList) {
|
||||
final BSONObject keys = (BSONObject)bson.get("key");
|
||||
if (null == keys) {
|
||||
continue;
|
||||
}
|
||||
for (final String key : keys.keySet()) {
|
||||
final Object value = keys.get(key);
|
||||
if (value instanceof String) {
|
||||
final Index index = new Index(key, Index.IndexTypeEnum.parse((String)value));
|
||||
indexList.add(index);
|
||||
}
|
||||
else if (value instanceof BSONObject) {
|
||||
final Index index = new Index();
|
||||
index.setName(key);
|
||||
for (final String k : ((BSONObject)value).keySet()) {
|
||||
final Object v = ((BSONObject)value).get(k);
|
||||
if (StringUtils.equalsIgnoreCase(k, "type")) {
|
||||
if (!(v instanceof String)) {
|
||||
continue;
|
||||
}
|
||||
index.setType(Index.IndexTypeEnum.parse((String)v));
|
||||
}
|
||||
else if (StringUtils.equalsIgnoreCase(k, "stored")) {
|
||||
if (!(v instanceof Boolean)) {
|
||||
continue;
|
||||
}
|
||||
index.setStored((Boolean) v);
|
||||
}
|
||||
else if (StringUtils.equalsIgnoreCase(k, "indexed")) {
|
||||
if (!(v instanceof Boolean)) {
|
||||
continue;
|
||||
}
|
||||
index.setIndexed((Boolean)v);
|
||||
}
|
||||
else if (StringUtils.equalsIgnoreCase(k, "multiValued")) {
|
||||
if (!(v instanceof Boolean)) {
|
||||
continue;
|
||||
}
|
||||
index.setMultiValued((Boolean)v);
|
||||
}
|
||||
else if (StringUtils.equalsIgnoreCase(k, "dynamic")) {
|
||||
if (!(v instanceof Boolean)) {
|
||||
continue;
|
||||
}
|
||||
index.setDynamic((Boolean)v);
|
||||
}
|
||||
else {
|
||||
if (!StringUtils.equalsIgnoreCase(k, "comments") || !(v instanceof String)) {
|
||||
continue;
|
||||
}
|
||||
index.setComments((String)v);
|
||||
}
|
||||
}
|
||||
if (index.getType() == null) {
|
||||
throw new IllegalArgumentException("illegal schema field: type");
|
||||
}
|
||||
indexList.add(index);
|
||||
}
|
||||
else {
|
||||
if (!this.logger.isErrorEnabled()) {
|
||||
continue;
|
||||
}
|
||||
this.logger.error("invalid key value of ensureIndex:{}", value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return indexList;
|
||||
}
|
||||
}
|
||||
|
||||
+17
-16
@@ -1,23 +1,24 @@
|
||||
package com.fangjia.datastore.handler;
|
||||
|
||||
import com.fangjia.datastore.mongo.protocol.MongoMessage;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
import com.fangjia.datastore.DataStoreContext;
|
||||
import com.fangjia.datastore.mongo.protocol.KillCursorsMessage;
|
||||
|
||||
public class KillCursorsMessageHandler extends MessageHandler<KillCursorsMessage> {
|
||||
|
||||
public KillCursorsMessageHandler(DataStoreContext dataStoreContext) {
|
||||
super(dataStoreContext);
|
||||
}
|
||||
|
||||
public void messageReceived(KillCursorsMessage message,String ip) {
|
||||
List<Long> cursorIds = message.getCursorIDs();
|
||||
if (cursorIds != null) {
|
||||
for (Long cursorId : cursorIds) {
|
||||
this.dataStoreContext.removeCursor(cursorId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class KillCursorsMessageHandler extends MessageHandler<KillCursorsMessage>
|
||||
{
|
||||
public KillCursorsMessageHandler(final DataStoreContext dataStoreContext) {
|
||||
super(dataStoreContext);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void messageReceived(final KillCursorsMessage message) {
|
||||
final List<Long> cursorIds = message.getCursorIDs();
|
||||
if (cursorIds != null) {
|
||||
for (final Long cursorId : cursorIds) {
|
||||
this.dataStoreContext.removeCursor(cursorId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+172
-176
@@ -1,184 +1,180 @@
|
||||
package com.fangjia.datastore.handler;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.mina.core.session.IoSession;
|
||||
import org.bson.BSONObject;
|
||||
import org.bson.BasicBSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.fangjia.datastore.DataStoreContext;
|
||||
import com.fangjia.datastore.DataStoreException;
|
||||
import com.fangjia.datastore.mongo.protocol.MongoMessage;
|
||||
import java.util.Arrays;
|
||||
import org.bson.BasicBSONObject;
|
||||
import java.util.ArrayList;
|
||||
import org.bson.BSONObject;
|
||||
import java.util.List;
|
||||
import com.fangjia.datastore.mongo.protocol.ReplyMessage;
|
||||
import java.net.InetSocketAddress;
|
||||
import org.apache.mina.core.session.IoSession;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.slf4j.Logger;
|
||||
import com.fangjia.datastore.DataStoreContext;
|
||||
import java.util.Map;
|
||||
import com.fangjia.datastore.mongo.protocol.MongoMessage;
|
||||
|
||||
public abstract class MessageHandler<T extends MongoMessage> {
|
||||
private final static String LAST_ERROR_KEY = "com.goojia.datastore.LAST_ERROR_KEY";
|
||||
private final static String PREV_ERROR_KEY = "com.goojia.datastore.PREV_ERROR_KEY";
|
||||
|
||||
private final static String SESSION_KEY = "com.goojia.datastore.SESSION_KEY";
|
||||
private final static String ERROR_OCCURRED_KEY = "com.goojia.datastore.ERROR_OCCURRED_KEY";
|
||||
private final static ThreadLocal<Map<String, Object>> THREAD_LOCAL = new ThreadLocal<Map<String, Object>>() {
|
||||
@Override
|
||||
protected Map<String, Object> initialValue() {
|
||||
return new HashMap<String, Object>();
|
||||
}
|
||||
};
|
||||
|
||||
protected final DataStoreContext dataStoreContext;
|
||||
|
||||
protected final Logger logger = LoggerFactory.getLogger(this.getClass());
|
||||
|
||||
public MessageHandler(DataStoreContext dataStoreContext) {
|
||||
this.dataStoreContext = dataStoreContext;
|
||||
}
|
||||
|
||||
/**
|
||||
* ´¦ÀíÌØ¶¨µÄÏûÏ¢
|
||||
*
|
||||
* @param message
|
||||
*/
|
||||
public abstract void messageReceived(T message,String ip) throws Exception;
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public void messageReceived(IoSession session, MongoMessage message) throws Exception {
|
||||
System.out.println(session.getRemoteAddress());
|
||||
try {
|
||||
setSession(session);
|
||||
messageReceived((T) message,session.getRemoteAddress().toString());
|
||||
if (!isErrorOccured()) {
|
||||
this.clearLastError();
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
String hostAddress = "";
|
||||
if(session.getRemoteAddress() instanceof InetSocketAddress) {
|
||||
hostAddress = ((InetSocketAddress)session.getRemoteAddress()).getAddress().getHostAddress();
|
||||
}
|
||||
this.setLastError(buildExceptionMessage(e));
|
||||
logger.error("error processing host:" + hostAddress + ",message: " + message, e);
|
||||
} finally {
|
||||
clearErrorOccured();
|
||||
}
|
||||
}
|
||||
|
||||
private void clearErrorOccured() {
|
||||
THREAD_LOCAL.get().remove(ERROR_OCCURRED_KEY);
|
||||
}
|
||||
|
||||
private void setErrorOccured() {
|
||||
THREAD_LOCAL.get().put(ERROR_OCCURRED_KEY, true);
|
||||
}
|
||||
|
||||
private boolean isErrorOccured() {
|
||||
Boolean b = (Boolean) THREAD_LOCAL.get().get(ERROR_OCCURRED_KEY);
|
||||
return null == b ? false : b;
|
||||
}
|
||||
|
||||
protected IoSession getSession() {
|
||||
return (IoSession) THREAD_LOCAL.get().get(SESSION_KEY);
|
||||
}
|
||||
|
||||
private void setSession(IoSession session) {
|
||||
THREAD_LOCAL.get().put(SESSION_KEY, session);
|
||||
}
|
||||
|
||||
protected void setLastError(Object error) {
|
||||
if (null != error) {
|
||||
IoSession session = getSession();
|
||||
Object lastError = session.getAttribute(LAST_ERROR_KEY);
|
||||
if (null != lastError) {
|
||||
session.setAttribute(PREV_ERROR_KEY, lastError);
|
||||
}
|
||||
setErrorOccured();
|
||||
session.setAttribute(LAST_ERROR_KEY, String.valueOf(error));
|
||||
}
|
||||
}
|
||||
|
||||
private void clearLastError() {
|
||||
getSession().removeAttribute(LAST_ERROR_KEY);
|
||||
}
|
||||
|
||||
protected Object getLastError() {
|
||||
return getSession().getAttribute(LAST_ERROR_KEY);
|
||||
}
|
||||
|
||||
protected Object getPrevError() {
|
||||
return getSession().getAttribute(PREV_ERROR_KEY);
|
||||
}
|
||||
|
||||
protected void writeMessage(ReplyMessage message) {
|
||||
getSession().write(message);
|
||||
}
|
||||
|
||||
protected void writeAsReplyMessage(int responseTo, List<? extends BSONObject> docs) {
|
||||
writeMessage(prepareReplyMessage(responseTo, docs));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
protected ReplyMessage prepareReplyMessage(int responseTo, List<? extends BSONObject> docs) {
|
||||
ReplyMessage reply = new ReplyMessage();
|
||||
reply.setNumberReturned(null == docs ? 0 : docs.size());
|
||||
reply.setResponseFlag(8);
|
||||
reply.getMessageHeader().setResponseTo(responseTo);
|
||||
if (null != docs && docs.size() > 0) {
|
||||
reply.setDocuments((List<BSONObject>) docs);
|
||||
}
|
||||
return reply;
|
||||
}
|
||||
|
||||
protected void writeAsReplyMessage(int responseTo, BSONObject... docs) {
|
||||
writeMessage(prepareReplyMessage(responseTo, docs));
|
||||
}
|
||||
|
||||
protected ReplyMessage prepareReplyMessage(int responseTo, BSONObject... docs) {
|
||||
if (null != docs && docs.length > 0) {
|
||||
List<BSONObject> docList = new ArrayList<BSONObject>(docs.length);
|
||||
for (BSONObject doc : docs) {
|
||||
docList.add(doc);
|
||||
}
|
||||
return prepareReplyMessage(responseTo, docList);
|
||||
} else {
|
||||
return prepareReplyMessage(responseTo, (List<? extends BSONObject>) null);
|
||||
}
|
||||
}
|
||||
|
||||
protected void writeAsErrorMessage(int responseTo, Object errorMessage) {
|
||||
writeMessage(prepareErrorMessage(responseTo, errorMessage));
|
||||
}
|
||||
|
||||
protected ReplyMessage prepareErrorMessage(int responseTo, Object errorMessage) {
|
||||
ReplyMessage reply = new ReplyMessage();
|
||||
reply.setNumberReturned(1);
|
||||
reply.setResponseFlag(8);
|
||||
reply.getMessageHeader().setResponseTo(responseTo);
|
||||
BSONObject msg = new BasicBSONObject("$err", errorMessage == null ? null : errorMessage.toString());
|
||||
msg.put("err", errorMessage == null ? null : errorMessage.toString());
|
||||
msg.put("errmsg", errorMessage == null ? null : errorMessage.toString());
|
||||
msg.put("ok", 0.0);
|
||||
reply.setDocuments(Arrays.asList(msg));
|
||||
return reply;
|
||||
}
|
||||
|
||||
protected String buildExceptionMessage(Throwable e) {
|
||||
StringBuilder ret = new StringBuilder();
|
||||
ret.append(e.getClass().getName()).append(": ").append(e.getMessage());
|
||||
Throwable cause = e;
|
||||
while ((cause = cause.getCause()) != null) {
|
||||
ret.append(", caused by ").append(cause.getClass().getName()).append(": ").append(cause.getMessage());
|
||||
}
|
||||
return ret.toString();
|
||||
}
|
||||
|
||||
protected void assertNotReadOnly() throws DataStoreException {
|
||||
if (this.dataStoreContext.getDataStoreConfig().isReadOnly()) {
|
||||
throw new DataStoreException("read only server !");
|
||||
}
|
||||
}
|
||||
public abstract class MessageHandler<T extends MongoMessage>
|
||||
{
|
||||
private static final String LAST_ERROR_KEY = "com.goojia.datastore.LAST_ERROR_KEY";
|
||||
private static final String PREV_ERROR_KEY = "com.goojia.datastore.PREV_ERROR_KEY";
|
||||
private static final String SESSION_KEY = "com.goojia.datastore.SESSION_KEY";
|
||||
private static final String ERROR_OCCURRED_KEY = "com.goojia.datastore.ERROR_OCCURRED_KEY";
|
||||
private static final ThreadLocal<Map<String, Object>> THREAD_LOCAL;
|
||||
protected final DataStoreContext dataStoreContext;
|
||||
protected final Logger logger = LoggerFactory.getLogger(this.getClass());
|
||||
|
||||
public MessageHandler(final DataStoreContext dataStoreContext) {
|
||||
this.dataStoreContext = dataStoreContext;
|
||||
}
|
||||
|
||||
public abstract void messageReceived(T message) throws Exception;
|
||||
|
||||
public void messageReceived(final IoSession session, final MongoMessage message) throws Exception {
|
||||
try {
|
||||
this.setSession(session);
|
||||
this.messageReceived((T)message);
|
||||
if (!this.isErrorOccured()) {
|
||||
this.clearLastError();
|
||||
}
|
||||
}
|
||||
catch (Throwable e) {
|
||||
String hostAddress = "";
|
||||
if (session.getRemoteAddress() instanceof InetSocketAddress) {
|
||||
hostAddress = ((InetSocketAddress)session.getRemoteAddress()).getAddress().getHostAddress();
|
||||
}
|
||||
this.setLastError(this.buildExceptionMessage(e));
|
||||
this.logger.error("error processing host:" + hostAddress + ",message: " + message, e);
|
||||
}
|
||||
finally {
|
||||
this.clearErrorOccured();
|
||||
}
|
||||
}
|
||||
|
||||
private void clearErrorOccured() {
|
||||
MessageHandler.THREAD_LOCAL.get().remove("com.goojia.datastore.ERROR_OCCURRED_KEY");
|
||||
}
|
||||
|
||||
private void setErrorOccured() {
|
||||
MessageHandler.THREAD_LOCAL.get().put("com.goojia.datastore.ERROR_OCCURRED_KEY", true);
|
||||
}
|
||||
|
||||
private boolean isErrorOccured() {
|
||||
final Boolean b = (Boolean) MessageHandler.THREAD_LOCAL.get().get("com.goojia.datastore.ERROR_OCCURRED_KEY");
|
||||
return null != b && b;
|
||||
}
|
||||
|
||||
protected IoSession getSession() {
|
||||
return (IoSession)MessageHandler.THREAD_LOCAL.get().get("com.goojia.datastore.SESSION_KEY");
|
||||
}
|
||||
|
||||
private void setSession(final IoSession session) {
|
||||
MessageHandler.THREAD_LOCAL.get().put("com.goojia.datastore.SESSION_KEY", session);
|
||||
}
|
||||
|
||||
protected void setLastError(final Object error) {
|
||||
if (null != error) {
|
||||
final IoSession session = this.getSession();
|
||||
final Object lastError = session.getAttribute((Object)"com.goojia.datastore.LAST_ERROR_KEY");
|
||||
if (null != lastError) {
|
||||
session.setAttribute((Object)"com.goojia.datastore.PREV_ERROR_KEY", lastError);
|
||||
}
|
||||
this.setErrorOccured();
|
||||
session.setAttribute((Object)"com.goojia.datastore.LAST_ERROR_KEY", (Object)String.valueOf(error));
|
||||
}
|
||||
}
|
||||
|
||||
private void clearLastError() {
|
||||
this.getSession().removeAttribute((Object)"com.goojia.datastore.LAST_ERROR_KEY");
|
||||
}
|
||||
|
||||
protected Object getLastError() {
|
||||
return this.getSession().getAttribute((Object)"com.goojia.datastore.LAST_ERROR_KEY");
|
||||
}
|
||||
|
||||
protected Object getPrevError() {
|
||||
return this.getSession().getAttribute((Object)"com.goojia.datastore.PREV_ERROR_KEY");
|
||||
}
|
||||
|
||||
protected void writeMessage(final ReplyMessage message) {
|
||||
this.getSession().write((Object)message);
|
||||
}
|
||||
|
||||
protected void writeAsReplyMessage(final int responseTo, final List<? extends BSONObject> docs) {
|
||||
this.writeMessage(this.prepareReplyMessage(responseTo, docs));
|
||||
}
|
||||
|
||||
protected ReplyMessage prepareReplyMessage(final int responseTo, final List<? extends BSONObject> docs) {
|
||||
final ReplyMessage reply = new ReplyMessage();
|
||||
reply.setNumberReturned((null == docs) ? 0 : docs.size());
|
||||
reply.setResponseFlag(8);
|
||||
reply.getMessageHeader().setResponseTo(responseTo);
|
||||
if (null != docs && docs.size() > 0) {
|
||||
reply.setDocuments((List<BSONObject>)docs);
|
||||
}
|
||||
return reply;
|
||||
}
|
||||
|
||||
protected void writeAsReplyMessage(final int responseTo, final BSONObject... docs) {
|
||||
this.writeMessage(this.prepareReplyMessage(responseTo, docs));
|
||||
}
|
||||
|
||||
protected ReplyMessage prepareReplyMessage(final int responseTo, final BSONObject... docs) {
|
||||
if (null != docs && docs.length > 0) {
|
||||
final List<BSONObject> docList = new ArrayList<BSONObject>(docs.length);
|
||||
for (final BSONObject doc : docs) {
|
||||
docList.add(doc);
|
||||
}
|
||||
return this.prepareReplyMessage(responseTo, docList);
|
||||
}
|
||||
return this.prepareReplyMessage(responseTo, (List<? extends BSONObject>)null);
|
||||
}
|
||||
|
||||
protected void writeAsErrorMessage(final int responseTo, final Object errorMessage) {
|
||||
this.writeMessage(this.prepareErrorMessage(responseTo, errorMessage));
|
||||
}
|
||||
|
||||
protected ReplyMessage prepareErrorMessage(final int responseTo, final Object errorMessage) {
|
||||
final ReplyMessage reply = new ReplyMessage();
|
||||
reply.setNumberReturned(1);
|
||||
reply.setResponseFlag(8);
|
||||
reply.getMessageHeader().setResponseTo(responseTo);
|
||||
final BSONObject msg = (BSONObject)new BasicBSONObject("$err", (Object)((errorMessage == null) ? null : errorMessage.toString()));
|
||||
msg.put("err", (Object)((errorMessage == null) ? null : errorMessage.toString()));
|
||||
msg.put("errmsg", (Object)((errorMessage == null) ? null : errorMessage.toString()));
|
||||
msg.put("ok", (Object)0.0);
|
||||
reply.setDocuments(Arrays.<BSONObject>asList(msg));
|
||||
return reply;
|
||||
}
|
||||
|
||||
protected String buildExceptionMessage(final Throwable e) {
|
||||
final StringBuilder ret = new StringBuilder();
|
||||
ret.append(e.getClass().getName()).append(": ").append(e.getMessage());
|
||||
Throwable cause = e;
|
||||
while ((cause = cause.getCause()) != null) {
|
||||
ret.append(", caused by ").append(cause.getClass().getName()).append(": ").append(cause.getMessage());
|
||||
}
|
||||
return ret.toString();
|
||||
}
|
||||
|
||||
protected void assertNotReadOnly() throws DataStoreException {
|
||||
if (this.dataStoreContext.getDataStoreConfig().isReadOnly()) {
|
||||
throw new DataStoreException("read only server !");
|
||||
}
|
||||
}
|
||||
|
||||
protected boolean useCursor() {
|
||||
return (Boolean) this.getSession().getAttribute((Object)"useCursor", false);
|
||||
}
|
||||
|
||||
static {
|
||||
THREAD_LOCAL = new ThreadLocal<Map<String, Object>>() {
|
||||
@Override
|
||||
protected Map<String, Object> initialValue() {
|
||||
return new HashMap<String, Object>();
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
+491
-480
@@ -1,490 +1,501 @@
|
||||
package com.fangjia.datastore.handler;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import com.fangjia.datastore.mongo.protocol.MongoMessage;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import com.mongodb.BasicDBList;
|
||||
import org.apache.mina.core.session.IoSession;
|
||||
import org.bson.BSONObject;
|
||||
import org.bson.BasicBSONObject;
|
||||
|
||||
import com.fangjia.datastore.ConfigurationException;
|
||||
import com.fangjia.datastore.DataStore;
|
||||
import com.fangjia.datastore.DataStoreContext;
|
||||
import com.fangjia.datastore.DataStoreException;
|
||||
import com.fangjia.datastore.QueryResponse;
|
||||
import com.fangjia.datastore.config.Field;
|
||||
import com.fangjia.datastore.config.StoreConfig;
|
||||
import com.fangjia.datastore.mongo.protocol.QueryMessage;
|
||||
import com.fangjia.datastore.config.Field;
|
||||
import com.fangjia.datastore.mongo.protocol.ReplyMessage;
|
||||
import com.fangjia.datastore.QueryResponse;
|
||||
import com.fangjia.datastore.Cursor;
|
||||
import java.util.Collection;
|
||||
import com.fangjia.datastore.solr.Index;
|
||||
import com.fangjia.datastore.utils.MongoUtils;
|
||||
import java.util.Iterator;
|
||||
import java.util.Set;
|
||||
import com.mongodb.BasicDBObject;
|
||||
import com.mongodb.DBObject;
|
||||
import java.util.ArrayList;
|
||||
import com.fangjia.datastore.ConfigurationException;
|
||||
import com.mongodb.BasicDBList;
|
||||
import java.util.List;
|
||||
import com.fangjia.datastore.DataStore;
|
||||
import org.bson.types.BasicBSONList;
|
||||
import org.bson.BasicBSONObject;
|
||||
import org.bson.BSONObject;
|
||||
import com.fangjia.datastore.DataStoreException;
|
||||
import org.apache.mina.core.session.IoSession;
|
||||
import java.net.InetSocketAddress;
|
||||
import com.fangjia.datastore.utils.MongoUtils;
|
||||
import com.fangjia.datastore.DataStoreContext;
|
||||
import com.fangjia.datastore.mongo.protocol.QueryMessage;
|
||||
|
||||
public class QueryMessageHandler extends MessageHandler<QueryMessage> {
|
||||
|
||||
public QueryMessageHandler(DataStoreContext dataStoreContext) {
|
||||
super(dataStoreContext);
|
||||
}
|
||||
|
||||
public void messageReceived(QueryMessage message,String ip) throws Exception {
|
||||
Object error = null;
|
||||
try {
|
||||
String[] nameSplit = MongoUtils.parseFullCollectionName(message.getFullCollectionName());
|
||||
if (null == nameSplit) {
|
||||
error = "invalid full collection name: " + message.getFullCollectionName();
|
||||
} else {
|
||||
doMessageReceived(message, nameSplit[0], nameSplit[1]);
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
IoSession session = getSession();
|
||||
String hostAddress = "";
|
||||
if(session != null && session.getRemoteAddress() instanceof InetSocketAddress) {
|
||||
hostAddress = ((InetSocketAddress)session.getRemoteAddress()).getAddress().getHostAddress();
|
||||
}
|
||||
|
||||
error = buildExceptionMessage(e);
|
||||
this.writeAsErrorMessage(message.getMessageHeader().getRequestID(), error);
|
||||
if (logger.isErrorEnabled()) {
|
||||
logger.error("error process host:" + hostAddress + ",message: " + message, e);
|
||||
}
|
||||
} finally {
|
||||
this.setLastError(error);
|
||||
}
|
||||
}
|
||||
|
||||
private void doMessageReceived(QueryMessage message, String dbName, String collectionName) throws DataStoreException {
|
||||
if (collectionName.equals("$cmd")) {
|
||||
onCommand(message, dbName);
|
||||
return;
|
||||
}
|
||||
if (collectionName.equals("system.namespaces")) {
|
||||
showCollections(message, dbName, collectionName);
|
||||
} else if (collectionName.equals("system.indexes")) {
|
||||
getIndexes(message, dbName, collectionName);
|
||||
} else {
|
||||
find(message, dbName, collectionName);
|
||||
}
|
||||
}
|
||||
|
||||
private void onCommand(QueryMessage message, String dbName) throws DataStoreException {
|
||||
if (dbName.equals("admin")) {
|
||||
onAdminCommand(message);
|
||||
return;
|
||||
}
|
||||
BSONObject query = message.getQuery();
|
||||
if (query.containsField("getlasterror")) {
|
||||
getLastError(message, dbName);
|
||||
} else if (query.containsField("getpreverror")) {
|
||||
getPrevError(message, dbName);
|
||||
} else if (query.containsField("count")) {
|
||||
count(message, dbName);
|
||||
} else if (query.containsField("getSchema")) {
|
||||
getSchema(message, dbName);
|
||||
} else if (query.containsField("reIndex")) {
|
||||
reIndex(message, dbName);
|
||||
} else if (query.containsField("reload")) {
|
||||
reload(message, dbName);
|
||||
} else if(query.containsField("dbStats")){
|
||||
dbStats(message, dbName);
|
||||
} else if(query.containsField("listCollections")){
|
||||
listCollections(message, dbName);
|
||||
} else if (query.containsField("isMaster") || query.containsField("ismaster")) {
|
||||
isMaster(message);
|
||||
} else if(query.containsField("buildinfo") || query.containsField("buildInfo")){
|
||||
buildinfo(message);
|
||||
} else if(query.containsField("serverStatus")){
|
||||
serverStatus(message);
|
||||
} else if(query.containsField("connectionStatus")){
|
||||
connectionStatus(message);
|
||||
} else if (query.containsField("find")) {
|
||||
find(message, dbName, query.get("find").toString() );
|
||||
} else {
|
||||
delegateCommand(message, dbName);
|
||||
}
|
||||
}
|
||||
|
||||
private void onAdminCommand(QueryMessage message) throws DataStoreException {
|
||||
BSONObject query = message.getQuery();
|
||||
if (query.containsField("whatsmyuri")) {
|
||||
whatsMyUri(message);
|
||||
} else if (query.containsField("listDatabases")) {
|
||||
listDatabases(message);
|
||||
} else if (query.containsField("nonce")) {
|
||||
nonce(message);
|
||||
} else if (query.containsField("isMaster") || query.containsField("ismaster")) {
|
||||
isMaster(message);
|
||||
} else if(query.containsField("buildinfo") || query.containsField("buildInfo")){
|
||||
buildinfo(message);
|
||||
} else if (query.containsField("replSetGetStatus")) {
|
||||
replSetGetStatus(message);
|
||||
} else if(query.containsField("getlasterror")){
|
||||
getLastError(message, "admin");
|
||||
} else if(query.containsField("serverStatus")){
|
||||
serverStatus(message);
|
||||
} else if(query.containsField("connectionStatus")){
|
||||
connectionStatus(message);
|
||||
} else if (query.containsField("getLog")) {
|
||||
getLog(message);
|
||||
} else {
|
||||
delegateAdminCommand(message);
|
||||
}
|
||||
}
|
||||
|
||||
private void replSetGetStatus(QueryMessage message) {
|
||||
BSONObject doc = new BasicBSONObject();
|
||||
doc.put("ok", 0);
|
||||
doc.put("errmsg", "not running with --replSet");
|
||||
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void delegateCommand(QueryMessage message, String dbName) throws DataStoreException {
|
||||
throw new DataStoreException("unsupported command:" + message.getQuery());
|
||||
}
|
||||
|
||||
private void getLog(QueryMessage message){
|
||||
|
||||
BasicDBList names = new BasicDBList();
|
||||
names.add("global");
|
||||
names.add("startupWarnings");
|
||||
|
||||
BSONObject doc = new BasicBSONObject();
|
||||
doc.put("ok", 1);
|
||||
doc.put("names", names);
|
||||
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
|
||||
}
|
||||
|
||||
private void dbStats(QueryMessage message, String dbName) throws DataStoreException {
|
||||
|
||||
BSONObject doc = new BasicBSONObject();
|
||||
doc.put("db", dbName);
|
||||
doc.put("collections", this.dataStoreContext.getDataStoreConfig().getStoreNames());
|
||||
doc.put("ok", 1.0);
|
||||
InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress();
|
||||
doc.put("primary", localAddress.getAddress().getHostAddress());
|
||||
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
|
||||
|
||||
private void delegateAdminCommand(QueryMessage message) throws DataStoreException {
|
||||
throw new DataStoreException("unsupported command:" + message.getQuery());
|
||||
}
|
||||
|
||||
private void reload(QueryMessage message, String dbName) throws DataStoreException {
|
||||
assertNotReadOnly();
|
||||
try {
|
||||
this.dataStoreContext.reload();
|
||||
this.dataStoreContext.getDataStore(dbName + "." + message.getQuery().get("reload")).reload();
|
||||
BSONObject doc = new BasicBSONObject();
|
||||
doc.put("ok", 1.0);
|
||||
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
} catch (ConfigurationException e) {
|
||||
throw new DataStoreException("error reload config: ", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* show collections
|
||||
*
|
||||
* @param message
|
||||
*/
|
||||
private void showCollections(QueryMessage message, String dbName, String collectionName) throws DataStoreException {
|
||||
Set<String> storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames();
|
||||
List<DBObject> docList = new ArrayList<DBObject>(storeNames.size());
|
||||
for (String sn : storeNames) {
|
||||
if (sn.startsWith(dbName + ".")) {
|
||||
docList.add(new BasicDBObject("name", sn));
|
||||
}
|
||||
}
|
||||
writeAsReplyMessage(message.getMessageHeader().getRequestID(), docList);
|
||||
}
|
||||
|
||||
/**
|
||||
* db.foo.getIndexes()
|
||||
*
|
||||
* @param message
|
||||
* @param dbName
|
||||
* @param collectionName
|
||||
* @throwstion
|
||||
*/
|
||||
private void getIndexes(QueryMessage message, String dbName, String collectionName) throws DataStoreException {
|
||||
String ns = (String) message.getQuery().get("ns");
|
||||
Collection<Index> indexes = this.dataStoreContext.getDataStore(ns).getIndexes();
|
||||
List<BSONObject> docList = new ArrayList<BSONObject>(indexes.size());
|
||||
for (Index i : indexes) {
|
||||
docList.add(i.toBSONObject());
|
||||
}
|
||||
writeAsReplyMessage(message.getMessageHeader().getRequestID(), docList);
|
||||
}
|
||||
|
||||
/**
|
||||
* db.foo.find()
|
||||
*
|
||||
* @param message
|
||||
* @param dbName
|
||||
* @param collectionName
|
||||
* @throws DataStoreException
|
||||
*/
|
||||
private void find(QueryMessage message, String dbName, String collectionName) throws DataStoreException {
|
||||
|
||||
String fullCollectionName;
|
||||
if(message.getFullCollectionName().contains("$cmd")){
|
||||
fullCollectionName = dbName + "." + collectionName;
|
||||
}else{
|
||||
fullCollectionName = message.getFullCollectionName();
|
||||
}
|
||||
DataStore dataStore = this.dataStoreContext.getDataStore(fullCollectionName);
|
||||
QueryResponse response = dataStore.find(message.getRealQuery(), message.getReturnFieldSelector(), message
|
||||
.getOrderByQuery(), message.getFacetQuery(), message.getNumberToSkip(), message.getNumberToReturn() - 1, message
|
||||
.isBulkQuery(), message.isFromMaster());
|
||||
ReplyMessage reply = prepareReplyMessage(message.getMessageHeader().getRequestID(), response.getDocList());
|
||||
reply.setStartingFrom(message.getNumberToSkip());
|
||||
// if (response.hasMore()) {
|
||||
// Cursor cursor = new Cursor((int) message.getNumberToSkip(), (int)
|
||||
// response.getNumReturned(), message,
|
||||
// response.getNumFound());
|
||||
// this.dataStoreContext.addCursor(cursor);
|
||||
// reply.setCursorID(cursor.getId());
|
||||
// }
|
||||
this.writeMessage(reply);
|
||||
}
|
||||
|
||||
/**
|
||||
* db.getLastError()
|
||||
*
|
||||
* @param message
|
||||
* @param dbName
|
||||
*/
|
||||
private void getLastError(QueryMessage message, String dbName) {
|
||||
Object lastError = this.getLastError();
|
||||
BSONObject doc = new BasicBSONObject();
|
||||
doc.put("ok", 1.0);
|
||||
doc.put("n", 1);
|
||||
doc.put("$err", lastError == null ? null : lastError.toString());
|
||||
doc.put("err", lastError == null ? null : lastError.toString());
|
||||
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
/**
|
||||
* db.getPrevError()
|
||||
*
|
||||
* @param message
|
||||
* @param dbName
|
||||
*/
|
||||
private void getPrevError(QueryMessage message, String dbName) {
|
||||
Object prevError = this.getPrevError();
|
||||
BSONObject doc = new BasicBSONObject();
|
||||
doc.put("ok", 1.0);
|
||||
doc.put("n", 0);
|
||||
doc.put("$err", prevError == null ? null : prevError.toString());
|
||||
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
/**
|
||||
* db.foo.count()
|
||||
*
|
||||
* @param message
|
||||
* @param dbName
|
||||
* @throws DataStoreException
|
||||
*/
|
||||
private void count(QueryMessage message, String dbName) throws DataStoreException {
|
||||
BSONObject query = message.getQuery();
|
||||
int count = this.dataStoreContext.getDataStore(dbName + "." + query.get("count")).count(message.getRealQuery(),
|
||||
message.isFromMaster());
|
||||
BSONObject doc = new BasicBSONObject();
|
||||
doc.put("ok", 1.0);
|
||||
doc.put("n", (double) count);
|
||||
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
/**
|
||||
* db.foo.rumCommnad('getSchema')
|
||||
*
|
||||
* @param message
|
||||
* @param dbName
|
||||
*/
|
||||
private void getSchema(QueryMessage message, String dbName) throws DataStoreException {
|
||||
BSONObject query = message.getQuery();
|
||||
String collectionName = (String) query.get("getSchema");
|
||||
StoreConfig storeConfig = this.dataStoreContext.getDataStore(dbName + "." + collectionName).getStoreConfig();
|
||||
Collection<Field> fields = storeConfig.getFields();
|
||||
List<DBObject> docList = new ArrayList<DBObject>(fields.size());
|
||||
for (Field field : fields) {
|
||||
BasicDBObject f = new BasicDBObject();
|
||||
f.put("name", field.getName());
|
||||
f.put("type", field.getType().getType());
|
||||
docList.add(f);
|
||||
}
|
||||
BSONObject doc = new BasicBSONObject();
|
||||
doc.put("ok", 1.0);
|
||||
doc.put("schema", docList);
|
||||
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* 不带参数重建:db.foo.reIndex() 或 db.foo.runCommand('reIndex')
|
||||
* </P>
|
||||
*
|
||||
* <p>
|
||||
* 带参数重建:db.foo.runCommand('reIndex',{query:{cat:'sell',city:'上海'}})
|
||||
* <p>
|
||||
*
|
||||
* @param message
|
||||
* @param dbName
|
||||
* @throws Exception
|
||||
*/
|
||||
private void reIndex(QueryMessage message, String dbName) throws DataStoreException {
|
||||
assertNotReadOnly();
|
||||
BSONObject query = message.getQuery();
|
||||
String collectionName = (String) query.get("reIndex");
|
||||
DataStore dataStore = this.dataStoreContext.getDataStore(dbName + "." + collectionName);
|
||||
// 新增查询条件 add by yukai 2011/07/14
|
||||
BSONObject doc = new BasicBSONObject();
|
||||
Object realQuery = query.get("query");
|
||||
if (null == realQuery)
|
||||
doc.put("version", dataStore.reIndex(null));
|
||||
else if (realQuery instanceof BSONObject)
|
||||
dataStore.reIndex((BSONObject) realQuery);
|
||||
else
|
||||
throw new DataStoreException("invalid re-index query: " + realQuery);
|
||||
doc.put("ok", 1.0);
|
||||
doc.put("n", 0D);
|
||||
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
/**
|
||||
* db.runCommand('whatsmyuri')
|
||||
*
|
||||
* @param message
|
||||
*/
|
||||
private void whatsMyUri(QueryMessage message) {
|
||||
BSONObject doc = new BasicBSONObject();
|
||||
InetSocketAddress address = (InetSocketAddress) getSession().getRemoteAddress();
|
||||
doc.put("you", address.getAddress().getHostAddress() + ":" + address.getPort());
|
||||
doc.put("ok", 1.0);
|
||||
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
/**
|
||||
* show dbs
|
||||
*
|
||||
* @param message
|
||||
*/
|
||||
private void listDatabases(QueryMessage message) {
|
||||
BSONObject doc = new BasicBSONObject();
|
||||
doc.put("ok", 1.0);
|
||||
Set<String> storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames();
|
||||
Set<String> dbNames = new HashSet<String>();
|
||||
for (String sn : storeNames) {
|
||||
int i = sn.indexOf('.');
|
||||
if (i > 0) {
|
||||
String name = sn.substring(0, i);
|
||||
dbNames.add(name);
|
||||
}
|
||||
}
|
||||
List<DBObject> docs = new ArrayList<DBObject>(dbNames.size());
|
||||
for (String dn : dbNames) {
|
||||
docs.add(new BasicDBObject("name", dn));
|
||||
}
|
||||
doc.put("databases", docs);
|
||||
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
/**
|
||||
* show dbs
|
||||
*
|
||||
* @param message
|
||||
*/
|
||||
private void listCollections(QueryMessage message, String dbName) {
|
||||
System.out.println(message.getMessageHeader().getRequestID());
|
||||
BSONObject doc = new BasicBSONObject();
|
||||
doc.put("ok", 1.0);
|
||||
BSONObject cursor = new BasicDBObject();
|
||||
doc.put("cursor", cursor);
|
||||
|
||||
cursor.put("id", (long)message.getMessageHeader().getRequestID());
|
||||
cursor.put("ns", dbName + ".$cmd.listCollections");
|
||||
BasicDBList firstBatch = new BasicDBList();
|
||||
cursor.put("firstBatch", firstBatch);
|
||||
Set<String> storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames();
|
||||
for (String sn : storeNames) {
|
||||
if (sn.startsWith(dbName + ".")) {
|
||||
BSONObject collection = new BasicDBObject();
|
||||
collection.put("name", sn.replace(dbName + ".", ""));
|
||||
collection.put("options", new BasicDBObject());
|
||||
firstBatch.add(collection);
|
||||
}
|
||||
}
|
||||
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* db.runCommand('nonce')
|
||||
*
|
||||
* @param message
|
||||
*/
|
||||
private void nonce(QueryMessage message) {
|
||||
BSONObject doc = new BasicBSONObject();
|
||||
doc.put("ok", 1.0);
|
||||
doc.put("errmsg", "no such cmd");
|
||||
doc.put("bad cmd", new BasicBSONObject("nonce", true));
|
||||
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void isMaster(QueryMessage message) {
|
||||
BSONObject doc = new BasicBSONObject();
|
||||
doc.put("ok", 1.0);
|
||||
doc.put("ismaster", true);
|
||||
doc.put("secondary", false);
|
||||
InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress();
|
||||
doc.put("primary", localAddress.getAddress().getHostAddress());
|
||||
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void buildinfo(QueryMessage message) {
|
||||
BasicDBList versionArray = new BasicDBList();
|
||||
versionArray.add(3);
|
||||
versionArray.add(2);
|
||||
versionArray.add(9);
|
||||
versionArray.add(0);
|
||||
BSONObject doc = new BasicBSONObject();
|
||||
doc.put("ok", 1.0);
|
||||
doc.put("bits", 64.0);
|
||||
doc.put("debug", false);
|
||||
doc.put("version" ,"3.2.9");
|
||||
doc.put("versionArray" ,versionArray);
|
||||
doc.put("javascriptEngine", "V8");
|
||||
InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress();
|
||||
doc.put("primary", localAddress.getAddress().getHostAddress());
|
||||
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void serverStatus(QueryMessage message){
|
||||
BSONObject doc = new BasicBSONObject();
|
||||
doc.put("ok", 1.0);
|
||||
InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress();
|
||||
doc.put("primary", localAddress.getAddress().getHostAddress());
|
||||
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void connectionStatus(QueryMessage message){
|
||||
BSONObject doc = new BasicBSONObject();
|
||||
doc.put("ok", 1.0);
|
||||
InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress();
|
||||
doc.put("primary", localAddress.getAddress().getHostAddress());
|
||||
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
|
||||
public class QueryMessageHandler extends MessageHandler<QueryMessage>
|
||||
{
|
||||
public QueryMessageHandler(final DataStoreContext dataStoreContext) {
|
||||
super(dataStoreContext);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void messageReceived(final QueryMessage message) throws Exception {
|
||||
Object error = null;
|
||||
try {
|
||||
final String[] nameSplit = MongoUtils.parseFullCollectionName(message.getFullCollectionName());
|
||||
if (null == nameSplit) {
|
||||
error = "invalid full collection name: " + message.getFullCollectionName();
|
||||
}
|
||||
else {
|
||||
this.doMessageReceived(message, nameSplit[0], nameSplit[1]);
|
||||
}
|
||||
}
|
||||
catch (Throwable e) {
|
||||
final IoSession session = this.getSession();
|
||||
String hostAddress = "";
|
||||
if (session != null && session.getRemoteAddress() instanceof InetSocketAddress) {
|
||||
hostAddress = ((InetSocketAddress)session.getRemoteAddress()).getAddress().getHostAddress();
|
||||
}
|
||||
error = this.buildExceptionMessage(e);
|
||||
this.writeAsErrorMessage(message.getMessageHeader().getRequestID(), error);
|
||||
if (this.logger.isErrorEnabled()) {
|
||||
this.logger.error("error process host:" + hostAddress + ",message: " + message, e);
|
||||
}
|
||||
}
|
||||
finally {
|
||||
this.setLastError(error);
|
||||
}
|
||||
}
|
||||
|
||||
private void doMessageReceived(final QueryMessage message, final String dbName, final String collectionName) throws DataStoreException {
|
||||
if (collectionName.equals("$cmd")) {
|
||||
this.onCommand(message, dbName);
|
||||
return;
|
||||
}
|
||||
if (collectionName.equals("system.namespaces")) {
|
||||
this.showCollections(message, dbName, collectionName);
|
||||
}
|
||||
else if (collectionName.equals("system.indexes")) {
|
||||
this.getIndexes(message, dbName, collectionName);
|
||||
}
|
||||
else {
|
||||
this.find(message, dbName, collectionName);
|
||||
}
|
||||
}
|
||||
|
||||
private void onCommand(final QueryMessage message, final String dbName) throws DataStoreException {
|
||||
if (dbName.equals("admin")) {
|
||||
this.onAdminCommand(message);
|
||||
return;
|
||||
}
|
||||
final BSONObject query = message.getQuery();
|
||||
if (query.containsField("getlasterror")) {
|
||||
this.getLastError(message, dbName);
|
||||
}
|
||||
else if (query.containsField("getpreverror")) {
|
||||
this.getPrevError(message, dbName);
|
||||
}
|
||||
else if (query.containsField("count")) {
|
||||
this.count(message, dbName);
|
||||
}
|
||||
else if (query.containsField("getSchema")) {
|
||||
this.getSchema(message, dbName);
|
||||
}
|
||||
else if (query.containsField("reIndex")) {
|
||||
this.reIndex(message, dbName);
|
||||
}
|
||||
else if (query.containsField("reload")) {
|
||||
this.reload(message, dbName);
|
||||
}
|
||||
else if (query.containsField("dbStats")) {
|
||||
this.dbStats(message, dbName);
|
||||
}
|
||||
else if (query.containsField("listCollections")) {
|
||||
this.listCollections(message, dbName);
|
||||
}
|
||||
else if (query.containsField("isMaster") || query.containsField("ismaster")) {
|
||||
this.isMaster(message);
|
||||
}
|
||||
else if (query.containsField("buildinfo") || query.containsField("buildInfo")) {
|
||||
this.buildinfo(message);
|
||||
}
|
||||
else if (query.containsField("serverStatus")) {
|
||||
this.serverStatus(message);
|
||||
}
|
||||
else if (query.containsField("connectionStatus")) {
|
||||
this.connectionStatus(message);
|
||||
}
|
||||
else if (query.containsField("update")) {
|
||||
this.update(message, dbName);
|
||||
}
|
||||
else if (query.containsField("insert")) {
|
||||
this.insert(message, dbName);
|
||||
}
|
||||
else if (query.containsField("ping")) {
|
||||
this.ping(message);
|
||||
}
|
||||
else if (query.containsField("find")) {
|
||||
this.find(message, dbName, query.get("find").toString());
|
||||
}
|
||||
else {
|
||||
this.delegateCommand(message, dbName);
|
||||
}
|
||||
}
|
||||
|
||||
private void onAdminCommand(final QueryMessage message) throws DataStoreException {
|
||||
final BSONObject query = message.getQuery();
|
||||
if (query.containsField("whatsmyuri")) {
|
||||
this.whatsMyUri(message);
|
||||
}
|
||||
else if (query.containsField("listDatabases")) {
|
||||
this.listDatabases(message);
|
||||
}
|
||||
else if (query.containsField("nonce")) {
|
||||
this.nonce(message);
|
||||
}
|
||||
else if (query.containsField("isMaster") || query.containsField("ismaster")) {
|
||||
this.isMaster(message);
|
||||
}
|
||||
else if (query.containsField("buildinfo") || query.containsField("buildInfo")) {
|
||||
this.buildinfo(message);
|
||||
}
|
||||
else if (query.containsField("replSetGetStatus")) {
|
||||
this.replSetGetStatus(message);
|
||||
}
|
||||
else if (query.containsField("getlasterror")) {
|
||||
this.getLastError(message, "admin");
|
||||
}
|
||||
else if (query.containsField("serverStatus")) {
|
||||
this.serverStatus(message);
|
||||
}
|
||||
else if (query.containsField("connectionStatus")) {
|
||||
this.connectionStatus(message);
|
||||
}
|
||||
else if (query.containsField("collstats")) {
|
||||
this.collstats(message);
|
||||
}
|
||||
else if (query.containsField("getLog")) {
|
||||
this.getLog(message);
|
||||
}
|
||||
else if (query.containsField("ping")) {
|
||||
this.ping(message);
|
||||
}
|
||||
else {
|
||||
this.delegateAdminCommand(message);
|
||||
}
|
||||
}
|
||||
|
||||
private void replSetGetStatus(final QueryMessage message) {
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
doc.put("ok", (Object)0);
|
||||
doc.put("errmsg", (Object)"not running with --replSet");
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void delegateCommand(final QueryMessage message, final String dbName) throws DataStoreException {
|
||||
throw new DataStoreException("unsupported command:" + message.getQuery());
|
||||
}
|
||||
|
||||
private void collstats(final QueryMessage message) {
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
doc.put("ok", (Object)1);
|
||||
doc.put("ns", (Object)message.getFullCollectionName());
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void ping(final QueryMessage message) throws DataStoreException {
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
doc.put("ok", (Object)1);
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void update(final QueryMessage message, final String dbName) throws DataStoreException {
|
||||
final DataStore dataStore = this.dataStoreContext.getDataStore(dbName + "." + message.getQuery().get("update"));
|
||||
final BasicBSONList updates = (BasicBSONList)message.getQuery().get("updates");
|
||||
final BSONObject selector = (BSONObject)((BasicBSONObject)updates.get(0)).get("q");
|
||||
final BSONObject update = (BSONObject)((BasicBSONObject)updates.get(0)).get("u");
|
||||
final int count = dataStore.update(selector, update);
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
doc.put("nModified", (Object)count);
|
||||
doc.put("n", (Object)count);
|
||||
doc.put("ok", (Object)1);
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void insert(final QueryMessage message, final String dbName) throws DataStoreException {
|
||||
final DataStore dataStore = this.dataStoreContext.getDataStore(dbName + "." + message.getQuery().get("insert"));
|
||||
final List documents = (List)message.getQuery().get("documents");
|
||||
dataStore.save(documents);
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
doc.put("n", (Object)documents.size());
|
||||
doc.put("ok", (Object)1);
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void getLog(final QueryMessage message) {
|
||||
final BasicDBList names = new BasicDBList();
|
||||
names.add((Object)"global");
|
||||
names.add((Object)"startupWarnings");
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
doc.put("ok", (Object)1);
|
||||
doc.put("names", (Object)names);
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void dbStats(final QueryMessage message, final String dbName) throws DataStoreException {
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
doc.put("db", (Object)dbName);
|
||||
doc.put("collections", (Object)this.dataStoreContext.getDataStoreConfig().getStoreNames());
|
||||
doc.put("ok", (Object)1.0);
|
||||
final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress();
|
||||
doc.put("primary", (Object)localAddress.getAddress().getHostAddress());
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void delegateAdminCommand(final QueryMessage message) throws DataStoreException {
|
||||
throw new DataStoreException("unsupported command:" + message.getQuery());
|
||||
}
|
||||
|
||||
private void reload(final QueryMessage message, final String dbName) throws DataStoreException {
|
||||
this.assertNotReadOnly();
|
||||
try {
|
||||
this.dataStoreContext.reload();
|
||||
this.dataStoreContext.getDataStore(dbName + "." + message.getQuery().get("reload")).reload();
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
doc.put("ok", (Object)1.0);
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
catch (ConfigurationException e) {
|
||||
throw new DataStoreException("error reload config: ", e);
|
||||
}
|
||||
}
|
||||
|
||||
private void showCollections(final QueryMessage message, final String dbName, final String collectionName) throws DataStoreException {
|
||||
final Set<String> storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames();
|
||||
final List<DBObject> docList = new ArrayList<DBObject>(storeNames.size());
|
||||
for (final String sn : storeNames) {
|
||||
if (sn.startsWith(dbName + ".")) {
|
||||
docList.add((DBObject)new BasicDBObject("name", (Object)sn));
|
||||
}
|
||||
}
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), (List<? extends BSONObject>)docList);
|
||||
}
|
||||
|
||||
private void getIndexes(final QueryMessage message, final String dbName, final String collectionName) throws DataStoreException {
|
||||
final String ns = (String)message.getQuery().get("ns");
|
||||
final Collection<Index> indexes = this.dataStoreContext.getDataStore(ns).getIndexes();
|
||||
final List<BSONObject> docList = new ArrayList<BSONObject>(indexes.size());
|
||||
for (final Index i : indexes) {
|
||||
docList.add(i.toBSONObject());
|
||||
}
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), docList);
|
||||
}
|
||||
|
||||
private void find(final QueryMessage message, final String dbName, final String collectionName) throws DataStoreException {
|
||||
String fullCollectionName;
|
||||
if (message.getFullCollectionName().contains("$cmd")) {
|
||||
fullCollectionName = dbName + "." + collectionName;
|
||||
}
|
||||
else {
|
||||
fullCollectionName = message.getFullCollectionName();
|
||||
}
|
||||
final DataStore dataStore = this.dataStoreContext.getDataStore(fullCollectionName);
|
||||
QueryResponse response;
|
||||
if (this.useCursor()) {
|
||||
response = dataStore.find(message.getRealQuery(), message.getReturnFieldSelector(), message.getOrderByQuery(), message.getFacetQuery(), message.getNumberToSkip(), message.getNumberToReturn(), message.isBulkQuery(), message.isFromMaster());
|
||||
}
|
||||
else {
|
||||
response = dataStore.find(message.getRealQuery(), message.getReturnFieldSelector(), message.getOrderByQuery(), message.getFacetQuery(), message.getNumberToSkip(), message.getNumberToReturn() - 1, message.isBulkQuery(), message.isFromMaster());
|
||||
}
|
||||
final List<DBObject> doc = response.getDocList();
|
||||
if (this.useCursor()) {
|
||||
doc.remove(0);
|
||||
}
|
||||
final ReplyMessage reply = this.prepareReplyMessage(message.getMessageHeader().getRequestID(), (List<? extends BSONObject>)doc);
|
||||
reply.setStartingFrom(message.getNumberToSkip());
|
||||
if (response.hasMore() && this.useCursor()) {
|
||||
final Cursor cursor = new Cursor(message.getNumberToSkip(), message.getNumberToReturn(), message, response.getNumFound());
|
||||
this.dataStoreContext.addCursor(cursor);
|
||||
reply.setCursorID(cursor.getId());
|
||||
}
|
||||
this.writeMessage(reply);
|
||||
}
|
||||
|
||||
private void getLastError(final QueryMessage message, final String dbName) {
|
||||
final Object lastError = this.getLastError();
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
doc.put("ok", (Object)1.0);
|
||||
doc.put("n", (Object)1);
|
||||
doc.put("$err", (Object)((lastError == null) ? null : lastError.toString()));
|
||||
doc.put("err", (Object)((lastError == null) ? null : lastError.toString()));
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void getPrevError(final QueryMessage message, final String dbName) {
|
||||
final Object prevError = this.getPrevError();
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
doc.put("ok", (Object)1.0);
|
||||
doc.put("n", (Object)0);
|
||||
doc.put("$err", (Object)((prevError == null) ? null : prevError.toString()));
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void count(final QueryMessage message, final String dbName) throws DataStoreException {
|
||||
final BSONObject query = message.getQuery();
|
||||
final int count = this.dataStoreContext.getDataStore(dbName + "." + query.get("count")).count(message.getRealQuery(), message.isFromMaster());
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
doc.put("ok", (Object)1.0);
|
||||
doc.put("n", (Object)(double)count);
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void getSchema(final QueryMessage message, final String dbName) throws DataStoreException {
|
||||
final BSONObject query = message.getQuery();
|
||||
final String collectionName = (String)query.get("getSchema");
|
||||
final StoreConfig storeConfig = this.dataStoreContext.getDataStore(dbName + "." + collectionName).getStoreConfig();
|
||||
final Collection<Field> fields = storeConfig.getFields();
|
||||
final List<DBObject> docList = new ArrayList<DBObject>(fields.size());
|
||||
for (final Field field : fields) {
|
||||
final BasicDBObject f = new BasicDBObject();
|
||||
f.put("name", field.getName());
|
||||
f.put("type", field.getType().getType());
|
||||
docList.add(f);
|
||||
}
|
||||
final BSONObject doc = new BasicBSONObject();
|
||||
doc.put("ok", 1.0);
|
||||
doc.put("schema", docList);
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void reIndex(final QueryMessage message, final String dbName) throws DataStoreException {
|
||||
this.assertNotReadOnly();
|
||||
final BSONObject query = message.getQuery();
|
||||
final String collectionName = (String)query.get("reIndex");
|
||||
final DataStore dataStore = this.dataStoreContext.getDataStore(dbName + "." + collectionName);
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
final Object realQuery = query.get("query");
|
||||
if (null == realQuery) {
|
||||
doc.put("version", (Object)dataStore.reIndex(null));
|
||||
}
|
||||
else {
|
||||
if (!(realQuery instanceof BSONObject)) {
|
||||
throw new DataStoreException("invalid re-index query: " + realQuery);
|
||||
}
|
||||
dataStore.reIndex((BSONObject)realQuery);
|
||||
}
|
||||
doc.put("ok", (Object)1.0);
|
||||
doc.put("n", (Object)0.0);
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void whatsMyUri(final QueryMessage message) {
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
final InetSocketAddress address = (InetSocketAddress)this.getSession().getRemoteAddress();
|
||||
doc.put("you", (Object)(address.getAddress().getHostAddress() + ":" + address.getPort()));
|
||||
doc.put("ok", (Object)1.0);
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void listDatabases(final QueryMessage message) {
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
doc.put("ok", (Object)1.0);
|
||||
final Set<String> storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames();
|
||||
final Set<String> dbNames = new HashSet<String>();
|
||||
for (final String sn : storeNames) {
|
||||
final int i = sn.indexOf(46);
|
||||
if (i > 0) {
|
||||
final String name = sn.substring(0, i);
|
||||
dbNames.add(name);
|
||||
}
|
||||
}
|
||||
final List<DBObject> docs = new ArrayList<DBObject>(dbNames.size());
|
||||
for (final String dn : dbNames) {
|
||||
docs.add((DBObject)new BasicDBObject("name", (Object)dn));
|
||||
}
|
||||
doc.put("databases", (Object)docs);
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void listCollections(final QueryMessage message, final String dbName) {
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
doc.put("ok", (Object)1.0);
|
||||
final BSONObject cursor = (BSONObject)new BasicDBObject();
|
||||
doc.put("cursor", (Object)cursor);
|
||||
cursor.put("id", (Object)(long)message.getMessageHeader().getRequestID());
|
||||
cursor.put("ns", (Object)(dbName + ".$cmd.listCollections"));
|
||||
final BasicDBList firstBatch = new BasicDBList();
|
||||
cursor.put("firstBatch", (Object)firstBatch);
|
||||
final Set<String> storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames();
|
||||
for (final String sn : storeNames) {
|
||||
if (sn.startsWith(dbName + ".")) {
|
||||
final BSONObject collection = (BSONObject)new BasicDBObject();
|
||||
collection.put("name", (Object)sn.replace(dbName + ".", ""));
|
||||
collection.put("options", (Object)new BasicDBObject());
|
||||
firstBatch.add((Object)collection);
|
||||
}
|
||||
}
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void nonce(final QueryMessage message) {
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
doc.put("ok", (Object)1.0);
|
||||
doc.put("errmsg", (Object)"no such cmd");
|
||||
doc.put("bad cmd", (Object)new BasicBSONObject("nonce", (Object)true));
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void isMaster(final QueryMessage message) {
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
doc.put("ok", (Object)1.0);
|
||||
doc.put("ismaster", (Object)true);
|
||||
doc.put("secondary", (Object)false);
|
||||
final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress();
|
||||
doc.put("primary", (Object)localAddress.getAddress().getHostAddress());
|
||||
doc.put("maxWireVersion", (Object)2);
|
||||
doc.put("minWireVersion", (Object)0);
|
||||
final BasicBSONObject client = (BasicBSONObject)message.getQuery().get("client");
|
||||
if (client != null) {
|
||||
final String platform = client.getString("platform");
|
||||
if (platform != null && (platform.contains("python") || platform.contains("Python"))) {
|
||||
this.getSession().setAttribute((Object)"useCursor", (Object)true);
|
||||
}
|
||||
final BasicBSONObject applicationInfo = (BasicBSONObject)client.get("application");
|
||||
if (applicationInfo != null && StringUtils.isNotBlank(applicationInfo.getString("name"))) {
|
||||
this.getSession().setAttribute((Object)"useCursor", (Object)true);
|
||||
}
|
||||
}
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void buildinfo(final QueryMessage message) {
|
||||
final BasicDBList versionArray = new BasicDBList();
|
||||
versionArray.add((Object)2);
|
||||
versionArray.add((Object)6);
|
||||
versionArray.add((Object)8);
|
||||
versionArray.add((Object)0);
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
doc.put("ok", (Object)1.0);
|
||||
doc.put("bits", (Object)64.0);
|
||||
doc.put("debug", (Object)false);
|
||||
doc.put("version", (Object)"2.6.8");
|
||||
doc.put("versionArray", (Object)versionArray);
|
||||
doc.put("javascriptEngine", (Object)"V8");
|
||||
final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress();
|
||||
doc.put("primary", (Object)localAddress.getAddress().getHostAddress());
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void serverStatus(final QueryMessage message) {
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
doc.put("ok", (Object)1.0);
|
||||
final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress();
|
||||
doc.put("primary", (Object)localAddress.getAddress().getHostAddress());
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
|
||||
private void connectionStatus(final QueryMessage message) {
|
||||
final BSONObject doc = (BSONObject)new BasicBSONObject();
|
||||
doc.put("ok", (Object)1.0);
|
||||
final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress();
|
||||
doc.put("primary", (Object)localAddress.getAddress().getHostAddress());
|
||||
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
|
||||
}
|
||||
}
|
||||
|
||||
+20
-20
@@ -1,27 +1,27 @@
|
||||
package com.fangjia.datastore.handler;
|
||||
|
||||
import com.fangjia.datastore.mongo.protocol.MongoMessage;
|
||||
import com.fangjia.datastore.DataStore;
|
||||
import com.fangjia.datastore.DataStoreContext;
|
||||
import com.fangjia.datastore.DataStoreException;
|
||||
import com.fangjia.datastore.DataStoreContext;
|
||||
import com.fangjia.datastore.mongo.protocol.UpdateMessage;
|
||||
|
||||
public class UpdateMessageHandler extends MessageHandler<UpdateMessage> {
|
||||
|
||||
public UpdateMessageHandler(DataStoreContext dataStoreContext) {
|
||||
super(dataStoreContext);
|
||||
}
|
||||
|
||||
public void messageReceived(UpdateMessage message,String ip) throws Exception {
|
||||
assertNotReadOnly();
|
||||
if (message.isUpsert()) {
|
||||
throw new DataStoreException("upsert is not supported.");
|
||||
}
|
||||
if (!message.isMultiUpdate()) {
|
||||
throw new DataStoreException("only multi update is supported.");
|
||||
}
|
||||
|
||||
DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName());
|
||||
dataStore.update(message.getSelector(), message.getUpdate(),ip);
|
||||
}
|
||||
|
||||
public class UpdateMessageHandler extends MessageHandler<UpdateMessage>
|
||||
{
|
||||
public UpdateMessageHandler(final DataStoreContext dataStoreContext) {
|
||||
super(dataStoreContext);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void messageReceived(final UpdateMessage message) throws Exception {
|
||||
this.assertNotReadOnly();
|
||||
if (message.isUpsert()) {
|
||||
throw new DataStoreException("upsert is not supported.");
|
||||
}
|
||||
if (!message.isMultiUpdate()) {
|
||||
throw new DataStoreException("only multi update is supported.");
|
||||
}
|
||||
final DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName());
|
||||
dataStore.update(message.getSelector(), message.getUpdate());
|
||||
}
|
||||
}
|
||||
|
||||
+69
-86
@@ -1,92 +1,75 @@
|
||||
/**
|
||||
* JMongo is a mongodb driver writtern in java.
|
||||
* Copyright (C) 2010 Xiaohu Huang
|
||||
*
|
||||
* JMongo is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* JMongo is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package com.fangjia.datastore.mongo.codec;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import org.apache.mina.core.buffer.IoBuffer;
|
||||
import java.io.InputStream;
|
||||
|
||||
import org.apache.mina.core.buffer.IoBuffer;
|
||||
|
||||
public class BSONInputStream extends InputStream {
|
||||
private final IoBuffer buffer;
|
||||
|
||||
public BSONInputStream(IoBuffer buffer) {
|
||||
this.buffer = buffer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read() {
|
||||
return buffer.get();
|
||||
}
|
||||
|
||||
public synchronized int read(byte b[], int off, int len) {
|
||||
if (b == null) {
|
||||
throw new NullPointerException();
|
||||
} else if (off < 0 || len < 0 || len > b.length - off) {
|
||||
throw new IndexOutOfBoundsException();
|
||||
}
|
||||
buffer.get(b, off, len);
|
||||
return len;
|
||||
}
|
||||
|
||||
public int readInteger() {
|
||||
return this.buffer.getInt();
|
||||
}
|
||||
|
||||
public long readLong() {
|
||||
return this.buffer.getLong();
|
||||
}
|
||||
|
||||
public double readDouble() {
|
||||
return this.buffer.getDouble();
|
||||
}
|
||||
|
||||
public String readCString() throws IOException {
|
||||
ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
|
||||
int b;
|
||||
while (true) {
|
||||
b = read();
|
||||
if (b != 0) {
|
||||
byteStream.write(b);
|
||||
} else {
|
||||
return new String(byteStream.toByteArray(), "UTF-8");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public String readString() throws IOException {
|
||||
int len = readInteger();
|
||||
if (len < 0) {
|
||||
throw new IOException("can not read string from the stream");
|
||||
}
|
||||
if (len == 0) {
|
||||
read();
|
||||
return "";
|
||||
}
|
||||
byte[] buf = new byte[len];
|
||||
read(buf);
|
||||
return new String(buf, 0, len - 1, "UTF-8");
|
||||
}
|
||||
|
||||
@Override
|
||||
public int available() throws IOException {
|
||||
return buffer.remaining();
|
||||
}
|
||||
|
||||
public class BSONInputStream extends InputStream
|
||||
{
|
||||
private final IoBuffer buffer;
|
||||
|
||||
public BSONInputStream(final IoBuffer buffer) {
|
||||
this.buffer = buffer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read() {
|
||||
return this.buffer.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized int read(final byte[] b, final int off, final int len) {
|
||||
if (b == null) {
|
||||
throw new NullPointerException();
|
||||
}
|
||||
if (off < 0 || len < 0 || len > b.length - off) {
|
||||
throw new IndexOutOfBoundsException();
|
||||
}
|
||||
this.buffer.get(b, off, len);
|
||||
return len;
|
||||
}
|
||||
|
||||
public int readInteger() {
|
||||
return this.buffer.getInt();
|
||||
}
|
||||
|
||||
public long readLong() {
|
||||
return this.buffer.getLong();
|
||||
}
|
||||
|
||||
public double readDouble() {
|
||||
return this.buffer.getDouble();
|
||||
}
|
||||
|
||||
public String readCString() throws IOException {
|
||||
final ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
|
||||
while (true) {
|
||||
final int b = this.read();
|
||||
if (b == 0) {
|
||||
break;
|
||||
}
|
||||
byteStream.write(b);
|
||||
}
|
||||
return new String(byteStream.toByteArray(), "UTF-8");
|
||||
}
|
||||
|
||||
public String readString() throws IOException {
|
||||
final int len = this.readInteger();
|
||||
if (len < 0) {
|
||||
throw new IOException("can not read string from the stream");
|
||||
}
|
||||
if (len == 0) {
|
||||
this.read();
|
||||
return "";
|
||||
}
|
||||
final byte[] buf = new byte[len];
|
||||
this.read(buf);
|
||||
return new String(buf, 0, len - 1, "UTF-8");
|
||||
}
|
||||
|
||||
@Override
|
||||
public int available() throws IOException {
|
||||
return this.buffer.remaining();
|
||||
}
|
||||
}
|
||||
|
||||
+76
-88
@@ -1,95 +1,83 @@
|
||||
package com.fangjia.datastore.mongo.codec;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
|
||||
import org.apache.mina.core.buffer.AbstractIoBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
import java.nio.ByteBuffer;
|
||||
import org.apache.mina.core.buffer.IoBuffer;
|
||||
import org.apache.mina.core.buffer.IoBufferAllocator;
|
||||
|
||||
public class BSONIoBufferAllocator implements IoBufferAllocator {
|
||||
|
||||
public IoBuffer allocate(int capacity, boolean direct) {
|
||||
return wrap(allocateNioBuffer(capacity, direct));
|
||||
}
|
||||
|
||||
public ByteBuffer allocateNioBuffer(int capacity, boolean direct) {
|
||||
ByteBuffer nioBuffer;
|
||||
if (direct) {
|
||||
nioBuffer = ByteBuffer.allocateDirect(capacity);
|
||||
} else {
|
||||
nioBuffer = ByteBuffer.allocate(capacity);
|
||||
}
|
||||
nioBuffer.order(ByteOrder.LITTLE_ENDIAN);
|
||||
return nioBuffer;
|
||||
}
|
||||
|
||||
public IoBuffer wrap(ByteBuffer nioBuffer) {
|
||||
return new SimpleBuffer(nioBuffer);
|
||||
}
|
||||
|
||||
public void dispose() {
|
||||
// Do nothing
|
||||
}
|
||||
|
||||
private class SimpleBuffer extends AbstractIoBuffer {
|
||||
private ByteBuffer buf;
|
||||
|
||||
protected SimpleBuffer(ByteBuffer buf) {
|
||||
super(BSONIoBufferAllocator.this, buf.capacity());
|
||||
this.buf = buf;
|
||||
this.buf.order(ByteOrder.LITTLE_ENDIAN);
|
||||
}
|
||||
|
||||
protected SimpleBuffer(SimpleBuffer parent, ByteBuffer buf) {
|
||||
super(parent);
|
||||
this.buf = buf;
|
||||
this.buf.order(ByteOrder.LITTLE_ENDIAN);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ByteBuffer buf() {
|
||||
return buf;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void buf(ByteBuffer buf) {
|
||||
this.buf = buf;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected IoBuffer duplicate0() {
|
||||
return new SimpleBuffer(this, this.buf.duplicate());
|
||||
}
|
||||
|
||||
@Override
|
||||
protected IoBuffer slice0() {
|
||||
return new SimpleBuffer(this, this.buf.slice());
|
||||
}
|
||||
|
||||
@Override
|
||||
protected IoBuffer asReadOnlyBuffer0() {
|
||||
return new SimpleBuffer(this, this.buf.asReadOnlyBuffer());
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] array() {
|
||||
return buf.array();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int arrayOffset() {
|
||||
return buf.arrayOffset();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasArray() {
|
||||
return buf.hasArray();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void free() {
|
||||
// Do nothing
|
||||
}
|
||||
}
|
||||
public class BSONIoBufferAllocator implements IoBufferAllocator
|
||||
{
|
||||
public IoBuffer allocate(final int capacity, final boolean direct) {
|
||||
return this.wrap(this.allocateNioBuffer(capacity, direct));
|
||||
}
|
||||
|
||||
public ByteBuffer allocateNioBuffer(final int capacity, final boolean direct) {
|
||||
ByteBuffer nioBuffer;
|
||||
if (direct) {
|
||||
nioBuffer = ByteBuffer.allocateDirect(capacity);
|
||||
}
|
||||
else {
|
||||
nioBuffer = ByteBuffer.allocate(capacity);
|
||||
}
|
||||
nioBuffer.order(ByteOrder.LITTLE_ENDIAN);
|
||||
return nioBuffer;
|
||||
}
|
||||
|
||||
public IoBuffer wrap(final ByteBuffer nioBuffer) {
|
||||
return (IoBuffer)new SimpleBuffer(nioBuffer);
|
||||
}
|
||||
|
||||
public void dispose() {
|
||||
}
|
||||
|
||||
private class SimpleBuffer extends AbstractIoBuffer
|
||||
{
|
||||
private ByteBuffer buf;
|
||||
|
||||
protected SimpleBuffer(final ByteBuffer buf) {
|
||||
super((IoBufferAllocator)BSONIoBufferAllocator.this, buf.capacity());
|
||||
(this.buf = buf).order(ByteOrder.LITTLE_ENDIAN);
|
||||
}
|
||||
|
||||
protected SimpleBuffer(final SimpleBuffer parent, final ByteBuffer buf) {
|
||||
super((AbstractIoBuffer)parent);
|
||||
(this.buf = buf).order(ByteOrder.LITTLE_ENDIAN);
|
||||
}
|
||||
|
||||
public ByteBuffer buf() {
|
||||
return this.buf;
|
||||
}
|
||||
|
||||
protected void buf(final ByteBuffer buf) {
|
||||
this.buf = buf;
|
||||
}
|
||||
|
||||
protected IoBuffer duplicate0() {
|
||||
return (IoBuffer)new SimpleBuffer(this, this.buf.duplicate());
|
||||
}
|
||||
|
||||
protected IoBuffer slice0() {
|
||||
return (IoBuffer)new SimpleBuffer(this, this.buf.slice());
|
||||
}
|
||||
|
||||
protected IoBuffer asReadOnlyBuffer0() {
|
||||
return (IoBuffer)new SimpleBuffer(this, this.buf.asReadOnlyBuffer());
|
||||
}
|
||||
|
||||
public byte[] array() {
|
||||
return this.buf.array();
|
||||
}
|
||||
|
||||
public int arrayOffset() {
|
||||
return this.buf.arrayOffset();
|
||||
}
|
||||
|
||||
public boolean hasArray() {
|
||||
return this.buf.hasArray();
|
||||
}
|
||||
|
||||
public void free() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+79
-97
@@ -1,102 +1,84 @@
|
||||
/**
|
||||
* JMongo is a mongodb driver writtern in java.
|
||||
* Copyright (C) 2010 Xiaohu Huang
|
||||
*
|
||||
* JMongo is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* JMongo is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package com.fangjia.datastore.mongo.codec;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.io.IOException;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
|
||||
public class BSONOutputStream extends ByteArrayOutputStream {
|
||||
|
||||
private static final String ENCDOING = "UTF-8";
|
||||
|
||||
public BSONOutputStream() {
|
||||
super();
|
||||
}
|
||||
|
||||
public BSONOutputStream(int size) {
|
||||
super(size);
|
||||
}
|
||||
|
||||
public void writeCString(String s) throws IOException {
|
||||
write(s.getBytes(ENCDOING));
|
||||
write(0);
|
||||
}
|
||||
|
||||
public void writeInteger(int i) throws IOException {
|
||||
write(i);
|
||||
write(i >> 8);
|
||||
write(i >> 16);
|
||||
write(i >> 24);
|
||||
}
|
||||
|
||||
public void writeLong(long l) throws IOException {
|
||||
write((byte) l);
|
||||
write((byte) (l >> 8));
|
||||
write((byte) (l >> 16));
|
||||
write((byte) (l >> 24));
|
||||
write((byte) (l >> 32));
|
||||
write((byte) (l >> 40));
|
||||
write((byte) (l >> 48));
|
||||
write((byte) (l >> 56));
|
||||
}
|
||||
|
||||
public void writeString(String s) throws IOException {
|
||||
byte[] bs = s.getBytes(ENCDOING);
|
||||
writeInteger(bs.length + 1);
|
||||
write(bs);
|
||||
write(0);
|
||||
}
|
||||
|
||||
public void set(int index, int i) {
|
||||
int newcount = Math.max(count, index + 4);
|
||||
if (newcount > buf.length) {
|
||||
buf = Arrays.copyOf(buf, Math.max(buf.length << 1, newcount));
|
||||
}
|
||||
buf[index] = (byte) i;
|
||||
buf[index + 1] = (byte) (i >> 8);
|
||||
buf[index + 2] = (byte) (i >> 16);
|
||||
buf[index + 3] = (byte) (i >> 24);
|
||||
count = newcount;
|
||||
}
|
||||
|
||||
public void set(int index, byte[] b, int off, int len) {
|
||||
if ((off < 0) || (off > b.length) || (len < 0) || ((off + len) > b.length) || ((off + len) < 0)) {
|
||||
throw new IndexOutOfBoundsException();
|
||||
} else if (len == 0) {
|
||||
return;
|
||||
}
|
||||
int newcount = Math.max(count, index + len);
|
||||
if (newcount > buf.length) {
|
||||
buf = Arrays.copyOf(buf, Math.max(buf.length << 1, newcount));
|
||||
}
|
||||
System.arraycopy(b, off, buf, index, len);
|
||||
count = newcount;
|
||||
}
|
||||
|
||||
public void setInteger(int index, int i) {
|
||||
byte[] b = new byte[] { (byte) i, (byte) (i >> 8), (byte) (i >> 16), (byte) (i >> 24) };
|
||||
set(index, b, 0, b.length);
|
||||
}
|
||||
|
||||
public void setLong(int index, long l) {
|
||||
byte[] b = new byte[] { (byte) l, (byte) (l >> 8), (byte) (l >> 16), (byte) (l >> 24), (byte) (l >> 32),
|
||||
(byte) (l >> 40), (byte) (l >> 48), (byte) (l >> 56) };
|
||||
set(index, b, 0, b.length);
|
||||
}
|
||||
public class BSONOutputStream extends ByteArrayOutputStream
|
||||
{
|
||||
private static final String ENCDOING = "UTF-8";
|
||||
|
||||
public BSONOutputStream() {
|
||||
}
|
||||
|
||||
public BSONOutputStream(final int size) {
|
||||
super(size);
|
||||
}
|
||||
|
||||
public void writeCString(final String s) throws IOException {
|
||||
this.write(s.getBytes("UTF-8"));
|
||||
this.write(0);
|
||||
}
|
||||
|
||||
public void writeInteger(final int i) throws IOException {
|
||||
this.write(i);
|
||||
this.write(i >> 8);
|
||||
this.write(i >> 16);
|
||||
this.write(i >> 24);
|
||||
}
|
||||
|
||||
public void writeLong(final long l) throws IOException {
|
||||
this.write((byte)l);
|
||||
this.write((byte)(l >> 8));
|
||||
this.write((byte)(l >> 16));
|
||||
this.write((byte)(l >> 24));
|
||||
this.write((byte)(l >> 32));
|
||||
this.write((byte)(l >> 40));
|
||||
this.write((byte)(l >> 48));
|
||||
this.write((byte)(l >> 56));
|
||||
}
|
||||
|
||||
public void writeString(final String s) throws IOException {
|
||||
final byte[] bs = s.getBytes("UTF-8");
|
||||
this.writeInteger(bs.length + 1);
|
||||
this.write(bs);
|
||||
this.write(0);
|
||||
}
|
||||
|
||||
public void set(final int index, final int i) {
|
||||
final int newcount = Math.max(this.count, index + 4);
|
||||
if (newcount > this.buf.length) {
|
||||
this.buf = Arrays.copyOf(this.buf, Math.max(this.buf.length << 1, newcount));
|
||||
}
|
||||
this.buf[index] = (byte)i;
|
||||
this.buf[index + 1] = (byte)(i >> 8);
|
||||
this.buf[index + 2] = (byte)(i >> 16);
|
||||
this.buf[index + 3] = (byte)(i >> 24);
|
||||
this.count = newcount;
|
||||
}
|
||||
|
||||
public void set(final int index, final byte[] b, final int off, final int len) {
|
||||
if (off < 0 || off > b.length || len < 0 || off + len > b.length || off + len < 0) {
|
||||
throw new IndexOutOfBoundsException();
|
||||
}
|
||||
if (len == 0) {
|
||||
return;
|
||||
}
|
||||
final int newcount = Math.max(this.count, index + len);
|
||||
if (newcount > this.buf.length) {
|
||||
this.buf = Arrays.copyOf(this.buf, Math.max(this.buf.length << 1, newcount));
|
||||
}
|
||||
System.arraycopy(b, off, this.buf, index, len);
|
||||
this.count = newcount;
|
||||
}
|
||||
|
||||
public void setInteger(final int index, final int i) {
|
||||
final byte[] b = { (byte)i, (byte)(i >> 8), (byte)(i >> 16), (byte)(i >> 24) };
|
||||
this.set(index, b, 0, b.length);
|
||||
}
|
||||
|
||||
public void setLong(final int index, final long l) {
|
||||
final byte[] b = { (byte)l, (byte)(l >> 8), (byte)(l >> 16), (byte)(l >> 24), (byte)(l >> 32), (byte)(l >> 40), (byte)(l >> 48), (byte)(l >> 56) };
|
||||
this.set(index, b, 0, b.length);
|
||||
}
|
||||
}
|
||||
|
||||
+100
-103
@@ -1,111 +1,108 @@
|
||||
package com.fangjia.datastore.mongo.codec;
|
||||
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.apache.mina.core.buffer.BufferDataException;
|
||||
import com.fangjia.datastore.mongo.protocol.UpdateMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.ReplyMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.QueryMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.KillCursorsMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.InsertMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.GetMoreMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.DeleteMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.MongoMessage;
|
||||
import org.apache.mina.filter.codec.ProtocolDecoderException;
|
||||
import com.fangjia.datastore.mongo.protocol.OperationCode;
|
||||
import org.apache.mina.filter.codec.ProtocolDecoderOutput;
|
||||
import org.apache.mina.core.buffer.IoBuffer;
|
||||
import org.apache.mina.core.session.IoSession;
|
||||
import org.apache.mina.filter.codec.CumulativeProtocolDecoder;
|
||||
import org.apache.mina.filter.codec.ProtocolDecoderException;
|
||||
import org.apache.mina.filter.codec.ProtocolDecoderOutput;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.fangjia.datastore.mongo.protocol.DeleteMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.GetMoreMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.InsertMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.KillCursorsMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.MongoMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.OperationCode;
|
||||
import com.fangjia.datastore.mongo.protocol.QueryMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.ReplyMessage;
|
||||
import com.fangjia.datastore.mongo.protocol.UpdateMessage;
|
||||
|
||||
public class MongoProtocolDecoder extends CumulativeProtocolDecoder {
|
||||
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(MongoProtocolDecoder.class);
|
||||
|
||||
@Override
|
||||
protected boolean doDecode(IoSession session, IoBuffer in, ProtocolDecoderOutput out) throws Exception {
|
||||
try {
|
||||
if (!prefixedDataAvailable(in, 4, true)) {
|
||||
return false;
|
||||
}
|
||||
int start = in.position();
|
||||
int limit = in.limit();
|
||||
int messageSize = in.getInt(start);
|
||||
in.limit(start + messageSize);
|
||||
try {
|
||||
final IoBuffer buf = in.slice();
|
||||
int opCodeValue = buf.getInt(buf.position() + 12);
|
||||
OperationCode opCode = OperationCode.valueOf(opCodeValue);
|
||||
MongoMessage message = newInstance(opCode);
|
||||
BSONInputStream input = new BSONInputStream(buf);
|
||||
message.read(input);
|
||||
out.write(message);
|
||||
return true;
|
||||
} finally {
|
||||
in.position(start + messageSize);
|
||||
in.limit(limit);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
ProtocolDecoderException pde = new ProtocolDecoderException("error decode message:", e);
|
||||
pde.setHexdump(in.getHexDump());
|
||||
LOGGER.error("", pde);
|
||||
throw pde;
|
||||
}
|
||||
}
|
||||
|
||||
private MongoMessage newInstance(OperationCode opCode) {
|
||||
switch (opCode) {
|
||||
case OP_DELETE:
|
||||
return new DeleteMessage();
|
||||
case OP_GET_MORE:
|
||||
return new GetMoreMessage();
|
||||
case OP_INSERT:
|
||||
return new InsertMessage();
|
||||
case OP_KILL_CURSORS:
|
||||
return new KillCursorsMessage();
|
||||
case OP_QUERY:
|
||||
return new QueryMessage();
|
||||
case OP_REPLY:
|
||||
return new ReplyMessage();
|
||||
case OP_UPDATE:
|
||||
return new UpdateMessage();
|
||||
case OP_MSG:
|
||||
case RESERVED:
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean prefixedDataAvailable(IoBuffer buffer, int prefixLength, boolean includePrefix) {
|
||||
if (buffer.remaining() < prefixLength) {
|
||||
return false;
|
||||
}
|
||||
|
||||
int dataLength;
|
||||
switch (prefixLength) {
|
||||
case 1:
|
||||
dataLength = buffer.getUnsigned(buffer.position());
|
||||
break;
|
||||
case 2:
|
||||
dataLength = buffer.getUnsignedShort(buffer.position());
|
||||
break;
|
||||
case 4:
|
||||
dataLength = buffer.getInt(buffer.position());
|
||||
break;
|
||||
default:
|
||||
throw new IllegalArgumentException("prefixLength: " + prefixLength);
|
||||
}
|
||||
|
||||
if (dataLength < 0) {
|
||||
throw new BufferDataException("dataLength: " + dataLength);
|
||||
}
|
||||
|
||||
if (includePrefix) {
|
||||
return buffer.remaining() >= dataLength;
|
||||
} else {
|
||||
return buffer.remaining() - prefixLength >= dataLength;
|
||||
}
|
||||
}
|
||||
import org.apache.mina.filter.codec.CumulativeProtocolDecoder;
|
||||
|
||||
public class MongoProtocolDecoder extends CumulativeProtocolDecoder
|
||||
{
|
||||
private static final Logger LOGGER;
|
||||
|
||||
protected boolean doDecode(final IoSession session, final IoBuffer in, final ProtocolDecoderOutput out) throws Exception {
|
||||
try {
|
||||
if (!this.prefixedDataAvailable(in, 4, true)) {
|
||||
return false;
|
||||
}
|
||||
final int start = in.position();
|
||||
final int limit = in.limit();
|
||||
final int messageSize = in.getInt(start);
|
||||
in.limit(start + messageSize);
|
||||
try {
|
||||
final IoBuffer buf = in.slice();
|
||||
final int opCodeValue = buf.getInt(buf.position() + 12);
|
||||
final OperationCode opCode = OperationCode.valueOf(opCodeValue);
|
||||
final MongoMessage message = this.newInstance(opCode);
|
||||
final BSONInputStream input = new BSONInputStream(buf);
|
||||
message.read(input);
|
||||
out.write((Object)message);
|
||||
return true;
|
||||
}
|
||||
finally {
|
||||
in.position(start + messageSize);
|
||||
in.limit(limit);
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
final ProtocolDecoderException pde = new ProtocolDecoderException("error decode message:", (Throwable)e);
|
||||
pde.setHexdump(in.getHexDump());
|
||||
MongoProtocolDecoder.LOGGER.error("", (Throwable)pde);
|
||||
throw pde;
|
||||
}
|
||||
}
|
||||
|
||||
private MongoMessage newInstance(final OperationCode opCode) {
|
||||
switch (opCode) {
|
||||
case OP_DELETE:
|
||||
return new DeleteMessage();
|
||||
case OP_GET_MORE:
|
||||
return new GetMoreMessage();
|
||||
case OP_INSERT:
|
||||
return new InsertMessage();
|
||||
case OP_KILL_CURSORS:
|
||||
return new KillCursorsMessage();
|
||||
case OP_QUERY:
|
||||
return new QueryMessage();
|
||||
case OP_REPLY:
|
||||
return new ReplyMessage();
|
||||
case OP_UPDATE:
|
||||
return new UpdateMessage();
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean prefixedDataAvailable(final IoBuffer buffer, final int prefixLength, final boolean includePrefix) {
|
||||
if (buffer.remaining() < prefixLength) {
|
||||
return false;
|
||||
}
|
||||
int dataLength = 0;
|
||||
switch (prefixLength) {
|
||||
case 1:
|
||||
dataLength = buffer.getUnsigned(buffer.position());
|
||||
break;
|
||||
case 2:
|
||||
dataLength = buffer.getUnsignedShort(buffer.position());
|
||||
break;
|
||||
case 4:
|
||||
dataLength = buffer.getInt(buffer.position());
|
||||
break;
|
||||
default:
|
||||
throw new IllegalArgumentException("prefixLength: " + prefixLength);
|
||||
}
|
||||
if (dataLength < 0) {
|
||||
throw new BufferDataException("dataLength: " + dataLength);
|
||||
}
|
||||
if (includePrefix) {
|
||||
return buffer.remaining() >= dataLength;
|
||||
}
|
||||
return buffer.remaining() - prefixLength >= dataLength;
|
||||
}
|
||||
|
||||
static {
|
||||
LOGGER = LoggerFactory.getLogger((Class)MongoProtocolDecoder.class);
|
||||
}
|
||||
}
|
||||
|
||||
+32
-30
@@ -1,36 +1,38 @@
|
||||
package com.fangjia.datastore.mongo.codec;
|
||||
|
||||
import org.apache.mina.core.buffer.IoBuffer;
|
||||
import org.apache.mina.core.session.IoSession;
|
||||
import org.apache.mina.filter.codec.ProtocolEncoder;
|
||||
import org.apache.mina.filter.codec.ProtocolEncoderException;
|
||||
import org.apache.mina.filter.codec.ProtocolEncoderOutput;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.mina.filter.codec.ProtocolEncoderException;
|
||||
import org.apache.mina.core.buffer.IoBuffer;
|
||||
import com.fangjia.datastore.mongo.protocol.MongoMessage;
|
||||
import org.apache.mina.filter.codec.ProtocolEncoderOutput;
|
||||
import org.apache.mina.core.session.IoSession;
|
||||
import org.slf4j.Logger;
|
||||
import org.apache.mina.filter.codec.ProtocolEncoder;
|
||||
|
||||
public class MongoProtocolEncoder implements ProtocolEncoder {
|
||||
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(MongoProtocolEncoder.class);
|
||||
|
||||
public void encode(IoSession session, Object message, ProtocolEncoderOutput out) throws Exception {
|
||||
try {
|
||||
if (message instanceof MongoMessage) {
|
||||
MongoMessage mongoMessage = (MongoMessage) message;
|
||||
BSONOutputStream output = new BSONOutputStream(mongoMessage.getMessageHeader().getMessageLength());
|
||||
mongoMessage.write(output);
|
||||
out.write(IoBuffer.wrap(output.toByteArray()));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
ProtocolEncoderException pe = new ProtocolEncoderException("error encode message: " + message, e);
|
||||
LOGGER.error("", pe);
|
||||
throw pe;
|
||||
}
|
||||
}
|
||||
|
||||
public void dispose(IoSession session) throws Exception {
|
||||
// empty
|
||||
}
|
||||
|
||||
public class MongoProtocolEncoder implements ProtocolEncoder
|
||||
{
|
||||
private static final Logger LOGGER;
|
||||
|
||||
public void encode(final IoSession session, final Object message, final ProtocolEncoderOutput out) throws Exception {
|
||||
try {
|
||||
if (message instanceof MongoMessage) {
|
||||
final MongoMessage mongoMessage = (MongoMessage)message;
|
||||
final BSONOutputStream output = new BSONOutputStream(mongoMessage.getMessageHeader().getMessageLength());
|
||||
mongoMessage.write(output);
|
||||
out.write((Object)IoBuffer.wrap(output.toByteArray()));
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
final ProtocolEncoderException pe = new ProtocolEncoderException("error encode message: " + message, (Throwable)e);
|
||||
MongoProtocolEncoder.LOGGER.error("", (Throwable)pe);
|
||||
throw pe;
|
||||
}
|
||||
}
|
||||
|
||||
public void dispose(final IoSession session) throws Exception {
|
||||
}
|
||||
|
||||
static {
|
||||
LOGGER = LoggerFactory.getLogger((Class)MongoProtocolEncoder.class);
|
||||
}
|
||||
}
|
||||
|
||||
+93
-106
@@ -1,111 +1,98 @@
|
||||
/**
|
||||
* JMongo is a mongodb driver writtern in java.
|
||||
* Copyright (C) 2010 Xiaohu Huang
|
||||
*
|
||||
* JMongo is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* JMongo is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package com.fangjia.datastore.mongo.protocol;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.bson.BSONDecoder;
|
||||
import org.bson.BSONEncoder;
|
||||
import org.bson.BSONObject;
|
||||
import org.bson.BasicBSONDecoder;
|
||||
import org.bson.BasicBSONObject;
|
||||
|
||||
import com.fangjia.datastore.mongo.codec.BSONInputStream;
|
||||
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
|
||||
import com.fangjia.datastore.utils.BSONUtils;
|
||||
import org.bson.BSONDecoder;
|
||||
import java.io.InputStream;
|
||||
import org.bson.BasicBSONDecoder;
|
||||
import com.fangjia.datastore.mongo.codec.BSONInputStream;
|
||||
import java.io.IOException;
|
||||
import org.bson.BSONEncoder;
|
||||
import org.bson.BasicBSONEncoder;
|
||||
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
|
||||
import org.bson.BasicBSONObject;
|
||||
import org.bson.BSONObject;
|
||||
|
||||
public class DeleteMessage implements MongoMessage {
|
||||
|
||||
private static final long serialVersionUID = -3350216587439425208L;
|
||||
private MessageHeader messageHeader;
|
||||
private String fullCollectionName;
|
||||
private int flags;
|
||||
private BSONObject selector;
|
||||
|
||||
public DeleteMessage() {
|
||||
messageHeader = new MessageHeader(OperationCode.OP_DELETE);
|
||||
selector = new BasicBSONObject();
|
||||
}
|
||||
|
||||
public void write(BSONOutputStream out) throws IOException {
|
||||
messageHeader.write(out);
|
||||
out.writeInteger(0);
|
||||
out.writeCString(this.fullCollectionName);
|
||||
out.writeInteger(this.flags);
|
||||
BSONEncoder encoder = new BSONEncoder();
|
||||
out.write(encoder.encode(selector));
|
||||
out.set(0, out.size());
|
||||
}
|
||||
|
||||
public void read(BSONInputStream in) throws IOException {
|
||||
messageHeader.read(in);
|
||||
in.readInteger();
|
||||
this.fullCollectionName = in.readCString();
|
||||
this.flags = in.readInteger();
|
||||
BSONDecoder decoder = new BasicBSONDecoder();
|
||||
selector = decoder.readObject(in);
|
||||
}
|
||||
|
||||
public MessageHeader getMessageHeader() {
|
||||
return messageHeader;
|
||||
}
|
||||
|
||||
public String getFullCollectionName() {
|
||||
return fullCollectionName;
|
||||
}
|
||||
|
||||
public void setFullCollectionName(String fullCollectionName) {
|
||||
this.fullCollectionName = fullCollectionName;
|
||||
}
|
||||
|
||||
public int getFlags() {
|
||||
return flags;
|
||||
}
|
||||
|
||||
public void setFlags(int flags) {
|
||||
this.flags = flags;
|
||||
}
|
||||
|
||||
public void setSingleRemove(boolean b) {
|
||||
this.flags = (int) BSONUtils.bitSet(this.flags, 0, b);
|
||||
}
|
||||
|
||||
public BSONObject getSelector() {
|
||||
return selector;
|
||||
}
|
||||
|
||||
public void setSelector(BSONObject selector) {
|
||||
this.selector = selector;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("messageHeader:").append(messageHeader);
|
||||
builder.append(", fullCollectionName:").append(fullCollectionName);
|
||||
builder.append(", flags:").append(flags);
|
||||
builder.append(", selector:").append(selector);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
public String toPrettyString() {
|
||||
return toString();
|
||||
}
|
||||
|
||||
public class DeleteMessage implements MongoMessage
|
||||
{
|
||||
private static final long serialVersionUID = -3350216587439425208L;
|
||||
private MessageHeader messageHeader;
|
||||
private String fullCollectionName;
|
||||
private int flags;
|
||||
private BSONObject selector;
|
||||
|
||||
public DeleteMessage() {
|
||||
this.messageHeader = new MessageHeader(OperationCode.OP_DELETE);
|
||||
this.selector = (BSONObject)new BasicBSONObject();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(final BSONOutputStream out) throws IOException {
|
||||
this.messageHeader.write(out);
|
||||
out.writeInteger(0);
|
||||
out.writeCString(this.fullCollectionName);
|
||||
out.writeInteger(this.flags);
|
||||
final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder();
|
||||
out.write(encoder.encode(this.selector));
|
||||
out.set(0, out.size());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void read(final BSONInputStream in) throws IOException {
|
||||
this.messageHeader.read(in);
|
||||
in.readInteger();
|
||||
this.fullCollectionName = in.readCString();
|
||||
this.flags = in.readInteger();
|
||||
final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder();
|
||||
this.selector = decoder.readObject((InputStream)in);
|
||||
}
|
||||
|
||||
@Override
|
||||
public MessageHeader getMessageHeader() {
|
||||
return this.messageHeader;
|
||||
}
|
||||
|
||||
public String getFullCollectionName() {
|
||||
return this.fullCollectionName;
|
||||
}
|
||||
|
||||
public void setFullCollectionName(final String fullCollectionName) {
|
||||
this.fullCollectionName = fullCollectionName;
|
||||
}
|
||||
|
||||
public int getFlags() {
|
||||
return this.flags;
|
||||
}
|
||||
|
||||
public void setFlags(final int flags) {
|
||||
this.flags = flags;
|
||||
}
|
||||
|
||||
public void setSingleRemove(final boolean b) {
|
||||
this.flags = (int)BSONUtils.bitSet(this.flags, 0, b);
|
||||
}
|
||||
|
||||
public BSONObject getSelector() {
|
||||
return this.selector;
|
||||
}
|
||||
|
||||
public void setSelector(final BSONObject selector) {
|
||||
this.selector = selector;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("messageHeader:").append(this.messageHeader);
|
||||
builder.append(", fullCollectionName:").append(this.fullCollectionName);
|
||||
builder.append(", flags:").append(this.flags);
|
||||
builder.append(", selector:").append(this.selector);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toPrettyString() {
|
||||
return this.toString();
|
||||
}
|
||||
}
|
||||
|
||||
+77
-91
@@ -1,97 +1,83 @@
|
||||
/**
|
||||
* JMongo is a mongodb driver writtern in java.
|
||||
* Copyright (C) 2010 Xiaohu Huang
|
||||
*
|
||||
* JMongo is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* JMongo is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package com.fangjia.datastore.mongo.protocol;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import com.fangjia.datastore.mongo.codec.BSONInputStream;
|
||||
import java.io.IOException;
|
||||
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
|
||||
|
||||
public class GetMoreMessage implements MongoMessage {
|
||||
|
||||
private static final long serialVersionUID = -3350216587439425208L;
|
||||
private MessageHeader messageHeader;
|
||||
private String fullCollectionName;
|
||||
private int numberToReturn;
|
||||
private long cursorID;
|
||||
|
||||
public GetMoreMessage() {
|
||||
messageHeader = new MessageHeader(OperationCode.OP_GET_MORE);
|
||||
}
|
||||
|
||||
public void write(BSONOutputStream out) throws IOException {
|
||||
messageHeader.write(out);
|
||||
out.writeInteger(0);
|
||||
out.writeCString(this.fullCollectionName);
|
||||
out.writeInteger(this.numberToReturn);
|
||||
out.writeLong(cursorID);
|
||||
out.set(0, out.size());
|
||||
}
|
||||
|
||||
public void read(BSONInputStream in) throws IOException {
|
||||
messageHeader.read(in);
|
||||
in.readInteger();
|
||||
this.fullCollectionName = in.readCString();
|
||||
this.numberToReturn = in.readInteger();
|
||||
this.cursorID = in.readLong();
|
||||
}
|
||||
|
||||
public MessageHeader getMessageHeader() {
|
||||
return messageHeader;
|
||||
}
|
||||
|
||||
public String getFullCollectionName() {
|
||||
return fullCollectionName;
|
||||
}
|
||||
|
||||
public void setFullCollectionName(String fullCollectionName) {
|
||||
this.fullCollectionName = fullCollectionName;
|
||||
}
|
||||
|
||||
public int getNumberToReturn() {
|
||||
return numberToReturn;
|
||||
}
|
||||
|
||||
public void setNumberToReturn(int numberToReturn) {
|
||||
this.numberToReturn = numberToReturn;
|
||||
}
|
||||
|
||||
public long getCursorID() {
|
||||
return cursorID;
|
||||
}
|
||||
|
||||
public void setCursorID(long cursorID) {
|
||||
this.cursorID = cursorID;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("messageHeader:").append(messageHeader);
|
||||
builder.append(", fullCollectionName:").append(fullCollectionName);
|
||||
builder.append(", numberToReturn:").append(numberToReturn);
|
||||
builder.append(", cursorID:").append(cursorID);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
public String toPrettyString() {
|
||||
return toString();
|
||||
}
|
||||
|
||||
public class GetMoreMessage implements MongoMessage
|
||||
{
|
||||
private static final long serialVersionUID = -3350216587439425208L;
|
||||
private MessageHeader messageHeader;
|
||||
private String fullCollectionName;
|
||||
private int numberToReturn;
|
||||
private long cursorID;
|
||||
|
||||
public GetMoreMessage() {
|
||||
this.messageHeader = new MessageHeader(OperationCode.OP_GET_MORE);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(final BSONOutputStream out) throws IOException {
|
||||
this.messageHeader.write(out);
|
||||
out.writeInteger(0);
|
||||
out.writeCString(this.fullCollectionName);
|
||||
out.writeInteger(this.numberToReturn);
|
||||
out.writeLong(this.cursorID);
|
||||
out.set(0, out.size());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void read(final BSONInputStream in) throws IOException {
|
||||
this.messageHeader.read(in);
|
||||
in.readInteger();
|
||||
this.fullCollectionName = in.readCString();
|
||||
this.numberToReturn = in.readInteger();
|
||||
this.cursorID = in.readLong();
|
||||
}
|
||||
|
||||
@Override
|
||||
public MessageHeader getMessageHeader() {
|
||||
return this.messageHeader;
|
||||
}
|
||||
|
||||
public String getFullCollectionName() {
|
||||
return this.fullCollectionName;
|
||||
}
|
||||
|
||||
public void setFullCollectionName(final String fullCollectionName) {
|
||||
this.fullCollectionName = fullCollectionName;
|
||||
}
|
||||
|
||||
public int getNumberToReturn() {
|
||||
return this.numberToReturn;
|
||||
}
|
||||
|
||||
public void setNumberToReturn(final int numberToReturn) {
|
||||
this.numberToReturn = numberToReturn;
|
||||
}
|
||||
|
||||
public long getCursorID() {
|
||||
return this.cursorID;
|
||||
}
|
||||
|
||||
public void setCursorID(final long cursorID) {
|
||||
this.cursorID = cursorID;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("messageHeader:").append(this.messageHeader);
|
||||
builder.append(", fullCollectionName:").append(this.fullCollectionName);
|
||||
builder.append(", numberToReturn:").append(this.numberToReturn);
|
||||
builder.append(", cursorID:").append(this.cursorID);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toPrettyString() {
|
||||
return this.toString();
|
||||
}
|
||||
}
|
||||
|
||||
+99
-110
@@ -1,116 +1,105 @@
|
||||
/**
|
||||
* JMongo is a mongodb driver writtern in java.
|
||||
* Copyright (C) 2010 Xiaohu Huang
|
||||
*
|
||||
* JMongo is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* JMongo is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package com.fangjia.datastore.mongo.protocol;
|
||||
|
||||
import java.io.IOException;
|
||||
import org.bson.BSONDecoder;
|
||||
import java.io.InputStream;
|
||||
import org.bson.BasicBSONDecoder;
|
||||
import java.util.ArrayList;
|
||||
import com.fangjia.datastore.mongo.codec.BSONInputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
import org.bson.BSONEncoder;
|
||||
import org.bson.BasicBSONEncoder;
|
||||
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
|
||||
import org.bson.BSONObject;
|
||||
import java.util.List;
|
||||
|
||||
import org.bson.BSONDecoder;
|
||||
import org.bson.BSONEncoder;
|
||||
import org.bson.BSONObject;
|
||||
import org.bson.BasicBSONDecoder;
|
||||
|
||||
import com.fangjia.datastore.mongo.codec.BSONInputStream;
|
||||
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
|
||||
|
||||
public class InsertMessage implements MongoMessage {
|
||||
|
||||
private static final long serialVersionUID = -3350216587439425208L;
|
||||
private MessageHeader messageHeader;
|
||||
private String fullCollectionName;
|
||||
private List<BSONObject> documents;
|
||||
|
||||
public InsertMessage() {
|
||||
messageHeader = new MessageHeader(OperationCode.OP_INSERT);
|
||||
}
|
||||
|
||||
public void write(BSONOutputStream out) throws IOException {
|
||||
messageHeader.write(out);
|
||||
out.writeInteger(0);
|
||||
out.writeCString(this.fullCollectionName);
|
||||
if (null != documents && documents.size() > 0) {
|
||||
BSONEncoder encoder = new BSONEncoder();
|
||||
for (BSONObject doc : documents) {
|
||||
out.write(encoder.encode(doc));
|
||||
}
|
||||
}
|
||||
out.setInteger(0, out.size());
|
||||
}
|
||||
|
||||
public void read(BSONInputStream in) throws IOException {
|
||||
messageHeader.read(in);
|
||||
in.readInteger();
|
||||
this.fullCollectionName = in.readCString();
|
||||
while (in.available() > 0) {
|
||||
if (null == documents) {
|
||||
documents = new ArrayList<BSONObject>();
|
||||
}
|
||||
BSONDecoder decoder = new BasicBSONDecoder();
|
||||
documents.add(decoder.readObject(in));
|
||||
}
|
||||
}
|
||||
|
||||
public MessageHeader getMessageHeader() {
|
||||
return messageHeader;
|
||||
}
|
||||
|
||||
public String getFullCollectionName() {
|
||||
return fullCollectionName;
|
||||
}
|
||||
|
||||
public void setFullCollectionName(String fullCollectionName) {
|
||||
this.fullCollectionName = fullCollectionName;
|
||||
}
|
||||
|
||||
public List<BSONObject> getDocuments() {
|
||||
return documents;
|
||||
}
|
||||
|
||||
public void setDocuments(List<BSONObject> documents) {
|
||||
this.documents = documents;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("messageHeader:").append(messageHeader);
|
||||
builder.append(", fullCollectionName:").append(fullCollectionName);
|
||||
builder.append(", documents:").append(documents);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
public String toPrettyString() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("messageHeader:").append(messageHeader);
|
||||
builder.append(", fullCollectionName:").append(fullCollectionName);
|
||||
if (null != documents && documents.size() > 0) {
|
||||
builder.append(", documents:").append(documents.get(0).get("_id"));
|
||||
if (documents.size() > 1) {
|
||||
builder.append("...").append(documents.size() - 1).append(" more");
|
||||
}
|
||||
} else {
|
||||
builder.append(", documents:").append(documents);
|
||||
}
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
public class InsertMessage implements MongoMessage
|
||||
{
|
||||
private static final long serialVersionUID = -3350216587439425208L;
|
||||
private MessageHeader messageHeader;
|
||||
private String fullCollectionName;
|
||||
private List<BSONObject> documents;
|
||||
|
||||
public InsertMessage() {
|
||||
this.messageHeader = new MessageHeader(OperationCode.OP_INSERT);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(final BSONOutputStream out) throws IOException {
|
||||
this.messageHeader.write(out);
|
||||
out.writeInteger(0);
|
||||
out.writeCString(this.fullCollectionName);
|
||||
if (null != this.documents && this.documents.size() > 0) {
|
||||
final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder();
|
||||
for (final BSONObject doc : this.documents) {
|
||||
out.write(encoder.encode(doc));
|
||||
}
|
||||
}
|
||||
out.setInteger(0, out.size());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void read(final BSONInputStream in) throws IOException {
|
||||
this.messageHeader.read(in);
|
||||
in.readInteger();
|
||||
this.fullCollectionName = in.readCString();
|
||||
while (in.available() > 0) {
|
||||
if (null == this.documents) {
|
||||
this.documents = new ArrayList<BSONObject>();
|
||||
}
|
||||
final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder();
|
||||
this.documents.add(decoder.readObject((InputStream)in));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public MessageHeader getMessageHeader() {
|
||||
return this.messageHeader;
|
||||
}
|
||||
|
||||
public String getFullCollectionName() {
|
||||
return this.fullCollectionName;
|
||||
}
|
||||
|
||||
public void setFullCollectionName(final String fullCollectionName) {
|
||||
this.fullCollectionName = fullCollectionName;
|
||||
}
|
||||
|
||||
public List<BSONObject> getDocuments() {
|
||||
return this.documents;
|
||||
}
|
||||
|
||||
public void setDocuments(final List<BSONObject> documents) {
|
||||
this.documents = documents;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("messageHeader:").append(this.messageHeader);
|
||||
builder.append(", fullCollectionName:").append(this.fullCollectionName);
|
||||
builder.append(", documents:").append(this.documents);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toPrettyString() {
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("messageHeader:").append(this.messageHeader);
|
||||
builder.append(", fullCollectionName:").append(this.fullCollectionName);
|
||||
if (null != this.documents && this.documents.size() > 0) {
|
||||
builder.append(", documents:").append(this.documents.get(0).get("_id"));
|
||||
if (this.documents.size() > 1) {
|
||||
builder.append("...").append(this.documents.size() - 1).append(" more");
|
||||
}
|
||||
}
|
||||
else {
|
||||
builder.append(", documents:").append(this.documents);
|
||||
}
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
}
|
||||
|
||||
+77
-90
@@ -1,96 +1,83 @@
|
||||
/**
|
||||
* JMongo is a mongodb driver writtern in java.
|
||||
* Copyright (C) 2010 Xiaohu Huang
|
||||
*
|
||||
* JMongo is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* JMongo is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package com.fangjia.datastore.mongo.protocol;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import com.fangjia.datastore.mongo.codec.BSONInputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
|
||||
import java.util.List;
|
||||
|
||||
import com.fangjia.datastore.mongo.codec.BSONInputStream;
|
||||
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
|
||||
|
||||
public class KillCursorsMessage implements MongoMessage {
|
||||
|
||||
private static final long serialVersionUID = -3350216587439425208L;
|
||||
private MessageHeader messageHeader;
|
||||
private int numberOfCursorIDs;
|
||||
private List<Long> cursorIDs;
|
||||
|
||||
public KillCursorsMessage() {
|
||||
messageHeader = new MessageHeader(OperationCode.OP_KILL_CURSORS);
|
||||
}
|
||||
|
||||
public MessageHeader getMessageHeader() {
|
||||
return messageHeader;
|
||||
}
|
||||
|
||||
public void write(BSONOutputStream out) throws IOException {
|
||||
messageHeader.write(out);
|
||||
out.writeInteger(0);
|
||||
out.writeInteger(numberOfCursorIDs);
|
||||
if (null != cursorIDs) {
|
||||
for (Long id : cursorIDs) {
|
||||
out.writeLong(id);
|
||||
}
|
||||
}
|
||||
out.set(0, out.size());
|
||||
}
|
||||
|
||||
public void read(BSONInputStream in) throws IOException {
|
||||
messageHeader.read(in);
|
||||
in.readInteger();
|
||||
numberOfCursorIDs = in.readInteger();
|
||||
while (in.available() > 0) {
|
||||
if (null == cursorIDs) {
|
||||
cursorIDs = new ArrayList<Long>();
|
||||
}
|
||||
cursorIDs.add(in.readLong());
|
||||
}
|
||||
}
|
||||
|
||||
public int getNumberOfCursorIDs() {
|
||||
return numberOfCursorIDs;
|
||||
}
|
||||
|
||||
public void setNumberOfCursorIDs(int numberOfCursorIDs) {
|
||||
this.numberOfCursorIDs = numberOfCursorIDs;
|
||||
}
|
||||
|
||||
public List<Long> getCursorIDs() {
|
||||
return cursorIDs;
|
||||
}
|
||||
|
||||
public void setCursorIDs(List<Long> cursorIDs) {
|
||||
this.cursorIDs = cursorIDs;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("messageHeader:").append(messageHeader);
|
||||
builder.append(", numberOfCursorIDs:").append(numberOfCursorIDs);
|
||||
builder.append(", cursorIDs:").append(cursorIDs);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
public String toPrettyString() {
|
||||
return toString();
|
||||
}
|
||||
|
||||
public class KillCursorsMessage implements MongoMessage
|
||||
{
|
||||
private static final long serialVersionUID = -3350216587439425208L;
|
||||
private MessageHeader messageHeader;
|
||||
private int numberOfCursorIDs;
|
||||
private List<Long> cursorIDs;
|
||||
|
||||
public KillCursorsMessage() {
|
||||
this.messageHeader = new MessageHeader(OperationCode.OP_KILL_CURSORS);
|
||||
}
|
||||
|
||||
@Override
|
||||
public MessageHeader getMessageHeader() {
|
||||
return this.messageHeader;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(final BSONOutputStream out) throws IOException {
|
||||
this.messageHeader.write(out);
|
||||
out.writeInteger(0);
|
||||
out.writeInteger(this.numberOfCursorIDs);
|
||||
if (null != this.cursorIDs) {
|
||||
for (final Long id : this.cursorIDs) {
|
||||
out.writeLong(id);
|
||||
}
|
||||
}
|
||||
out.set(0, out.size());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void read(final BSONInputStream in) throws IOException {
|
||||
this.messageHeader.read(in);
|
||||
in.readInteger();
|
||||
this.numberOfCursorIDs = in.readInteger();
|
||||
while (in.available() > 0) {
|
||||
if (null == this.cursorIDs) {
|
||||
this.cursorIDs = new ArrayList<Long>();
|
||||
}
|
||||
this.cursorIDs.add(in.readLong());
|
||||
}
|
||||
}
|
||||
|
||||
public int getNumberOfCursorIDs() {
|
||||
return this.numberOfCursorIDs;
|
||||
}
|
||||
|
||||
public void setNumberOfCursorIDs(final int numberOfCursorIDs) {
|
||||
this.numberOfCursorIDs = numberOfCursorIDs;
|
||||
}
|
||||
|
||||
public List<Long> getCursorIDs() {
|
||||
return this.cursorIDs;
|
||||
}
|
||||
|
||||
public void setCursorIDs(final List<Long> cursorIDs) {
|
||||
this.cursorIDs = cursorIDs;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("messageHeader:").append(this.messageHeader);
|
||||
builder.append(", numberOfCursorIDs:").append(this.numberOfCursorIDs);
|
||||
builder.append(", cursorIDs:").append(this.cursorIDs);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toPrettyString() {
|
||||
return this.toString();
|
||||
}
|
||||
}
|
||||
|
||||
+88
-101
@@ -1,106 +1,93 @@
|
||||
/**
|
||||
* JMongo is a mongodb driver writtern in java.
|
||||
* Copyright (C) 2010 Xiaohu Huang
|
||||
*
|
||||
* JMongo is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* JMongo is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package com.fangjia.datastore.mongo.protocol;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Serializable;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import com.fangjia.datastore.mongo.codec.BSONInputStream;
|
||||
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
|
||||
import java.io.IOException;
|
||||
import com.fangjia.datastore.mongo.codec.BSONInputStream;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.io.Serializable;
|
||||
|
||||
public final class MessageHeader implements Writable, Readable, Serializable {
|
||||
private static final long serialVersionUID = 8505270142383450097L;
|
||||
private static final AtomicInteger ID_GENERATOR = new AtomicInteger(1);
|
||||
|
||||
private int messageLength;
|
||||
private int requestID;
|
||||
private int responseTo;
|
||||
private OperationCode opCode;
|
||||
|
||||
public MessageHeader(OperationCode opCode) {
|
||||
this.opCode = opCode;
|
||||
this.requestID = ID_GENERATOR.getAndIncrement();
|
||||
}
|
||||
|
||||
public MessageHeader() {
|
||||
}
|
||||
|
||||
public void read(BSONInputStream in) throws IOException {
|
||||
messageLength = in.readInteger();
|
||||
requestID = in.readInteger();
|
||||
responseTo = in.readInteger();
|
||||
opCode = OperationCode.valueOf(in.readInteger());
|
||||
}
|
||||
|
||||
public void write(BSONOutputStream out) throws IOException {
|
||||
out.writeInteger(messageLength);
|
||||
out.writeInteger(requestID);
|
||||
out.writeInteger(responseTo);
|
||||
out.writeInteger(opCode.getValue());
|
||||
}
|
||||
|
||||
public int size() {
|
||||
return 16;
|
||||
}
|
||||
|
||||
public int getMessageLength() {
|
||||
return messageLength;
|
||||
}
|
||||
|
||||
public void setMessageLength(int messageLength) {
|
||||
this.messageLength = messageLength;
|
||||
}
|
||||
|
||||
public int getRequestID() {
|
||||
return requestID;
|
||||
}
|
||||
|
||||
public void setRequestID(int requestID) {
|
||||
this.requestID = requestID;
|
||||
}
|
||||
|
||||
public int getResponseTo() {
|
||||
return responseTo;
|
||||
}
|
||||
|
||||
public void setResponseTo(int responseTo) {
|
||||
this.responseTo = responseTo;
|
||||
}
|
||||
|
||||
public OperationCode getOpCode() {
|
||||
return opCode;
|
||||
}
|
||||
|
||||
public void setOpCode(OperationCode opCode) {
|
||||
this.opCode = opCode;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("messageLength:").append(messageLength);
|
||||
builder.append(", requestID:").append(requestID);
|
||||
builder.append(", responseTo:").append(responseTo);
|
||||
builder.append(", opCode:").append(opCode);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
public final class MessageHeader implements Writable, Readable, Serializable
|
||||
{
|
||||
private static final long serialVersionUID = 8505270142383450097L;
|
||||
private static final AtomicInteger ID_GENERATOR;
|
||||
private int messageLength;
|
||||
private int requestID;
|
||||
private int responseTo;
|
||||
private OperationCode opCode;
|
||||
|
||||
public MessageHeader(final OperationCode opCode) {
|
||||
this.opCode = opCode;
|
||||
this.requestID = MessageHeader.ID_GENERATOR.getAndIncrement();
|
||||
}
|
||||
|
||||
public MessageHeader() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void read(final BSONInputStream in) throws IOException {
|
||||
this.messageLength = in.readInteger();
|
||||
this.requestID = in.readInteger();
|
||||
this.responseTo = in.readInteger();
|
||||
this.opCode = OperationCode.valueOf(in.readInteger());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(final BSONOutputStream out) throws IOException {
|
||||
out.writeInteger(this.messageLength);
|
||||
out.writeInteger(this.requestID);
|
||||
out.writeInteger(this.responseTo);
|
||||
out.writeInteger(this.opCode.getValue());
|
||||
}
|
||||
|
||||
public int size() {
|
||||
return 16;
|
||||
}
|
||||
|
||||
public int getMessageLength() {
|
||||
return this.messageLength;
|
||||
}
|
||||
|
||||
public void setMessageLength(final int messageLength) {
|
||||
this.messageLength = messageLength;
|
||||
}
|
||||
|
||||
public int getRequestID() {
|
||||
return this.requestID;
|
||||
}
|
||||
|
||||
public void setRequestID(final int requestID) {
|
||||
this.requestID = requestID;
|
||||
}
|
||||
|
||||
public int getResponseTo() {
|
||||
return this.responseTo;
|
||||
}
|
||||
|
||||
public void setResponseTo(final int responseTo) {
|
||||
this.responseTo = responseTo;
|
||||
}
|
||||
|
||||
public OperationCode getOpCode() {
|
||||
return this.opCode;
|
||||
}
|
||||
|
||||
public void setOpCode(final OperationCode opCode) {
|
||||
this.opCode = opCode;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("messageLength:").append(this.messageLength);
|
||||
builder.append(", requestID:").append(this.requestID);
|
||||
builder.append(", responseTo:").append(this.responseTo);
|
||||
builder.append(", opCode:").append(this.opCode);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
static {
|
||||
ID_GENERATOR = new AtomicInteger(1);
|
||||
}
|
||||
}
|
||||
|
||||
+5
-21
@@ -1,26 +1,10 @@
|
||||
/**
|
||||
* JMongo is a mongodb driver writtern in java.
|
||||
* Copyright (C) 2010 Xiaohu Huang
|
||||
*
|
||||
* JMongo is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* JMongo is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package com.fangjia.datastore.mongo.protocol;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
public interface MongoMessage extends Readable, Writable, Serializable {
|
||||
public MessageHeader getMessageHeader();
|
||||
|
||||
public String toPrettyString();
|
||||
public interface MongoMessage extends Readable, Writable, Serializable
|
||||
{
|
||||
MessageHeader getMessageHeader();
|
||||
|
||||
String toPrettyString();
|
||||
}
|
||||
|
||||
+33
-76
@@ -1,82 +1,39 @@
|
||||
/**
|
||||
* JMongo is a mongodb driver writtern in java.
|
||||
* Copyright (C) 2010 Xiaohu Huang
|
||||
*
|
||||
* JMongo is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* JMongo is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package com.fangjia.datastore.mongo.protocol;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public enum OperationCode {
|
||||
/**
|
||||
* Reply to a client request. responseTo is set
|
||||
*/
|
||||
OP_REPLY(1),
|
||||
/**
|
||||
* generic msg command followed by a string
|
||||
*/
|
||||
OP_MSG(1000),
|
||||
/**
|
||||
* update document
|
||||
*/
|
||||
OP_UPDATE(2001),
|
||||
/**
|
||||
* insert new document
|
||||
*/
|
||||
OP_INSERT(2002),
|
||||
/**
|
||||
* formerly used for OP_GET_BY_OID
|
||||
*/
|
||||
RESERVED(2003),
|
||||
/**
|
||||
* query a collection
|
||||
*/
|
||||
OP_QUERY(2004),
|
||||
/**
|
||||
* Get more data from a query. See Cursors
|
||||
*/
|
||||
OP_GET_MORE(2005),
|
||||
/**
|
||||
* Delete documents
|
||||
*/
|
||||
OP_DELETE(2006),
|
||||
/**
|
||||
* Tell database client is done with a cursor
|
||||
*/
|
||||
OP_KILL_CURSORS(2007);
|
||||
|
||||
private final int value;
|
||||
|
||||
private final static Map<Integer, OperationCode> TYPE_MAP = new HashMap<Integer, OperationCode>();
|
||||
static {
|
||||
for (OperationCode e : OperationCode.values()) {
|
||||
TYPE_MAP.put(e.value, e);
|
||||
}
|
||||
}
|
||||
|
||||
private OperationCode(int value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public int getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
public static OperationCode valueOf(int value) {
|
||||
return TYPE_MAP.get(value);
|
||||
}
|
||||
|
||||
public enum OperationCode
|
||||
{
|
||||
OP_REPLY(1),
|
||||
OP_MSG(1000),
|
||||
OP_UPDATE(2001),
|
||||
OP_INSERT(2002),
|
||||
RESERVED(2003),
|
||||
OP_QUERY(2004),
|
||||
OP_GET_MORE(2005),
|
||||
OP_DELETE(2006),
|
||||
OP_KILL_CURSORS(2007);
|
||||
|
||||
private final int value;
|
||||
private static final Map<Integer, OperationCode> TYPE_MAP;
|
||||
|
||||
private OperationCode(final int value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public int getValue() {
|
||||
return this.value;
|
||||
}
|
||||
|
||||
public static OperationCode valueOf(final int value) {
|
||||
return OperationCode.TYPE_MAP.get(value);
|
||||
}
|
||||
|
||||
static {
|
||||
TYPE_MAP = new HashMap<Integer, OperationCode>();
|
||||
for (final OperationCode e : values()) {
|
||||
OperationCode.TYPE_MAP.put(e.value, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+192
-208
@@ -1,213 +1,197 @@
|
||||
/**
|
||||
* JMongo is a mongodb driver writtern in java.
|
||||
* Copyright (C) 2010 Xiaohu Huang
|
||||
*
|
||||
* JMongo is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* JMongo is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package com.fangjia.datastore.mongo.protocol;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.bson.BSONDecoder;
|
||||
import org.bson.BSONEncoder;
|
||||
import org.bson.BSONObject;
|
||||
import org.bson.BasicBSONDecoder;
|
||||
|
||||
import com.fangjia.datastore.mongo.codec.BSONInputStream;
|
||||
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
|
||||
import com.fangjia.datastore.utils.BSONUtils;
|
||||
import org.bson.BSONDecoder;
|
||||
import java.io.InputStream;
|
||||
import org.bson.BasicBSONDecoder;
|
||||
import com.fangjia.datastore.mongo.codec.BSONInputStream;
|
||||
import java.io.IOException;
|
||||
import org.bson.BSONEncoder;
|
||||
import org.bson.BasicBSONEncoder;
|
||||
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
|
||||
import org.bson.BSONObject;
|
||||
|
||||
public class QueryMessage implements MongoMessage {
|
||||
|
||||
private static final long serialVersionUID = -3350216587439425208L;
|
||||
private MessageHeader messageHeader;
|
||||
private int options;
|
||||
private int numberToSkip;
|
||||
private int numberToReturn;
|
||||
private String fullCollectionName;
|
||||
private BSONObject query;
|
||||
private BSONObject returnFieldSelector;
|
||||
|
||||
public QueryMessage() {
|
||||
messageHeader = new MessageHeader(OperationCode.OP_QUERY);
|
||||
}
|
||||
|
||||
public void write(BSONOutputStream out) throws IOException {
|
||||
messageHeader.write(out);
|
||||
out.writeInteger(options);
|
||||
out.writeCString(this.fullCollectionName);
|
||||
out.writeInteger(numberToSkip);
|
||||
out.writeInteger(numberToReturn);
|
||||
BSONEncoder encoder = new BSONEncoder();
|
||||
out.write(encoder.encode(query));
|
||||
if (null != returnFieldSelector) {
|
||||
out.write(encoder.encode(returnFieldSelector));
|
||||
}
|
||||
out.set(0, out.size());
|
||||
}
|
||||
|
||||
public void read(BSONInputStream in) throws IOException {
|
||||
messageHeader.read(in);
|
||||
options = in.readInteger();
|
||||
this.fullCollectionName = in.readCString();
|
||||
this.numberToSkip = in.readInteger();
|
||||
this.numberToReturn = in.readInteger();
|
||||
BSONDecoder decoder = new BasicBSONDecoder();
|
||||
query = decoder.readObject(in);
|
||||
if (in.available() > 0) {
|
||||
returnFieldSelector = decoder.readObject(in);
|
||||
}
|
||||
}
|
||||
|
||||
public MessageHeader getMessageHeader() {
|
||||
return messageHeader;
|
||||
}
|
||||
|
||||
public String getFullCollectionName() {
|
||||
return fullCollectionName;
|
||||
}
|
||||
|
||||
public void setFullCollectionName(String fullCollectionName) {
|
||||
this.fullCollectionName = fullCollectionName;
|
||||
}
|
||||
|
||||
public int getOptions() {
|
||||
return options;
|
||||
}
|
||||
|
||||
public void setOptions(int options) {
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
public void setNoCursorTimeout(boolean b) {
|
||||
this.options = (int) BSONUtils.bitSet(options, 4, b);
|
||||
}
|
||||
|
||||
public void setTailable(boolean b) {
|
||||
this.options = (int) BSONUtils.bitSet(options, 1, b);
|
||||
}
|
||||
|
||||
public void setSlaveOk(boolean b) {
|
||||
this.options = (int) BSONUtils.bitSet(options, 2, b);
|
||||
}
|
||||
|
||||
public void setAwaitData(boolean b) {
|
||||
this.options = (int) BSONUtils.bitSet(options, 5, b);
|
||||
}
|
||||
|
||||
public int getNumberToSkip() {
|
||||
return numberToSkip;
|
||||
}
|
||||
|
||||
public void setNumberToSkip(int numberToSkip) {
|
||||
this.numberToSkip = numberToSkip;
|
||||
}
|
||||
|
||||
public int getNumberToReturn() {
|
||||
return numberToReturn;
|
||||
}
|
||||
|
||||
public void setNumberToReturn(int numberToReturn) {
|
||||
this.numberToReturn = numberToReturn;
|
||||
}
|
||||
|
||||
public BSONObject getQuery() {
|
||||
return query;
|
||||
}
|
||||
|
||||
public void setQuery(BSONObject query) {
|
||||
this.query = query;
|
||||
}
|
||||
|
||||
public BSONObject getRealQuery() {
|
||||
if (null == query) {
|
||||
return null;
|
||||
}
|
||||
BSONObject q = (BSONObject) query.get("$query");
|
||||
if (null == q) {
|
||||
q = (BSONObject) query.get("query");
|
||||
if (null == q) {
|
||||
q = query;
|
||||
}
|
||||
}
|
||||
return q;
|
||||
}
|
||||
|
||||
public BSONObject getOrderByQuery() {
|
||||
if (null == query) {
|
||||
return null;
|
||||
}
|
||||
BSONObject q = (BSONObject) query.get("$orderby");
|
||||
if (null == q) {
|
||||
q = (BSONObject) query.get("orderby");
|
||||
}
|
||||
return q;
|
||||
}
|
||||
|
||||
public BSONObject getFacetQuery() {
|
||||
BSONObject q = getRealQuery();
|
||||
if (null == q) {
|
||||
return null;
|
||||
}
|
||||
return (BSONObject) q.get("$facet");
|
||||
}
|
||||
|
||||
public boolean isBulkQuery() {
|
||||
BSONObject q = getRealQuery();
|
||||
if (null == q) {
|
||||
return true;
|
||||
}
|
||||
Boolean bulk = (Boolean) q.get("$bulk");
|
||||
|
||||
return null == bulk ? true : bulk;
|
||||
}
|
||||
|
||||
public boolean isFromMaster() {
|
||||
BSONObject q = getRealQuery();
|
||||
if (null == q) {
|
||||
return false;
|
||||
}
|
||||
Boolean master = (Boolean) q.get("$master");
|
||||
|
||||
return null == master ? false : master;
|
||||
}
|
||||
|
||||
public BSONObject getReturnFieldSelector() {
|
||||
return returnFieldSelector;
|
||||
}
|
||||
|
||||
public void setReturnFieldSelector(BSONObject returnFieldSelector) {
|
||||
this.returnFieldSelector = returnFieldSelector;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("messageHeader:").append(messageHeader);
|
||||
builder.append(", options:").append(options);
|
||||
builder.append(", numberToSkip:").append(numberToSkip);
|
||||
builder.append(", numberToReturn:").append(numberToReturn);
|
||||
builder.append(", fullCollectionName:").append(fullCollectionName);
|
||||
builder.append(", query:").append(query);
|
||||
builder.append(", returnFieldSelector:").append(returnFieldSelector);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
public String toPrettyString() {
|
||||
return toString();
|
||||
}
|
||||
|
||||
public class QueryMessage implements MongoMessage
|
||||
{
|
||||
private static final long serialVersionUID = -3350216587439425208L;
|
||||
private MessageHeader messageHeader;
|
||||
private int options;
|
||||
private int numberToSkip;
|
||||
private int numberToReturn;
|
||||
private String fullCollectionName;
|
||||
private BSONObject query;
|
||||
private BSONObject returnFieldSelector;
|
||||
|
||||
public QueryMessage() {
|
||||
this.messageHeader = new MessageHeader(OperationCode.OP_QUERY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(final BSONOutputStream out) throws IOException {
|
||||
this.messageHeader.write(out);
|
||||
out.writeInteger(this.options);
|
||||
out.writeCString(this.fullCollectionName);
|
||||
out.writeInteger(this.numberToSkip);
|
||||
out.writeInteger(this.numberToReturn);
|
||||
final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder();
|
||||
out.write(encoder.encode(this.query));
|
||||
if (null != this.returnFieldSelector) {
|
||||
out.write(encoder.encode(this.returnFieldSelector));
|
||||
}
|
||||
out.set(0, out.size());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void read(final BSONInputStream in) throws IOException {
|
||||
this.messageHeader.read(in);
|
||||
this.options = in.readInteger();
|
||||
this.fullCollectionName = in.readCString();
|
||||
this.numberToSkip = in.readInteger();
|
||||
this.numberToReturn = in.readInteger();
|
||||
final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder();
|
||||
this.query = decoder.readObject((InputStream)in);
|
||||
if (in.available() > 0) {
|
||||
this.returnFieldSelector = decoder.readObject((InputStream)in);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public MessageHeader getMessageHeader() {
|
||||
return this.messageHeader;
|
||||
}
|
||||
|
||||
public String getFullCollectionName() {
|
||||
return this.fullCollectionName;
|
||||
}
|
||||
|
||||
public void setFullCollectionName(final String fullCollectionName) {
|
||||
this.fullCollectionName = fullCollectionName;
|
||||
}
|
||||
|
||||
public int getOptions() {
|
||||
return this.options;
|
||||
}
|
||||
|
||||
public void setOptions(final int options) {
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
public void setNoCursorTimeout(final boolean b) {
|
||||
this.options = (int)BSONUtils.bitSet(this.options, 4, b);
|
||||
}
|
||||
|
||||
public void setTailable(final boolean b) {
|
||||
this.options = (int)BSONUtils.bitSet(this.options, 1, b);
|
||||
}
|
||||
|
||||
public void setSlaveOk(final boolean b) {
|
||||
this.options = (int)BSONUtils.bitSet(this.options, 2, b);
|
||||
}
|
||||
|
||||
public void setAwaitData(final boolean b) {
|
||||
this.options = (int)BSONUtils.bitSet(this.options, 5, b);
|
||||
}
|
||||
|
||||
public int getNumberToSkip() {
|
||||
return this.numberToSkip;
|
||||
}
|
||||
|
||||
public void setNumberToSkip(final int numberToSkip) {
|
||||
this.numberToSkip = numberToSkip;
|
||||
}
|
||||
|
||||
public int getNumberToReturn() {
|
||||
return this.numberToReturn;
|
||||
}
|
||||
|
||||
public void setNumberToReturn(final int numberToReturn) {
|
||||
this.numberToReturn = numberToReturn;
|
||||
}
|
||||
|
||||
public BSONObject getQuery() {
|
||||
return this.query;
|
||||
}
|
||||
|
||||
public void setQuery(final BSONObject query) {
|
||||
this.query = query;
|
||||
}
|
||||
|
||||
public BSONObject getRealQuery() {
|
||||
if (null == this.query) {
|
||||
return null;
|
||||
}
|
||||
BSONObject q = (BSONObject)this.query.get("$query");
|
||||
if (null == q) {
|
||||
q = (BSONObject)this.query.get("query");
|
||||
if (null == q) {
|
||||
q = this.query;
|
||||
}
|
||||
}
|
||||
return q;
|
||||
}
|
||||
|
||||
public BSONObject getOrderByQuery() {
|
||||
if (null == this.query) {
|
||||
return null;
|
||||
}
|
||||
BSONObject q = (BSONObject)this.query.get("$orderby");
|
||||
if (null == q) {
|
||||
q = (BSONObject)this.query.get("orderby");
|
||||
}
|
||||
return q;
|
||||
}
|
||||
|
||||
public BSONObject getFacetQuery() {
|
||||
final BSONObject q = this.getRealQuery();
|
||||
if (null == q) {
|
||||
return null;
|
||||
}
|
||||
return (BSONObject)q.get("$facet");
|
||||
}
|
||||
|
||||
public boolean isBulkQuery() {
|
||||
final BSONObject q = this.getRealQuery();
|
||||
if (null == q) {
|
||||
return true;
|
||||
}
|
||||
final Boolean bulk = (Boolean)q.get("$bulk");
|
||||
return null == bulk || bulk;
|
||||
}
|
||||
|
||||
public boolean isFromMaster() {
|
||||
final BSONObject q = this.getRealQuery();
|
||||
if (null == q) {
|
||||
return false;
|
||||
}
|
||||
final Boolean master = (Boolean)q.get("$master");
|
||||
return null != master && master;
|
||||
}
|
||||
|
||||
public BSONObject getReturnFieldSelector() {
|
||||
return this.returnFieldSelector;
|
||||
}
|
||||
|
||||
public void setReturnFieldSelector(final BSONObject returnFieldSelector) {
|
||||
this.returnFieldSelector = returnFieldSelector;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("messageHeader:").append(this.messageHeader);
|
||||
builder.append(", options:").append(this.options);
|
||||
builder.append(", numberToSkip:").append(this.numberToSkip);
|
||||
builder.append(", numberToReturn:").append(this.numberToReturn);
|
||||
builder.append(", fullCollectionName:").append(this.fullCollectionName);
|
||||
builder.append(", query:").append(this.query);
|
||||
builder.append(", returnFieldSelector:").append(this.returnFieldSelector);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toPrettyString() {
|
||||
return this.toString();
|
||||
}
|
||||
}
|
||||
|
||||
+3
-20
@@ -1,26 +1,9 @@
|
||||
/**
|
||||
* JMongo is a mongodb driver writtern in java.
|
||||
* Copyright (C) 2010 Xiaohu Huang
|
||||
*
|
||||
* JMongo is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* JMongo is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package com.fangjia.datastore.mongo.protocol;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import com.fangjia.datastore.mongo.codec.BSONInputStream;
|
||||
|
||||
public interface Readable {
|
||||
public void read(BSONInputStream in) throws IOException;
|
||||
public interface Readable
|
||||
{
|
||||
void read(final BSONInputStream p0) throws IOException;
|
||||
}
|
||||
|
||||
+119
-132
@@ -1,138 +1,125 @@
|
||||
/**
|
||||
* JMongo is a mongodb driver writtern in java.
|
||||
* Copyright (C) 2010 Xiaohu Huang
|
||||
*
|
||||
* JMongo is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* JMongo is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package com.fangjia.datastore.mongo.protocol;
|
||||
|
||||
import java.io.IOException;
|
||||
import org.bson.BSONDecoder;
|
||||
import java.io.InputStream;
|
||||
import org.bson.BasicBSONDecoder;
|
||||
import java.util.ArrayList;
|
||||
import com.fangjia.datastore.mongo.codec.BSONInputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
import org.bson.BSONEncoder;
|
||||
import org.bson.BasicBSONEncoder;
|
||||
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
|
||||
import org.bson.BSONObject;
|
||||
import java.util.List;
|
||||
|
||||
import org.bson.BSONDecoder;
|
||||
import org.bson.BSONEncoder;
|
||||
import org.bson.BSONObject;
|
||||
import org.bson.BasicBSONDecoder;
|
||||
|
||||
import com.fangjia.datastore.mongo.codec.BSONInputStream;
|
||||
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
|
||||
|
||||
public class ReplyMessage implements MongoMessage {
|
||||
|
||||
private static final long serialVersionUID = -3350216587439425208L;
|
||||
private MessageHeader messageHeader;
|
||||
private int responseFlag;
|
||||
private long cursorID;
|
||||
private int startingFrom;
|
||||
private int numberReturned;
|
||||
private List<BSONObject> documents;
|
||||
|
||||
public ReplyMessage() {
|
||||
messageHeader = new MessageHeader(OperationCode.OP_REPLY);
|
||||
}
|
||||
|
||||
public void write(BSONOutputStream out) throws IOException {
|
||||
messageHeader.write(out);
|
||||
out.writeInteger(responseFlag);
|
||||
out.writeLong(cursorID);
|
||||
out.writeInteger(startingFrom);
|
||||
out.writeInteger(numberReturned);
|
||||
if (null != this.documents) {
|
||||
BSONEncoder encoder = new BSONEncoder();
|
||||
for (BSONObject doc : this.documents) {
|
||||
out.write(encoder.encode(doc));
|
||||
}
|
||||
}
|
||||
out.set(0, out.size());
|
||||
}
|
||||
|
||||
public void read(BSONInputStream in) throws IOException {
|
||||
messageHeader.read(in);
|
||||
responseFlag = in.readInteger();
|
||||
cursorID = in.readLong();
|
||||
startingFrom = in.readInteger();
|
||||
numberReturned = in.readInteger();
|
||||
documents = new ArrayList<BSONObject>(numberReturned);
|
||||
if (numberReturned > 0) {
|
||||
BSONDecoder decoder = new BasicBSONDecoder();
|
||||
for (int i = 0; i < numberReturned; i++) {
|
||||
documents.add(decoder.readObject(in));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public MessageHeader getMessageHeader() {
|
||||
return messageHeader;
|
||||
}
|
||||
|
||||
public int getResponseFlag() {
|
||||
return responseFlag;
|
||||
}
|
||||
|
||||
public void setResponseFlag(int responseFlag) {
|
||||
this.responseFlag = responseFlag;
|
||||
}
|
||||
|
||||
public long getCursorID() {
|
||||
return cursorID;
|
||||
}
|
||||
|
||||
public void setCursorID(long cursorID) {
|
||||
this.cursorID = cursorID;
|
||||
}
|
||||
|
||||
public int getStartingFrom() {
|
||||
return startingFrom;
|
||||
}
|
||||
|
||||
public void setStartingFrom(int startingFrom) {
|
||||
this.startingFrom = startingFrom;
|
||||
}
|
||||
|
||||
public int getNumberReturned() {
|
||||
return numberReturned;
|
||||
}
|
||||
|
||||
public void setNumberReturned(int numberReturned) {
|
||||
this.numberReturned = numberReturned;
|
||||
}
|
||||
|
||||
public List<BSONObject> getDocuments() {
|
||||
return documents;
|
||||
}
|
||||
|
||||
public void setDocuments(List<BSONObject> documents) {
|
||||
this.documents = documents;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("messageHeader:").append(messageHeader);
|
||||
builder.append(", responseFlag:").append(responseFlag);
|
||||
builder.append(", cursorID:").append(cursorID);
|
||||
builder.append(", startingFrom:").append(startingFrom);
|
||||
builder.append(", numberReturned:").append(numberReturned);
|
||||
builder.append(", documents:").append(documents);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
public String toPrettyString() {
|
||||
return toString();
|
||||
}
|
||||
|
||||
public class ReplyMessage implements MongoMessage
|
||||
{
|
||||
private static final long serialVersionUID = -3350216587439425208L;
|
||||
private MessageHeader messageHeader;
|
||||
private int responseFlag;
|
||||
private long cursorID;
|
||||
private int startingFrom;
|
||||
private int numberReturned;
|
||||
private List<BSONObject> documents;
|
||||
|
||||
public ReplyMessage() {
|
||||
this.messageHeader = new MessageHeader(OperationCode.OP_REPLY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(final BSONOutputStream out) throws IOException {
|
||||
this.messageHeader.write(out);
|
||||
out.writeInteger(this.responseFlag);
|
||||
out.writeLong(this.cursorID);
|
||||
out.writeInteger(this.startingFrom);
|
||||
out.writeInteger(this.numberReturned);
|
||||
if (null != this.documents) {
|
||||
final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder();
|
||||
for (final BSONObject doc : this.documents) {
|
||||
out.write(encoder.encode(doc));
|
||||
}
|
||||
}
|
||||
out.set(0, out.size());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void read(final BSONInputStream in) throws IOException {
|
||||
this.messageHeader.read(in);
|
||||
this.responseFlag = in.readInteger();
|
||||
this.cursorID = in.readLong();
|
||||
this.startingFrom = in.readInteger();
|
||||
this.numberReturned = in.readInteger();
|
||||
this.documents = new ArrayList<BSONObject>(this.numberReturned);
|
||||
if (this.numberReturned > 0) {
|
||||
final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder();
|
||||
for (int i = 0; i < this.numberReturned; ++i) {
|
||||
this.documents.add(decoder.readObject((InputStream)in));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public MessageHeader getMessageHeader() {
|
||||
return this.messageHeader;
|
||||
}
|
||||
|
||||
public int getResponseFlag() {
|
||||
return this.responseFlag;
|
||||
}
|
||||
|
||||
public void setResponseFlag(final int responseFlag) {
|
||||
this.responseFlag = responseFlag;
|
||||
}
|
||||
|
||||
public long getCursorID() {
|
||||
return this.cursorID;
|
||||
}
|
||||
|
||||
public void setCursorID(final long cursorID) {
|
||||
this.cursorID = cursorID;
|
||||
}
|
||||
|
||||
public int getStartingFrom() {
|
||||
return this.startingFrom;
|
||||
}
|
||||
|
||||
public void setStartingFrom(final int startingFrom) {
|
||||
this.startingFrom = startingFrom;
|
||||
}
|
||||
|
||||
public int getNumberReturned() {
|
||||
return this.numberReturned;
|
||||
}
|
||||
|
||||
public void setNumberReturned(final int numberReturned) {
|
||||
this.numberReturned = numberReturned;
|
||||
}
|
||||
|
||||
public List<BSONObject> getDocuments() {
|
||||
return this.documents;
|
||||
}
|
||||
|
||||
public void setDocuments(final List<BSONObject> documents) {
|
||||
this.documents = documents;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("messageHeader:").append(this.messageHeader);
|
||||
builder.append(", responseFlag:").append(this.responseFlag);
|
||||
builder.append(", cursorID:").append(this.cursorID);
|
||||
builder.append(", startingFrom:").append(this.startingFrom);
|
||||
builder.append(", numberReturned:").append(this.numberReturned);
|
||||
builder.append(", documents:").append(this.documents);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toPrettyString() {
|
||||
return this.toString();
|
||||
}
|
||||
}
|
||||
|
||||
+118
-132
@@ -1,137 +1,123 @@
|
||||
/**
|
||||
* JMongo is a mongodb driver writtern in java.
|
||||
* Copyright (C) 2010 Xiaohu Huang
|
||||
*
|
||||
* JMongo is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* JMongo is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package com.fangjia.datastore.mongo.protocol;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.bson.BSONDecoder;
|
||||
import org.bson.BSONEncoder;
|
||||
import org.bson.BSONObject;
|
||||
import org.bson.BasicBSONDecoder;
|
||||
import org.bson.BasicBSONObject;
|
||||
|
||||
import com.fangjia.datastore.mongo.codec.BSONInputStream;
|
||||
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
|
||||
import com.fangjia.datastore.utils.BSONUtils;
|
||||
import org.bson.BSONDecoder;
|
||||
import java.io.InputStream;
|
||||
import org.bson.BasicBSONDecoder;
|
||||
import com.fangjia.datastore.mongo.codec.BSONInputStream;
|
||||
import java.io.IOException;
|
||||
import org.bson.BSONEncoder;
|
||||
import org.bson.BasicBSONEncoder;
|
||||
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
|
||||
import org.bson.BasicBSONObject;
|
||||
import org.bson.BSONObject;
|
||||
|
||||
public class UpdateMessage implements MongoMessage {
|
||||
|
||||
private static final long serialVersionUID = -5278293474222136247L;
|
||||
|
||||
private MessageHeader messageHeader;
|
||||
private String fullCollectionName;
|
||||
private int flags;
|
||||
private BSONObject selector;
|
||||
private BSONObject update;
|
||||
|
||||
public UpdateMessage() {
|
||||
messageHeader = new MessageHeader(OperationCode.OP_UPDATE);
|
||||
this.selector = new BasicBSONObject();
|
||||
this.update = new BasicBSONObject();
|
||||
}
|
||||
|
||||
public void write(BSONOutputStream out) throws IOException {
|
||||
messageHeader.write(out);
|
||||
out.writeInteger(0);
|
||||
out.writeCString(this.fullCollectionName);
|
||||
out.writeInteger(flags);
|
||||
BSONEncoder encoder = new BSONEncoder();
|
||||
out.write(encoder.encode(selector));
|
||||
out.write(encoder.encode(update));
|
||||
out.set(0, out.size());
|
||||
}
|
||||
|
||||
public void read(BSONInputStream in) throws IOException {
|
||||
messageHeader.read(in);
|
||||
in.readInteger();
|
||||
this.fullCollectionName = in.readCString();
|
||||
this.flags = in.readInteger();
|
||||
BSONDecoder decoder = new BasicBSONDecoder();
|
||||
selector = decoder.readObject(in);
|
||||
update = decoder.readObject(in);
|
||||
}
|
||||
|
||||
public void setUpsert(boolean upsert) {
|
||||
this.flags = (int) BSONUtils.bitSet(this.flags, 0, upsert);
|
||||
}
|
||||
|
||||
public boolean isUpsert() {
|
||||
return BSONUtils.isBitSet(flags, 0);
|
||||
}
|
||||
|
||||
public void setMultiUpdate(boolean multiUpdate) {
|
||||
this.flags = (int) BSONUtils.bitSet(this.flags, 1, multiUpdate);
|
||||
}
|
||||
|
||||
public boolean isMultiUpdate() {
|
||||
return BSONUtils.isBitSet(flags, 1);
|
||||
}
|
||||
|
||||
public MessageHeader getMessageHeader() {
|
||||
return messageHeader;
|
||||
}
|
||||
|
||||
public String getFullCollectionName() {
|
||||
return fullCollectionName;
|
||||
}
|
||||
|
||||
public void setFullCollectionName(String fullCollectionName) {
|
||||
this.fullCollectionName = fullCollectionName;
|
||||
}
|
||||
|
||||
public int getFlags() {
|
||||
return flags;
|
||||
}
|
||||
|
||||
public void setFlags(int flags) {
|
||||
this.flags = flags;
|
||||
}
|
||||
|
||||
public BSONObject getSelector() {
|
||||
return selector;
|
||||
}
|
||||
|
||||
public void setSelector(BSONObject selector) {
|
||||
this.selector = selector;
|
||||
}
|
||||
|
||||
public BSONObject getUpdate() {
|
||||
return update;
|
||||
}
|
||||
|
||||
public void setUpdate(BSONObject update) {
|
||||
this.update = update;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("messageHeader:").append(messageHeader);
|
||||
builder.append(", fullCollectionName:").append(fullCollectionName);
|
||||
builder.append(", flags:").append(flags);
|
||||
builder.append(", selector:").append(selector);
|
||||
builder.append(", update:").append(update);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
public String toPrettyString() {
|
||||
return toString();
|
||||
}
|
||||
|
||||
public class UpdateMessage implements MongoMessage
|
||||
{
|
||||
private static final long serialVersionUID = -5278293474222136247L;
|
||||
private MessageHeader messageHeader;
|
||||
private String fullCollectionName;
|
||||
private int flags;
|
||||
private BSONObject selector;
|
||||
private BSONObject update;
|
||||
|
||||
public UpdateMessage() {
|
||||
this.messageHeader = new MessageHeader(OperationCode.OP_UPDATE);
|
||||
this.selector = (BSONObject)new BasicBSONObject();
|
||||
this.update = (BSONObject)new BasicBSONObject();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(final BSONOutputStream out) throws IOException {
|
||||
this.messageHeader.write(out);
|
||||
out.writeInteger(0);
|
||||
out.writeCString(this.fullCollectionName);
|
||||
out.writeInteger(this.flags);
|
||||
final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder();
|
||||
out.write(encoder.encode(this.selector));
|
||||
out.write(encoder.encode(this.update));
|
||||
out.set(0, out.size());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void read(final BSONInputStream in) throws IOException {
|
||||
this.messageHeader.read(in);
|
||||
in.readInteger();
|
||||
this.fullCollectionName = in.readCString();
|
||||
this.flags = in.readInteger();
|
||||
final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder();
|
||||
this.selector = decoder.readObject((InputStream)in);
|
||||
this.update = decoder.readObject((InputStream)in);
|
||||
}
|
||||
|
||||
public void setUpsert(final boolean upsert) {
|
||||
this.flags = (int)BSONUtils.bitSet(this.flags, 0, upsert);
|
||||
}
|
||||
|
||||
public boolean isUpsert() {
|
||||
return BSONUtils.isBitSet(this.flags, 0);
|
||||
}
|
||||
|
||||
public void setMultiUpdate(final boolean multiUpdate) {
|
||||
this.flags = (int)BSONUtils.bitSet(this.flags, 1, multiUpdate);
|
||||
}
|
||||
|
||||
public boolean isMultiUpdate() {
|
||||
return BSONUtils.isBitSet(this.flags, 1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public MessageHeader getMessageHeader() {
|
||||
return this.messageHeader;
|
||||
}
|
||||
|
||||
public String getFullCollectionName() {
|
||||
return this.fullCollectionName;
|
||||
}
|
||||
|
||||
public void setFullCollectionName(final String fullCollectionName) {
|
||||
this.fullCollectionName = fullCollectionName;
|
||||
}
|
||||
|
||||
public int getFlags() {
|
||||
return this.flags;
|
||||
}
|
||||
|
||||
public void setFlags(final int flags) {
|
||||
this.flags = flags;
|
||||
}
|
||||
|
||||
public BSONObject getSelector() {
|
||||
return this.selector;
|
||||
}
|
||||
|
||||
public void setSelector(final BSONObject selector) {
|
||||
this.selector = selector;
|
||||
}
|
||||
|
||||
public BSONObject getUpdate() {
|
||||
return this.update;
|
||||
}
|
||||
|
||||
public void setUpdate(final BSONObject update) {
|
||||
this.update = update;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
builder.append("messageHeader:").append(this.messageHeader);
|
||||
builder.append(", fullCollectionName:").append(this.fullCollectionName);
|
||||
builder.append(", flags:").append(this.flags);
|
||||
builder.append(", selector:").append(this.selector);
|
||||
builder.append(", update:").append(this.update);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toPrettyString() {
|
||||
return this.toString();
|
||||
}
|
||||
}
|
||||
|
||||
+3
-20
@@ -1,26 +1,9 @@
|
||||
/**
|
||||
* JMongo is a mongodb driver writtern in java.
|
||||
* Copyright (C) 2010 Xiaohu Huang
|
||||
*
|
||||
* JMongo is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* JMongo is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package com.fangjia.datastore.mongo.protocol;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
|
||||
|
||||
public interface Writable {
|
||||
public void write(BSONOutputStream out) throws IOException;
|
||||
public interface Writable
|
||||
{
|
||||
void write(final BSONOutputStream p0) throws IOException;
|
||||
}
|
||||
|
||||
+53
-62
@@ -1,11 +1,7 @@
|
||||
package com.fangjia.datastore.solr;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.HttpStatus;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
@@ -15,66 +11,61 @@ import org.apache.solr.common.util.NamedList;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class CommonsHttpSolrServer extends HttpSolrServer {
|
||||
private static final long serialVersionUID = -3881911436089005421L;
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(CommonsHttpSolrServer.class);
|
||||
public class CommonsHttpSolrServer extends HttpSolrServer
|
||||
{
|
||||
private static final long serialVersionUID = -3881911436089005421L;
|
||||
private static final Logger LOGGER;
|
||||
private final HttpSolrClient solrServer;
|
||||
private final String host;
|
||||
private final String coreName;
|
||||
|
||||
public CommonsHttpSolrServer(final String host, final String coreName, final HttpClient httpClient) throws MalformedURLException {
|
||||
this.host = host;
|
||||
this.coreName = coreName;
|
||||
(this.solrServer = new HttpSolrClient("http://" + host + '/' + coreName)).setAllowCompression(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return this.host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getCoreName() {
|
||||
return this.coreName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reloadCore() throws SolrException {
|
||||
try {
|
||||
final HttpGet method = new HttpGet("http://" + this.host + "/admin/cores?action=RELOAD&core=" + this.coreName);
|
||||
final HttpResponse response = this.solrServer.getHttpClient().execute(method);
|
||||
if (response.getStatusLine().getStatusCode() != 200) {
|
||||
throw new SolrException("[" + this.coreName + "] error reload core of solr[" + this.host + "]: " + IOUtils.toString(response.getEntity().getContent()));
|
||||
}
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("[" + this.coreName + "] solr core of server[" + this.host + "] reloaded.");
|
||||
}
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("[" + this.coreName + "] core reload success");
|
||||
}
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new SolrException("[" + this.coreName + "] error reload core", e);
|
||||
}
|
||||
}
|
||||
|
||||
private final HttpSolrClient solrServer;
|
||||
private final String host;
|
||||
private final String coreName;
|
||||
public NamedList<Object> request(final SolrRequest request, final String arg1) throws SolrServerException, IOException {
|
||||
return this.solrServer.request(request);
|
||||
}
|
||||
|
||||
public CommonsHttpSolrServer(String host, String coreName, HttpClient httpClient) throws MalformedURLException {
|
||||
this.host = host;
|
||||
this.coreName = coreName;
|
||||
this.solrServer = new HttpSolrClient("http://" + host + '/' + coreName);
|
||||
this.solrServer.setAllowCompression(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getCoreName() {
|
||||
return coreName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reloadCore() throws SolrException {
|
||||
try {
|
||||
HttpGet method = new HttpGet("http://" + host + "/admin/cores?action=RELOAD&core=" + this.coreName);
|
||||
HttpResponse response = solrServer.getHttpClient().execute(method);
|
||||
if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
|
||||
throw new SolrException("[" + this.coreName + "] error reload core of solr[" + host + "]: "
|
||||
+ IOUtils.toString(response.getEntity().getContent()));
|
||||
} else if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("[" + this.coreName + "] solr core of server[" + host + "] reloaded.");
|
||||
}
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("[" + this.coreName + "] core reload success");
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new SolrException("[" + this.coreName + "] error reload core", e);
|
||||
}
|
||||
}
|
||||
|
||||
/*@Override
|
||||
public NamedList<Object> request(SolrRequest request) throws SolrServerException, IOException {
|
||||
return solrServer.request(request);
|
||||
|
||||
}*/
|
||||
|
||||
@Override
|
||||
public NamedList<Object> request(SolrRequest request, String arg1)
|
||||
throws SolrServerException, IOException {
|
||||
return solrServer.request(request);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void shutdown() {
|
||||
|
||||
}
|
||||
public void shutdown() {
|
||||
}
|
||||
|
||||
static {
|
||||
LOGGER = LoggerFactory.getLogger(CommonsHttpSolrServer.class);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,138 +1,32 @@
|
||||
package com.fangjia.datastore.solr;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang.time.FastDateFormat;
|
||||
import org.bson.BSONObject;
|
||||
import org.bson.types.ObjectId;
|
||||
|
||||
import com.fangjia.datastore.QueryResponse;
|
||||
import org.bson.types.ObjectId;
|
||||
import java.util.List;
|
||||
import org.bson.BSONObject;
|
||||
import java.util.Iterator;
|
||||
import java.util.Collection;
|
||||
import org.apache.commons.lang.time.FastDateFormat;
|
||||
import com.fangjia.datastore.config.Reloadable;
|
||||
|
||||
/**
|
||||
* 一个Core对应一个solr索引
|
||||
*
|
||||
* @author velna
|
||||
*
|
||||
*/
|
||||
public interface Core extends Reloadable {
|
||||
|
||||
public final static int INDEX_BATCH_SIZE = 30000;
|
||||
public final static FastDateFormat VERSION_DATE_FORMAT = FastDateFormat.getInstance("yyMMddHHmmssSSS");
|
||||
|
||||
/**
|
||||
* 得到索引名称
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public String getName();
|
||||
|
||||
/**
|
||||
* 得到所有索引字段
|
||||
*
|
||||
* @return
|
||||
* @throws SolrException
|
||||
*/
|
||||
Collection<Index> getIndexes() throws SolrException;
|
||||
|
||||
/**
|
||||
* 添加索引字段,必要的话重建索引
|
||||
*
|
||||
* @param indexes
|
||||
* 添加的索引字段列表
|
||||
* @param iterator
|
||||
* 用于遍历数据
|
||||
* @throws SolrException
|
||||
*/
|
||||
void ensureIndex(Collection<Index> indexes, Iterator<? extends BSONObject> iterator) throws SolrException;
|
||||
|
||||
/**
|
||||
* 添加索引字段,必要的话重建索引
|
||||
*
|
||||
* @param index
|
||||
* 添加的索引字段列表
|
||||
* @param iterator
|
||||
* 用于遍历数据
|
||||
* @throws SolrException
|
||||
*/
|
||||
void ensureIndex(Index index, Iterator<? extends BSONObject> iterator) throws SolrException;
|
||||
|
||||
/**
|
||||
* 重建索引
|
||||
*
|
||||
* @param objIdList
|
||||
* 待重建的ObjIdList,全部重建时为null
|
||||
* @param iterator
|
||||
* 用于遍历数据
|
||||
*/
|
||||
void reIndex(List<ObjectId> objIdList, Iterator<? extends BSONObject> iterator) throws SolrException;
|
||||
|
||||
/**
|
||||
* 保存索引文档
|
||||
*
|
||||
* @param doc
|
||||
* @throws SolrException
|
||||
*/
|
||||
// void save(BSONObject doc) throws SolrException;
|
||||
/**
|
||||
* 保存多个索引文档
|
||||
*
|
||||
* @param docList
|
||||
* @throws SolrException
|
||||
*/
|
||||
// void save(List<? extends BSONObject> docList) throws SolrException;
|
||||
/**
|
||||
* 保存多个索引文档
|
||||
*
|
||||
* @param iterator
|
||||
* @throws SolrException
|
||||
*/
|
||||
void save(Iterator<? extends BSONObject> iterator) throws SolrException;
|
||||
|
||||
/**
|
||||
* 根据ID删除索引文档
|
||||
*
|
||||
* @param id
|
||||
* @throws SolrException
|
||||
*/
|
||||
// void deleteById(String id) throws SolrException;
|
||||
/**
|
||||
* 根据ID列表删除索引文档
|
||||
*
|
||||
* @param ids
|
||||
* @throws SolrException
|
||||
*/
|
||||
// void deleteById(List<String> ids) throws SolrException;
|
||||
/**
|
||||
* 根据查询删除索引文档
|
||||
*
|
||||
* @param query
|
||||
* @throws SolrException
|
||||
*/
|
||||
void deleteByQuery(BSONObject query) throws SolrException;
|
||||
|
||||
/**
|
||||
* 查询索引
|
||||
*
|
||||
* @param query
|
||||
* 查询条件
|
||||
* @param sort
|
||||
* 排序条件
|
||||
* @param facet
|
||||
* facet查询条件
|
||||
* @param skip
|
||||
* 从第skip条开始返回结果
|
||||
* @param limit
|
||||
* 返回limit条结果
|
||||
* @param fromMaster
|
||||
* 是否从主索引中读取数据
|
||||
* @param bulk
|
||||
* 是否使用bulk查询
|
||||
* @return
|
||||
* @throws SolrException
|
||||
*/
|
||||
QueryResponse find(BSONObject query, BSONObject sort, BSONObject facet, int skip, int limit, boolean fromMaster, boolean bulk)
|
||||
throws SolrException;
|
||||
public interface Core extends Reloadable
|
||||
{
|
||||
public static final int INDEX_BATCH_SIZE = 30000;
|
||||
public static final FastDateFormat VERSION_DATE_FORMAT = FastDateFormat.getInstance("yyMMddHHmmssSSS");
|
||||
|
||||
String getName();
|
||||
|
||||
Collection<Index> getIndexes() throws SolrException;
|
||||
|
||||
void ensureIndex(final Collection<Index> p0, final Iterator<? extends BSONObject> p1) throws SolrException;
|
||||
|
||||
void ensureIndex(final Index p0, final Iterator<? extends BSONObject> p1) throws SolrException;
|
||||
|
||||
void reIndex(final List<ObjectId> p0, final Iterator<? extends BSONObject> p1) throws SolrException;
|
||||
|
||||
void save(final Iterator<? extends BSONObject> p0) throws SolrException;
|
||||
|
||||
void deleteByQuery(final BSONObject p0) throws SolrException;
|
||||
|
||||
QueryResponse find(final BSONObject p0, final BSONObject p1, final BSONObject p2, final int p3, final int p4, final boolean p5, final boolean p6) throws SolrException;
|
||||
}
|
||||
|
||||
+465
-470
@@ -1,485 +1,480 @@
|
||||
package com.fangjia.datastore.solr;
|
||||
|
||||
import com.fangjia.datastore.utils.LoggerFactory;
|
||||
import com.fangjia.datastore.TriggerExecuteException;
|
||||
import com.fangjia.datastore.config.Trigger;
|
||||
import com.fangjia.datastore.TriggerExecutor;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.client.solrj.response.FacetField;
|
||||
import org.apache.solr.common.SolrDocument;
|
||||
import java.util.ArrayList;
|
||||
import com.fangjia.datastore.QueryResponse;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import com.mongodb.BasicDBObject;
|
||||
import org.apache.solr.client.solrj.SolrQuery;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.bson.types.ObjectId;
|
||||
import java.util.List;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.Executor;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.solr.client.solrj.SolrQuery;
|
||||
import org.apache.solr.client.solrj.SolrQuery.ORDER;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.SpatialParams;
|
||||
import org.bson.BSONObject;
|
||||
import org.bson.types.ObjectId;
|
||||
import org.slf4j.Logger;
|
||||
import org.sothis.core.util.BlockingThreadPoolExecutor;
|
||||
|
||||
import com.fangjia.datastore.ConfigurationException;
|
||||
import com.fangjia.datastore.QueryResponse;
|
||||
import com.fangjia.datastore.TriggerExecuteException;
|
||||
import com.fangjia.datastore.TriggerExecutor;
|
||||
import com.fangjia.datastore.config.ServerConfig;
|
||||
import com.fangjia.datastore.config.StoreConfig;
|
||||
import com.fangjia.datastore.config.Trigger;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import org.sothis.core.util.BlockingThreadPoolExecutor;
|
||||
import java.util.concurrent.Executor;
|
||||
import org.bson.BSONObject;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import com.fangjia.datastore.utils.AsyncOperation;
|
||||
import com.fangjia.datastore.utils.LoggerFactory;
|
||||
import com.mongodb.BasicDBObject;
|
||||
import com.fangjia.datastore.config.StoreConfig;
|
||||
import com.fangjia.datastore.config.ServerConfig;
|
||||
import org.slf4j.Logger;
|
||||
|
||||
public class CoreImpl implements Core {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(CoreImpl.class);
|
||||
private final static Logger PERFORMANCE_LOGGER = LoggerFactory.getPerformanceLogger(CoreImpl.class);
|
||||
private final static int TRIIGER_POOL_SIZE = Integer.getInteger("datastore.triggerPoolSize", 10);
|
||||
public class CoreImpl implements Core
|
||||
{
|
||||
private static final Logger LOGGER;
|
||||
private static final Logger PERFORMANCE_LOGGER;
|
||||
private static final int TRIIGER_POOL_SIZE;
|
||||
private final String name;
|
||||
private final SolrServer solrServer;
|
||||
private final ServerConfig serverConfig;
|
||||
private final StoreConfig storeConfig;
|
||||
private final AsyncOperation asynchOperation;
|
||||
private final BlockingQueue<BSONObject> solrAddQueue;
|
||||
private final BlockingQueue<BSONObject> triggerQueue;
|
||||
private final Executor triggerExecutorService;
|
||||
|
||||
public CoreImpl(final StoreConfig storeConfig) throws ConfigurationException {
|
||||
this.name = storeConfig.getName();
|
||||
this.storeConfig = storeConfig;
|
||||
this.solrServer = new SolrServerImpl(this.storeConfig);
|
||||
this.serverConfig = storeConfig.getDataStoreConfig().getServerConfig();
|
||||
if (CoreImpl.LOGGER.isInfoEnabled()) {
|
||||
CoreImpl.LOGGER.info("init solr core: " + this.name + ", storeConfig: " + storeConfig);
|
||||
}
|
||||
this.asynchOperation = AsyncOperation.getInstance();
|
||||
this.triggerExecutorService = new BlockingThreadPoolExecutor(this.name + "-Trigger", 1, CoreImpl.TRIIGER_POOL_SIZE, CoreImpl.TRIIGER_POOL_SIZE, 60000L);
|
||||
this.solrAddQueue = new LinkedBlockingQueue<BSONObject>(60000);
|
||||
final Thread solrIndexAddThread = new Thread(new SolrAddRunnable(), this.name + "-SolrAdd");
|
||||
solrIndexAddThread.setDaemon(true);
|
||||
solrIndexAddThread.start();
|
||||
this.triggerQueue = new LinkedBlockingQueue<BSONObject>(this.serverConfig.getTriggerBatchSize() * 4);
|
||||
final Thread triggerExecuteThread = new Thread(new TriggerRunnable(), this.name + "-Trigger");
|
||||
triggerExecuteThread.setDaemon(true);
|
||||
triggerExecuteThread.start();
|
||||
Runtime.getRuntime().addShutdownHook(new Thread() {
|
||||
@Override
|
||||
public void run() {
|
||||
while (!CoreImpl.this.solrAddQueue.isEmpty()) {
|
||||
CoreImpl.LOGGER.info(CoreImpl.this.name + ": wait for solr queue, current size: " + CoreImpl.this.solrAddQueue.size());
|
||||
try {
|
||||
Thread.sleep(1000L);
|
||||
}
|
||||
catch (InterruptedException e) {
|
||||
CoreImpl.LOGGER.error("", (Throwable)e);
|
||||
}
|
||||
}
|
||||
CoreImpl.LOGGER.info(CoreImpl.this.name + ": shutdown.");
|
||||
}
|
||||
});
|
||||
if (CoreImpl.LOGGER.isInfoEnabled()) {
|
||||
CoreImpl.LOGGER.info("solr core " + this.name + " initialized");
|
||||
}
|
||||
}
|
||||
|
||||
private void saveToSolr(final Iterator<? extends BSONObject> iterator, final String operationName) throws SolrException {
|
||||
long c = 0L;
|
||||
final long startTime = System.currentTimeMillis();
|
||||
while (iterator.hasNext()) {
|
||||
try {
|
||||
this.triggerQueue.put(iterator.next());
|
||||
}
|
||||
catch (InterruptedException e) {
|
||||
throw new SolrException("error put object: ", e);
|
||||
}
|
||||
++c;
|
||||
if (null != operationName && c % this.serverConfig.getTriggerBatchSize() == 0L && CoreImpl.LOGGER.isInfoEnabled()) {
|
||||
CoreImpl.LOGGER.info("{}: {}, {}/s", new Object[] { operationName, c, (int)(c / ((System.currentTimeMillis() - startTime) / 1000.0)) });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reload() throws ConfigurationException {
|
||||
this.solrServer.reload();
|
||||
try {
|
||||
this.solrServer.reloadCores();
|
||||
}
|
||||
catch (SolrException e) {
|
||||
throw new ConfigurationException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<Index> getIndexes() throws SolrException {
|
||||
return this.solrServer.getSolrSchema().getIndexes();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void ensureIndex(final Collection<Index> indexes, final Iterator<? extends BSONObject> iterator) throws SolrException {
|
||||
try {
|
||||
final String operationName = "ensureIndex_" + System.currentTimeMillis();
|
||||
if (CoreImpl.LOGGER.isInfoEnabled()) {
|
||||
CoreImpl.LOGGER.info("{}: ensure index[{}] {} ...", new Object[] { this.getName(), operationName, indexes });
|
||||
}
|
||||
if (this.solrServer.getSolrSchema().saveIndex(indexes) > 0) {
|
||||
this.solrServer.reloadCores();
|
||||
}
|
||||
this.saveToSolr(iterator, operationName);
|
||||
if (CoreImpl.LOGGER.isInfoEnabled()) {
|
||||
CoreImpl.LOGGER.info("{}: ensure index[{}] {} finished", new Object[] { this.getName(), operationName, indexes });
|
||||
}
|
||||
}
|
||||
catch (SolrException e) {
|
||||
CoreImpl.LOGGER.error("ensureIndex: ", (Throwable)e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void ensureIndex(final Index index, final Iterator<? extends BSONObject> iterator) throws SolrException {
|
||||
this.ensureIndex(Arrays.<Index>asList(index), iterator);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reIndex(final List<ObjectId> objIdList, final Iterator<? extends BSONObject> iterator) throws SolrException {
|
||||
final boolean logInfo = CollectionUtils.isEmpty((Collection)objIdList);
|
||||
final long startTime = System.currentTimeMillis();
|
||||
final String operationName = logInfo ? (this.getName() + "_re-index_" + startTime) : null;
|
||||
if (logInfo && CoreImpl.LOGGER.isInfoEnabled()) {
|
||||
CoreImpl.LOGGER.info(operationName + " started.");
|
||||
}
|
||||
this.saveToSolr(iterator, operationName);
|
||||
if (logInfo && CoreImpl.LOGGER.isInfoEnabled()) {
|
||||
CoreImpl.LOGGER.info("{} finished, total time: {}s.", operationName, (System.currentTimeMillis() - startTime) / 1000L);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void save(final Iterator<? extends BSONObject> iterator) throws SolrException {
|
||||
this.asynchOperation.run(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
CoreImpl.this.saveToSolr(iterator, "save_" + System.currentTimeMillis());
|
||||
}
|
||||
catch (SolrException e) {
|
||||
CoreImpl.LOGGER.error(CoreImpl.this.name + ": error save objects into solr:", (Throwable)e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteByQuery(final BSONObject query) throws SolrException {
|
||||
final String solrQuery = this.toSolrQuery(query).get("q");
|
||||
if (CoreImpl.LOGGER.isDebugEnabled()) {
|
||||
CoreImpl.LOGGER.debug("deleteByQuery: {}", (Object)solrQuery);
|
||||
}
|
||||
this.solrServer.deleteByQuery(solrQuery);
|
||||
}
|
||||
|
||||
private SolrQuery toSolrQuery(final BSONObject query) throws SolrException {
|
||||
return this.toSolrQuery(query, false);
|
||||
}
|
||||
|
||||
private SolrQuery toSolrQuery(final BSONObject orgQuery, final boolean useFilterQuery) throws SolrException {
|
||||
System.err.println(orgQuery.toString());
|
||||
final BSONObject query = (BSONObject)new BasicDBObject();
|
||||
query.putAll(orgQuery);
|
||||
final SolrQueryBuilder queryBuilder = new SolrQueryBuilder();
|
||||
final SolrQuery solrQuery = new SolrQuery();
|
||||
if (useFilterQuery && query != null && query.containsField("visible") && query.containsField("city") && query.containsField("cat") && query.get("cat") instanceof String) {
|
||||
final StringBuilder fq = new StringBuilder();
|
||||
fq.append("+cat:").append(query.get("cat"));
|
||||
fq.append(" +city:").append(query.get("city"));
|
||||
fq.append(" +visible:").append(((Number)query.get("visible")).intValue());
|
||||
solrQuery.addFilterQuery(fq.toString());
|
||||
query.removeField("cat");
|
||||
query.removeField("city");
|
||||
query.removeField("visible");
|
||||
}
|
||||
for (final String key : query.keySet()) {
|
||||
if (key.charAt(0) == '$') {
|
||||
continue;
|
||||
}
|
||||
final Object value = query.get(key);
|
||||
final Index index = this.assertFieldExistence(key);
|
||||
if (value instanceof BSONObject) {
|
||||
final BSONObject subQuery = (BSONObject)value;
|
||||
Object g = subQuery.get("$gt");
|
||||
boolean ge = false;
|
||||
if (null == g) {
|
||||
g = subQuery.get("$gte");
|
||||
ge = (g != null);
|
||||
}
|
||||
Object l = subQuery.get("$lt");
|
||||
boolean le = false;
|
||||
if (null == l) {
|
||||
l = subQuery.get("$lte");
|
||||
le = (l != null);
|
||||
}
|
||||
if (ge && null == l) {
|
||||
le = true;
|
||||
}
|
||||
if (le && null == g) {
|
||||
ge = true;
|
||||
}
|
||||
if (null != g || null != l) {
|
||||
queryBuilder.range(key, index.normalizeValue(g), index.normalizeValue(l), ge, le);
|
||||
}
|
||||
final Object in = subQuery.get("$in");
|
||||
if (null != in) {
|
||||
queryBuilder.in(key, index.normalizeValue((List<Object>)in));
|
||||
}
|
||||
final Object ne = subQuery.get("$ne");
|
||||
if (null == ne) {
|
||||
continue;
|
||||
}
|
||||
queryBuilder.ne(key, index.normalizeValue(ne));
|
||||
}
|
||||
else {
|
||||
queryBuilder.eq(key, index.normalizeValue(value));
|
||||
}
|
||||
}
|
||||
String q = queryBuilder.toQuery().toString();
|
||||
if (query.get("$sr") != null) {
|
||||
final Object value2 = query.get("$sr");
|
||||
final BSONObject bsonObject = (BSONObject)value2;
|
||||
final String val = bsonObject.get("sr.query").toString();
|
||||
if (val != null && !"".equals(val)) {
|
||||
q = q + " +(" + val + ")";
|
||||
}
|
||||
}
|
||||
if (query.get("$sd") != null) {
|
||||
final Object value2 = query.get("$sd");
|
||||
final BSONObject bsonObject = (BSONObject)value2;
|
||||
final String distance = (String)bsonObject.get("sd.distance");
|
||||
final String field = (String)bsonObject.get("sd.sfield");
|
||||
final String point = (String)bsonObject.get("sd.point");
|
||||
if (!StringUtils.isNotBlank(point) || !StringUtils.isNotBlank(distance) || !StringUtils.isNotBlank(field)) {
|
||||
throw new IllegalArgumentException("paramName $sd value illegal : " + value2);
|
||||
}
|
||||
solrQuery.setParam("sfield", field);
|
||||
solrQuery.setParam("d", distance);
|
||||
solrQuery.setParam("pt", point);
|
||||
solrQuery.addSort("geodist()", SolrQuery.ORDER.asc);
|
||||
solrQuery.addFilterQuery("{!geofilt}");
|
||||
}
|
||||
if (query.get("$fangjiawang_query_type") != null) {
|
||||
final Object value2 = query.get("$fangjiawang_query_type");
|
||||
solrQuery.add("q.op", value2.toString());
|
||||
}
|
||||
if (StringUtils.isEmpty(q)) {
|
||||
solrQuery.setQuery("*:*");
|
||||
}
|
||||
else {
|
||||
solrQuery.setQuery(q);
|
||||
}
|
||||
return solrQuery;
|
||||
}
|
||||
|
||||
private Index assertFieldExistence(final String key) throws SolrException {
|
||||
final Index index = this.solrServer.getSolrSchema().getIndex(key);
|
||||
if (null == index) {
|
||||
throw new SolrException("index '" + key + "' is not exist!");
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResponse find(final BSONObject query, final BSONObject sort, final BSONObject facet, int skip, int limit, final boolean fromMaster, final boolean bulk) throws SolrException {
|
||||
final long start = System.currentTimeMillis();
|
||||
if (limit < 0) {
|
||||
limit = this.serverConfig.getDefaultCursorLimit();
|
||||
}
|
||||
if (skip < 0) {
|
||||
skip = 0;
|
||||
}
|
||||
final SolrQuery solrQuery = this.toSolrQuery(query, true);
|
||||
solrQuery.setFields("_id");
|
||||
solrQuery.setStart(skip);
|
||||
solrQuery.setRows(limit);
|
||||
if (null != sort) {
|
||||
for (final String key : sort.keySet()) {
|
||||
final int value = ((Number)sort.get(key)).intValue();
|
||||
if (!"score".equals(key)) {
|
||||
this.assertFieldExistence(key);
|
||||
}
|
||||
solrQuery.addSort(key, (value > 0) ? SolrQuery.ORDER.asc : SolrQuery.ORDER.desc);
|
||||
}
|
||||
}
|
||||
if (null != facet) {
|
||||
final List<String> fields = (List<String>)facet.get("fields");
|
||||
if (null != fields) {
|
||||
solrQuery.setFacet(true);
|
||||
Integer facetMinCount = null;
|
||||
if (null != facet.get("minCount")) {
|
||||
facetMinCount = ((Number)facet.get("minCount")).intValue();
|
||||
}
|
||||
solrQuery.setFacetMinCount((null == facetMinCount) ? 1 : ((int)facetMinCount));
|
||||
Integer facetLimit = null;
|
||||
if (null != facet.get("limit")) {
|
||||
facetLimit = ((Number)facet.get("limit")).intValue();
|
||||
}
|
||||
solrQuery.setFacetLimit((null == facetLimit) ? 100 : facetLimit);
|
||||
for (final String field : fields) {
|
||||
this.assertFieldExistence(field);
|
||||
solrQuery.addFacetField(field);
|
||||
}
|
||||
}
|
||||
}
|
||||
final org.apache.solr.client.solrj.response.QueryResponse queryResponse = this.solrServer.query(solrQuery, fromMaster, bulk);
|
||||
final long time = System.currentTimeMillis() - start;
|
||||
if (time > 100L) {
|
||||
String host = solrQuery.get("__host");
|
||||
solrQuery.remove("__host");
|
||||
final StringBuilder sb = new StringBuilder();
|
||||
sb.append("query solr: ").append(solrQuery);
|
||||
if(StringUtils.isNotBlank(host)){
|
||||
sb.append(", host: ").append(host);
|
||||
}
|
||||
sb.append(", time: ").append(time);
|
||||
sb.append(", qtime: ").append(queryResponse.getQTime());
|
||||
sb.append(", numFound: ").append(queryResponse.getResults().getNumFound());
|
||||
sb.append(", numReturned: ").append(queryResponse.getResults().size());
|
||||
sb.append(", master: ").append(fromMaster);
|
||||
CoreImpl.PERFORMANCE_LOGGER.info(sb.toString());
|
||||
if (time < 1000L) {
|
||||
CoreImpl.PERFORMANCE_LOGGER.info(sb.toString());
|
||||
}
|
||||
else if (time < 3000L) {
|
||||
CoreImpl.PERFORMANCE_LOGGER.warn(sb.toString());
|
||||
}
|
||||
else {
|
||||
CoreImpl.PERFORMANCE_LOGGER.error(sb.toString());
|
||||
}
|
||||
}
|
||||
final QueryResponse response = this.toSolrResponse(queryResponse);
|
||||
return response;
|
||||
}
|
||||
|
||||
private QueryResponse toSolrResponse(final org.apache.solr.client.solrj.response.QueryResponse response) {
|
||||
if (response == null) {
|
||||
return QueryResponse.EMPTY_RESPONSE;
|
||||
}
|
||||
final SolrDocumentList solrDocumentList = response.getResults();
|
||||
final int numReturned = solrDocumentList.size();
|
||||
final List<ObjectId> idList = new ArrayList<ObjectId>(numReturned);
|
||||
for (int i = 0; i < numReturned; ++i) {
|
||||
final String value = ((SolrDocument)solrDocumentList.get(i)).getFirstValue("_id").toString();
|
||||
idList.add(new ObjectId(value));
|
||||
}
|
||||
final QueryResponse solrResponse = new QueryResponse();
|
||||
solrResponse.setNumFound((int)solrDocumentList.getNumFound());
|
||||
solrResponse.setNumReturned(numReturned);
|
||||
solrResponse.setNumToSkip((int)solrDocumentList.getStart());
|
||||
solrResponse.setIdList(idList);
|
||||
solrResponse.setFacetFieldList(response.getFacetFields());
|
||||
return solrResponse;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
private List<BSONObject> executeTrigger(final List<BSONObject> list) throws TriggerExecuteException {
|
||||
final Trigger trigger = this.storeConfig.getTrigger();
|
||||
if (null == trigger) {
|
||||
if (CoreImpl.LOGGER.isWarnEnabled()) {
|
||||
CoreImpl.LOGGER.warn("null trigger!");
|
||||
}
|
||||
return list;
|
||||
}
|
||||
if (null == trigger.getInterface()) {
|
||||
return list;
|
||||
}
|
||||
return TriggerExecutor.getInstance().execute(trigger, list);
|
||||
}
|
||||
|
||||
static {
|
||||
LOGGER = LoggerFactory.getLogger(CoreImpl.class);
|
||||
PERFORMANCE_LOGGER = LoggerFactory.getPerformanceLogger(CoreImpl.class);
|
||||
TRIIGER_POOL_SIZE = Integer.getInteger("datastore.triggerPoolSize", 10);
|
||||
}
|
||||
|
||||
private final String name;
|
||||
private final SolrServer solrServer;
|
||||
private final ServerConfig serverConfig;
|
||||
private final StoreConfig storeConfig;
|
||||
private final AsyncOperation asynchOperation;
|
||||
private final BlockingQueue<BSONObject> solrAddQueue;
|
||||
private final BlockingQueue<BSONObject> triggerQueue;
|
||||
private final Executor triggerExecutorService;
|
||||
private class TriggerRunnable implements Runnable {
|
||||
|
||||
public CoreImpl(StoreConfig storeConfig) throws ConfigurationException {
|
||||
this.name = storeConfig.getName();
|
||||
this.storeConfig = storeConfig;
|
||||
this.solrServer = new SolrServerImpl(this.storeConfig);
|
||||
this.serverConfig = storeConfig.getDataStoreConfig().getServerConfig();
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("init solr core: " + name + ", storeConfig: " + storeConfig);
|
||||
}
|
||||
public void run() {
|
||||
int sleepTime = 0;
|
||||
while (true) {
|
||||
try {
|
||||
final List<BSONObject> dbObjects = new ArrayList<BSONObject>(serverConfig.getTriggerBatchSize());
|
||||
triggerQueue.drainTo(dbObjects, serverConfig.getTriggerBatchSize());
|
||||
if (!dbObjects.isEmpty()) {
|
||||
triggerExecutorService.execute(new Runnable() {
|
||||
|
||||
this.asynchOperation = AsyncOperation.getInstance();
|
||||
public void run() {
|
||||
try {
|
||||
long time = System.currentTimeMillis();
|
||||
List<BSONObject> list = executeTrigger(dbObjects);
|
||||
time = System.currentTimeMillis() - time;
|
||||
if (LOGGER.isInfoEnabled() && time > 0) {
|
||||
LOGGER.info("execute trigger: {}/{} objects, {}ms, {}/s", new Object[] { list.size(),
|
||||
dbObjects.size(), time, dbObjects.size() * 1000 / time });
|
||||
}
|
||||
for (BSONObject obj : list) {
|
||||
solrAddQueue.put(obj);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("error execute trigger: ", e);
|
||||
}
|
||||
}
|
||||
|
||||
// trigger executor
|
||||
this.triggerExecutorService = new BlockingThreadPoolExecutor(name + "-Trigger", 1, TRIIGER_POOL_SIZE, TRIIGER_POOL_SIZE,
|
||||
60000);
|
||||
});
|
||||
sleepTime = serverConfig.getTriggerBatchSize() - dbObjects.size();
|
||||
if (sleepTime > 100) {
|
||||
Thread.sleep(sleepTime * 3);
|
||||
}
|
||||
} else {
|
||||
Thread.sleep(serverConfig.getTriggerBatchSize());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("error execute trigger: ", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// solr add thread
|
||||
this.solrAddQueue = new LinkedBlockingQueue<BSONObject>(INDEX_BATCH_SIZE * 2);
|
||||
Thread solrIndexAddThread = new Thread(new SolrAddRunnable(), name + "-SolrAdd");
|
||||
solrIndexAddThread.setDaemon(true);
|
||||
solrIndexAddThread.start();
|
||||
|
||||
// trigger thread
|
||||
this.triggerQueue = new LinkedBlockingQueue<BSONObject>(this.serverConfig.getTriggerBatchSize() * 4);
|
||||
Thread triggerExecuteThread = new Thread(new TriggerRunnable(), name + "-Trigger");
|
||||
triggerExecuteThread.setDaemon(true);
|
||||
triggerExecuteThread.start();
|
||||
|
||||
// shutdown hook
|
||||
Runtime.getRuntime().addShutdownHook(new Thread() {
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
while (!solrAddQueue.isEmpty()) {
|
||||
LOGGER.info(CoreImpl.this.name + ": wait for solr queue, current size: " + solrAddQueue.size());
|
||||
try {
|
||||
Thread.sleep(1000);
|
||||
} catch (InterruptedException e) {
|
||||
LOGGER.error("", e);
|
||||
}
|
||||
}
|
||||
LOGGER.info(CoreImpl.this.name + ": shutdown.");
|
||||
}
|
||||
|
||||
});
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("solr core " + name + " initialized");
|
||||
}
|
||||
}
|
||||
|
||||
private class TriggerRunnable implements Runnable {
|
||||
|
||||
public void run() {
|
||||
int sleepTime = 0;
|
||||
while (true) {
|
||||
try {
|
||||
final List<BSONObject> dbObjects = new ArrayList<BSONObject>(serverConfig.getTriggerBatchSize());
|
||||
triggerQueue.drainTo(dbObjects, serverConfig.getTriggerBatchSize());
|
||||
if (!dbObjects.isEmpty()) {
|
||||
triggerExecutorService.execute(new Runnable() {
|
||||
|
||||
public void run() {
|
||||
try {
|
||||
long time = System.currentTimeMillis();
|
||||
List<BSONObject> list = executeTrigger(dbObjects);
|
||||
time = System.currentTimeMillis() - time;
|
||||
if (LOGGER.isInfoEnabled() && time > 0) {
|
||||
LOGGER.info("execute trigger: {}/{} objects, {}ms, {}/s", new Object[] { list.size(),
|
||||
dbObjects.size(), time, dbObjects.size() * 1000 / time });
|
||||
}
|
||||
for (BSONObject obj : list) {
|
||||
solrAddQueue.put(obj);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("error execute trigger: ", e);
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
sleepTime = serverConfig.getTriggerBatchSize() - dbObjects.size();
|
||||
if (sleepTime > 100) {
|
||||
Thread.sleep(sleepTime * 3);
|
||||
}
|
||||
} else {
|
||||
Thread.sleep(serverConfig.getTriggerBatchSize());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("error execute trigger: ", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private class SolrAddRunnable implements Runnable {
|
||||
|
||||
public void run() {
|
||||
final List<BSONObject> dbObjects = new ArrayList<BSONObject>(INDEX_BATCH_SIZE);
|
||||
final int maxSleepTime = 10000;
|
||||
int sleepTime = 0;
|
||||
while (true) {
|
||||
try {
|
||||
solrAddQueue.drainTo(dbObjects, INDEX_BATCH_SIZE);
|
||||
if (!dbObjects.isEmpty()) {
|
||||
solrServer.add(dbObjects);
|
||||
sleepTime = maxSleepTime - dbObjects.size();
|
||||
if (sleepTime > 100) {
|
||||
Thread.sleep(sleepTime);
|
||||
}
|
||||
} else {
|
||||
Thread.sleep(maxSleepTime);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("error save to solr: ", e);
|
||||
} finally {
|
||||
dbObjects.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void saveToSolr(final Iterator<? extends BSONObject> iterator, String operationName) throws SolrException {
|
||||
long c = 0;
|
||||
long startTime = System.currentTimeMillis();
|
||||
while (iterator.hasNext()) {
|
||||
try {
|
||||
triggerQueue.put(iterator.next());
|
||||
} catch (InterruptedException e) {
|
||||
throw new SolrException("error put object: ", e);
|
||||
}
|
||||
c++;
|
||||
if (null != operationName && c % serverConfig.getTriggerBatchSize() == 0 && LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("{}: {}, {}/s", new Object[] { operationName, c,
|
||||
(int) (c / ((System.currentTimeMillis() - startTime) / 1000.0)) });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void reload() throws ConfigurationException {
|
||||
this.solrServer.reload();
|
||||
try {
|
||||
this.solrServer.reloadCores();
|
||||
} catch (SolrException e) {
|
||||
throw new ConfigurationException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public Collection<Index> getIndexes() throws SolrException {
|
||||
return this.solrServer.getSolrSchema().getIndexes();
|
||||
}
|
||||
|
||||
public void ensureIndex(final Collection<Index> indexes, final Iterator<? extends BSONObject> iterator) throws SolrException {
|
||||
try {
|
||||
String operationName = "ensureIndex_" + System.currentTimeMillis();
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("{}: ensure index[{}] {} ...", new Object[] { CoreImpl.this.getName(), operationName, indexes });
|
||||
}
|
||||
if (solrServer.getSolrSchema().saveIndex(indexes) > 0) {
|
||||
this.solrServer.reloadCores();
|
||||
}
|
||||
saveToSolr(iterator, operationName);
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("{}: ensure index[{}] {} finished", new Object[] { CoreImpl.this.getName(), operationName, indexes });
|
||||
}
|
||||
} catch (SolrException e) {
|
||||
LOGGER.error("ensureIndex: ", e);
|
||||
}
|
||||
}
|
||||
|
||||
public void ensureIndex(Index index, Iterator<? extends BSONObject> iterator) throws SolrException {
|
||||
ensureIndex(Arrays.asList(index), iterator);
|
||||
}
|
||||
|
||||
public void reIndex(final List<ObjectId> objIdList, final Iterator<? extends BSONObject> iterator) throws SolrException {
|
||||
boolean logInfo = CollectionUtils.isEmpty(objIdList);
|
||||
long startTime = System.currentTimeMillis();
|
||||
String operationName = logInfo ? CoreImpl.this.getName() + "_re-index_" + startTime : null;
|
||||
if (logInfo && LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info(operationName + " started.");
|
||||
}
|
||||
saveToSolr(iterator, operationName);
|
||||
if (logInfo && LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("{} finished, total time: {}s.", operationName, (System.currentTimeMillis() - startTime) / 1000);
|
||||
}
|
||||
}
|
||||
|
||||
public void save(final Iterator<? extends BSONObject> iterator) throws SolrException {
|
||||
asynchOperation.run(new Runnable() {
|
||||
public void run() {
|
||||
try {
|
||||
saveToSolr(iterator, "save_" + System.currentTimeMillis());
|
||||
} catch (SolrException e) {
|
||||
LOGGER.error(name + ": error save objects into solr:", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public void deleteByQuery(final BSONObject query) throws SolrException {
|
||||
String solrQuery = toSolrQuery(query).get(CommonParams.Q);
|
||||
if (LOGGER.isDebugEnabled()) {
|
||||
LOGGER.debug("deleteByQuery: {}", solrQuery);
|
||||
}
|
||||
this.solrServer.deleteByQuery(solrQuery);
|
||||
}
|
||||
|
||||
private SolrQuery toSolrQuery(BSONObject query) throws SolrException {
|
||||
return toSolrQuery(query, false);
|
||||
}
|
||||
|
||||
private SolrQuery toSolrQuery(final BSONObject orgQuery, boolean useFilterQuery) throws SolrException {
|
||||
System.err.println(orgQuery.toString());
|
||||
BSONObject query = new BasicDBObject();
|
||||
query.putAll(orgQuery);
|
||||
SolrQueryBuilder queryBuilder = new SolrQueryBuilder();
|
||||
|
||||
SolrQuery solrQuery = new SolrQuery();
|
||||
if (useFilterQuery && query != null && query.containsField("visible") && query.containsField("city")
|
||||
&& query.containsField("cat") && query.get("cat") instanceof String) {
|
||||
StringBuilder fq = new StringBuilder();
|
||||
fq.append("+cat:").append(query.get("cat"));
|
||||
fq.append(" +city:").append(query.get("city"));
|
||||
fq.append(" +visible:").append(((Number) query.get("visible")).intValue());
|
||||
solrQuery.addFilterQuery(fq.toString());
|
||||
query.removeField("cat");
|
||||
query.removeField("city");
|
||||
query.removeField("visible");
|
||||
}
|
||||
|
||||
if (null != query) {
|
||||
for (String key : query.keySet()) {
|
||||
if (key.charAt(0) == '$') {
|
||||
continue;
|
||||
}
|
||||
Object value = query.get(key);
|
||||
Index index = assertFieldExistence(key);
|
||||
if (value instanceof BSONObject) {
|
||||
BSONObject subQuery = (BSONObject) value;
|
||||
Object g = subQuery.get("$gt");
|
||||
boolean ge = false;
|
||||
if (null == g) {
|
||||
g = subQuery.get("$gte");
|
||||
ge = g != null;
|
||||
}
|
||||
Object l = subQuery.get("$lt");
|
||||
boolean le = false;
|
||||
if (null == l) {
|
||||
l = subQuery.get("$lte");
|
||||
le = l != null;
|
||||
}
|
||||
if (ge && null == l) {
|
||||
le = true;
|
||||
}
|
||||
if (le && null == g) {
|
||||
ge = true;
|
||||
}
|
||||
if (null != g || null != l) {
|
||||
queryBuilder.range(key, index.normalizeValue(g), index.normalizeValue(l), ge, le);
|
||||
}
|
||||
Object in = subQuery.get("$in");
|
||||
if (null != in) {
|
||||
queryBuilder.in(key, index.normalizeValue((List) in));
|
||||
}
|
||||
Object ne = subQuery.get("$ne");
|
||||
if (null != ne) {
|
||||
queryBuilder.ne(key, index.normalizeValue(ne));
|
||||
}
|
||||
|
||||
} else {
|
||||
queryBuilder.eq(key, index.normalizeValue(value));
|
||||
}
|
||||
}
|
||||
}
|
||||
String q = queryBuilder.toQuery().toString();
|
||||
|
||||
// $srģʽ
|
||||
if (null != query) {
|
||||
if (query.get("$sr") != null) {
|
||||
Object value = query.get("$sr");
|
||||
BSONObject bsonObject = (BSONObject) value;
|
||||
String val = bsonObject.get("sr.query").toString();
|
||||
if (val != null && !"".equals(val)) {
|
||||
q += " +(" + val + ")";
|
||||
}
|
||||
}
|
||||
if (query.get("$sd") != null) {
|
||||
Object value = query.get("$sd");
|
||||
BSONObject bsonObject = (BSONObject) value;
|
||||
String distance = (String) bsonObject.get("sd.distance");
|
||||
String field = (String) bsonObject.get("sd.sfield");
|
||||
String point = (String) bsonObject.get("sd.point");
|
||||
if (StringUtils.isNotBlank(point) && StringUtils.isNotBlank(distance) && StringUtils.isNotBlank(field)) {
|
||||
solrQuery.setParam(SpatialParams.FIELD, field);
|
||||
solrQuery.setParam(SpatialParams.DISTANCE, distance);
|
||||
solrQuery.setParam(SpatialParams.POINT, point);
|
||||
solrQuery.addSort("geodist()", ORDER.asc);
|
||||
solrQuery.addFilterQuery("{!geofilt}");
|
||||
} else {
|
||||
throw new IllegalArgumentException("paramName $sd value illegal : " + value);
|
||||
}
|
||||
}
|
||||
|
||||
if (query.get("$fangjiawang_query_type") != null) {
|
||||
Object value = query.get("$fangjiawang_query_type");
|
||||
solrQuery.add("q.op", value.toString());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
if (StringUtils.isEmpty(q)) {
|
||||
solrQuery.setQuery("*:*");
|
||||
} else {
|
||||
solrQuery.setQuery(q);
|
||||
}
|
||||
return solrQuery;
|
||||
}
|
||||
|
||||
private Index assertFieldExistence(String key) throws SolrException {
|
||||
Index index = this.solrServer.getSolrSchema().getIndex(key);
|
||||
if (null == index) {
|
||||
throw new SolrException("index '" + key + "' is not exist!");
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
public QueryResponse find(BSONObject query, BSONObject sort, BSONObject facet, int skip, int limit, boolean fromMaster,
|
||||
boolean bulk) throws SolrException {
|
||||
long start = System.currentTimeMillis();
|
||||
if (limit < 0) {
|
||||
limit = serverConfig.getDefaultCursorLimit();
|
||||
}
|
||||
if (skip < 0) {
|
||||
skip = 0;
|
||||
}
|
||||
SolrQuery solrQuery = toSolrQuery(query, true);
|
||||
|
||||
// fields, start, limit
|
||||
solrQuery.setFields(SolrSchema.ID);
|
||||
solrQuery.setStart(skip);
|
||||
solrQuery.setRows(limit);
|
||||
|
||||
// sort
|
||||
if (null != sort) {
|
||||
for (String key : sort.keySet()) {
|
||||
int value = ((Number) sort.get(key)).intValue();
|
||||
if (!"score".equals(key)) { // pseudo-field score
|
||||
assertFieldExistence(key);
|
||||
}
|
||||
solrQuery.addSort(key, value > 0 ? ORDER.asc : ORDER.desc);
|
||||
}
|
||||
}
|
||||
|
||||
// facet
|
||||
if (null != facet) {
|
||||
List<String> fields = (List<String>) facet.get("fields");
|
||||
if (null != fields) {
|
||||
solrQuery.setFacet(true);
|
||||
Integer facetMinCount = null;
|
||||
if (null != facet.get("minCount")) {
|
||||
facetMinCount = ((Number) facet.get("minCount")).intValue();
|
||||
}
|
||||
solrQuery.setFacetMinCount(null == facetMinCount ? 1 : facetMinCount);
|
||||
Integer facetLimit = null;
|
||||
if (null != facet.get("limit")) {
|
||||
facetLimit = ((Number) facet.get("limit")).intValue();
|
||||
}
|
||||
solrQuery.setFacetLimit(null == facetLimit ? 100 : facetLimit);
|
||||
for (String field : fields) {
|
||||
assertFieldExistence(field);
|
||||
solrQuery.addFacetField(field);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
org.apache.solr.client.solrj.response.QueryResponse queryResponse = this.solrServer.query(solrQuery, fromMaster, bulk);
|
||||
long time = System.currentTimeMillis() - start;
|
||||
if (time > 100) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("query solr: ").append(solrQuery);
|
||||
sb.append(", time: ").append(time);
|
||||
sb.append(", qtime: ").append(queryResponse.getQTime());
|
||||
sb.append(", numFound: ").append(queryResponse.getResults().getNumFound());
|
||||
sb.append(", numReturned: ").append(queryResponse.getResults().size());
|
||||
sb.append(", master: ").append(fromMaster);
|
||||
if (time < 1000) {
|
||||
PERFORMANCE_LOGGER.info(sb.toString());
|
||||
} else if (time < 3000) {
|
||||
PERFORMANCE_LOGGER.warn(sb.toString());
|
||||
} else {
|
||||
PERFORMANCE_LOGGER.error(sb.toString());
|
||||
}
|
||||
}
|
||||
QueryResponse response = toSolrResponse(queryResponse);
|
||||
return response;
|
||||
}
|
||||
|
||||
private QueryResponse toSolrResponse(org.apache.solr.client.solrj.response.QueryResponse response) {
|
||||
if (response == null) {
|
||||
return QueryResponse.EMPTY_RESPONSE;
|
||||
}
|
||||
|
||||
SolrDocumentList solrDocumentList = response.getResults();
|
||||
int numReturned = solrDocumentList.size();
|
||||
List<ObjectId> idList = new ArrayList<ObjectId>(numReturned);
|
||||
for (int i = 0; i < numReturned; i++) {
|
||||
String value = solrDocumentList.get(i).getFirstValue(SolrSchema.ID).toString();
|
||||
idList.add(new ObjectId(value));
|
||||
}
|
||||
|
||||
QueryResponse solrResponse = new QueryResponse();
|
||||
solrResponse.setNumFound((int) solrDocumentList.getNumFound());
|
||||
solrResponse.setNumReturned(numReturned);
|
||||
solrResponse.setNumToSkip((int) solrDocumentList.getStart());
|
||||
solrResponse.setIdList(idList);
|
||||
solrResponse.setFacetFieldList(response.getFacetFields());
|
||||
return solrResponse;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
private List<BSONObject> executeTrigger(List<BSONObject> list) throws TriggerExecuteException {
|
||||
Trigger trigger = this.storeConfig.getTrigger();
|
||||
if (null == trigger) {
|
||||
if (LOGGER.isWarnEnabled()) {
|
||||
LOGGER.warn("null trigger!");
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
if (null == trigger.getInterface()) {
|
||||
return list;
|
||||
}
|
||||
return TriggerExecutor.getInstance().execute(trigger, list);
|
||||
}
|
||||
}
|
||||
|
||||
private class SolrAddRunnable implements Runnable
|
||||
{
|
||||
@Override
|
||||
public void run() {
|
||||
final List<BSONObject> dbObjects = new ArrayList<BSONObject>(30000);
|
||||
final int maxSleepTime = 10000;
|
||||
int sleepTime = 0;
|
||||
while (true) {
|
||||
try {
|
||||
CoreImpl.this.solrAddQueue.drainTo(dbObjects, 30000);
|
||||
if (!dbObjects.isEmpty()) {
|
||||
CoreImpl.this.solrServer.add(dbObjects);
|
||||
sleepTime = 10000 - dbObjects.size();
|
||||
if (sleepTime <= 100) {
|
||||
continue;
|
||||
}
|
||||
Thread.sleep(sleepTime);
|
||||
}
|
||||
else {
|
||||
Thread.sleep(10000L);
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
CoreImpl.LOGGER.error("error save to solr: ", (Throwable)e);
|
||||
}
|
||||
finally {
|
||||
dbObjects.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+9
-9
@@ -2,13 +2,13 @@ package com.fangjia.datastore.solr;
|
||||
|
||||
import org.apache.solr.client.solrj.SolrServer;
|
||||
|
||||
public abstract class HttpSolrServer extends SolrServer {
|
||||
|
||||
private static final long serialVersionUID = -3967599614339379847L;
|
||||
|
||||
abstract public void reloadCore() throws SolrException;
|
||||
|
||||
abstract public String getCoreName();
|
||||
|
||||
abstract public String getHost();
|
||||
public abstract class HttpSolrServer extends SolrServer
|
||||
{
|
||||
private static final long serialVersionUID = -3967599614339379847L;
|
||||
|
||||
public abstract void reloadCore() throws SolrException;
|
||||
|
||||
public abstract String getCoreName();
|
||||
|
||||
public abstract String getHost();
|
||||
}
|
||||
|
||||
@@ -1,191 +1,196 @@
|
||||
package com.fangjia.datastore.solr;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang.builder.EqualsBuilder;
|
||||
import org.apache.commons.lang.builder.HashCodeBuilder;
|
||||
import org.bson.BSONObject;
|
||||
import org.bson.BasicBSONObject;
|
||||
import org.bson.BSONObject;
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class Index {
|
||||
public static enum IndexTypeEnum {
|
||||
STRING, INTEGER, INT, DOUBLE, TEXT_STD, TEXT_CJK, DATE, LONG, LOCATION;
|
||||
public String getType() {
|
||||
return this.name().toLowerCase();
|
||||
}
|
||||
|
||||
public static IndexTypeEnum parse(String type) {
|
||||
for (IndexTypeEnum e : IndexTypeEnum.values()) {
|
||||
if (e.getType().equals(type)) {
|
||||
return e;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String name;
|
||||
private IndexTypeEnum type;
|
||||
private boolean stored;
|
||||
private boolean indexed;
|
||||
private boolean multiValued;
|
||||
private String comments;
|
||||
private boolean dynamic;
|
||||
|
||||
public Index(String name, IndexTypeEnum type, boolean stored, boolean indexed, boolean multiValued, boolean dynamic,
|
||||
String comments) {
|
||||
this.name = name;
|
||||
this.type = type;
|
||||
this.stored = stored;
|
||||
this.indexed = indexed;
|
||||
this.multiValued = multiValued;
|
||||
this.dynamic = dynamic;
|
||||
this.comments = comments;
|
||||
}
|
||||
|
||||
public Index(String name, IndexTypeEnum type) {
|
||||
this.name = name;
|
||||
this.type = type;
|
||||
this.stored = false;
|
||||
this.indexed = true;
|
||||
this.multiValued = false;
|
||||
this.dynamic = false;
|
||||
this.comments = "";
|
||||
}
|
||||
|
||||
public List<Object> normalizeValue(List<Object> values) {
|
||||
if (null == values) {
|
||||
return null;
|
||||
}
|
||||
List<Object> ret = new ArrayList<Object>(values.size());
|
||||
for (Object v : values) {
|
||||
ret.add(normalizeValue(v));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public Object normalizeValue(Object value) {
|
||||
if (null == value) {
|
||||
return null;
|
||||
}
|
||||
switch (this.type) {
|
||||
case STRING:
|
||||
case TEXT_STD:
|
||||
return value.toString();
|
||||
case INTEGER:
|
||||
return ((Number) value).intValue();
|
||||
case LONG:
|
||||
return ((Number) value).longValue();
|
||||
case DOUBLE:
|
||||
return ((Number) value).doubleValue();
|
||||
case DATE:
|
||||
return (Date) value;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
public Index() {
|
||||
this(null, null);
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public IndexTypeEnum getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public boolean isStored() {
|
||||
return stored;
|
||||
}
|
||||
|
||||
public boolean isIndexed() {
|
||||
return indexed;
|
||||
}
|
||||
|
||||
public boolean isMultiValued() {
|
||||
return multiValued;
|
||||
}
|
||||
|
||||
public String getComments() {
|
||||
return comments;
|
||||
}
|
||||
|
||||
public boolean isDynamic() {
|
||||
return dynamic;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public void setType(IndexTypeEnum type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public void setStored(boolean stored) {
|
||||
this.stored = stored;
|
||||
}
|
||||
|
||||
public void setIndexed(boolean indexed) {
|
||||
this.indexed = indexed;
|
||||
}
|
||||
|
||||
public void setMultiValued(boolean multiValued) {
|
||||
this.multiValued = multiValued;
|
||||
}
|
||||
|
||||
public void setComments(String comments) {
|
||||
this.comments = comments;
|
||||
}
|
||||
|
||||
public void setDynamic(boolean dynamic) {
|
||||
this.dynamic = dynamic;
|
||||
}
|
||||
|
||||
public BSONObject toBSONObject() {
|
||||
BSONObject ret = new BasicBSONObject();
|
||||
ret.put("name", name);
|
||||
ret.put("type", this.type.getType());
|
||||
ret.put("stored", this.stored);
|
||||
ret.put("indexed", this.indexed);
|
||||
ret.put("multiValued", this.multiValued);
|
||||
ret.put("dynamic", this.dynamic);
|
||||
ret.put("comments", this.comments);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return new HashCodeBuilder().append(this.dynamic).append(this.indexed).append(this.multiValued).append(this.stored)
|
||||
.append(this.comments).append(this.name).append(this.type).toHashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (null == obj) {
|
||||
return false;
|
||||
}
|
||||
if (this == obj) {
|
||||
return true;
|
||||
}
|
||||
if (!(obj instanceof Index)) {
|
||||
return false;
|
||||
}
|
||||
Index other = (Index) obj;
|
||||
return new EqualsBuilder().append(this.dynamic, other.dynamic).append(this.indexed, other.indexed).append(
|
||||
this.multiValued, other.multiValued).append(this.stored, other.stored).append(this.comments, other.comments)
|
||||
.append(this.name, other.name).append(this.type, other.type).isEquals();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return new StringBuilder().append("{name=").append(name).append(", type=").append(type).append(", stored=")
|
||||
.append(stored).append(", indexed=").append(indexed).append(", multiValued=").append(multiValued).append(
|
||||
", dynamic=").append(dynamic).append(", comments=").append(this.comments).append("}").toString();
|
||||
}
|
||||
|
||||
public class Index
|
||||
{
|
||||
private String name;
|
||||
private IndexTypeEnum type;
|
||||
private boolean stored;
|
||||
private boolean indexed;
|
||||
private boolean multiValued;
|
||||
private String comments;
|
||||
private boolean dynamic;
|
||||
|
||||
public Index(final String name, final IndexTypeEnum type, final boolean stored, final boolean indexed, final boolean multiValued, final boolean dynamic, final String comments) {
|
||||
this.name = name;
|
||||
this.type = type;
|
||||
this.stored = stored;
|
||||
this.indexed = indexed;
|
||||
this.multiValued = multiValued;
|
||||
this.dynamic = dynamic;
|
||||
this.comments = comments;
|
||||
}
|
||||
|
||||
public Index(final String name, final IndexTypeEnum type) {
|
||||
this.name = name;
|
||||
this.type = type;
|
||||
this.stored = false;
|
||||
this.indexed = true;
|
||||
this.multiValued = false;
|
||||
this.dynamic = false;
|
||||
this.comments = "";
|
||||
}
|
||||
|
||||
public List<Object> normalizeValue(final List<Object> values) {
|
||||
if (null == values) {
|
||||
return null;
|
||||
}
|
||||
final List<Object> ret = new ArrayList<Object>(values.size());
|
||||
for (final Object v : values) {
|
||||
ret.add(this.normalizeValue(v));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public Object normalizeValue(final Object value) {
|
||||
if (null == value) {
|
||||
return null;
|
||||
}
|
||||
switch (this.type) {
|
||||
case STRING:
|
||||
case TEXT_STD:
|
||||
return value.toString();
|
||||
case INTEGER:
|
||||
return ((Number)value).intValue();
|
||||
case LONG:
|
||||
return ((Number)value).longValue();
|
||||
case DOUBLE:
|
||||
return ((Number)value).doubleValue();
|
||||
case DATE:
|
||||
return value;
|
||||
default:
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
public Index() {
|
||||
this(null, null);
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
public IndexTypeEnum getType() {
|
||||
return this.type;
|
||||
}
|
||||
|
||||
public boolean isStored() {
|
||||
return this.stored;
|
||||
}
|
||||
|
||||
public boolean isIndexed() {
|
||||
return this.indexed;
|
||||
}
|
||||
|
||||
public boolean isMultiValued() {
|
||||
return this.multiValued;
|
||||
}
|
||||
|
||||
public String getComments() {
|
||||
return this.comments;
|
||||
}
|
||||
|
||||
public boolean isDynamic() {
|
||||
return this.dynamic;
|
||||
}
|
||||
|
||||
public void setName(final String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public void setType(final IndexTypeEnum type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public void setStored(final boolean stored) {
|
||||
this.stored = stored;
|
||||
}
|
||||
|
||||
public void setIndexed(final boolean indexed) {
|
||||
this.indexed = indexed;
|
||||
}
|
||||
|
||||
public void setMultiValued(final boolean multiValued) {
|
||||
this.multiValued = multiValued;
|
||||
}
|
||||
|
||||
public void setComments(final String comments) {
|
||||
this.comments = comments;
|
||||
}
|
||||
|
||||
public void setDynamic(final boolean dynamic) {
|
||||
this.dynamic = dynamic;
|
||||
}
|
||||
|
||||
public BSONObject toBSONObject() {
|
||||
final BSONObject ret = (BSONObject)new BasicBSONObject();
|
||||
ret.put("name", (Object)this.name);
|
||||
ret.put("type", (Object)this.type.getType());
|
||||
ret.put("stored", (Object)this.stored);
|
||||
ret.put("indexed", (Object)this.indexed);
|
||||
ret.put("multiValued", (Object)this.multiValued);
|
||||
ret.put("dynamic", (Object)this.dynamic);
|
||||
ret.put("comments", (Object)this.comments);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return new HashCodeBuilder().append(this.dynamic).append(this.indexed).append(this.multiValued).append(this.stored).append((Object)this.comments).append((Object)this.name).append((Object)this.type).toHashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(final Object obj) {
|
||||
if (null == obj) {
|
||||
return false;
|
||||
}
|
||||
if (this == obj) {
|
||||
return true;
|
||||
}
|
||||
if (!(obj instanceof Index)) {
|
||||
return false;
|
||||
}
|
||||
final Index other = (Index)obj;
|
||||
return new EqualsBuilder().append(this.dynamic, other.dynamic).append(this.indexed, other.indexed).append(this.multiValued, other.multiValued).append(this.stored, other.stored).append((Object)this.comments, (Object)other.comments).append((Object)this.name, (Object)other.name).append((Object)this.type, (Object)other.type).isEquals();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "{name=" + this.name + ", type=" + this.type + ", stored=" + this.stored + ", indexed=" + this.indexed + ", multiValued=" + this.multiValued + ", dynamic=" + this.dynamic + ", comments=" + this.comments + "}";
|
||||
}
|
||||
|
||||
public enum IndexTypeEnum
|
||||
{
|
||||
STRING,
|
||||
INTEGER,
|
||||
INT,
|
||||
DOUBLE,
|
||||
TEXT_STD,
|
||||
TEXT_CJK,
|
||||
DATE,
|
||||
LONG,
|
||||
LOCATION;
|
||||
|
||||
public String getType() {
|
||||
return this.name().toLowerCase();
|
||||
}
|
||||
|
||||
public static IndexTypeEnum parse(final String type) {
|
||||
for (final IndexTypeEnum e : values()) {
|
||||
if (e.getType().equals(type)) {
|
||||
return e;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+216
-255
@@ -1,262 +1,223 @@
|
||||
package com.fangjia.datastore.solr;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.lang.ref.WeakReference;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.commons.lang.math.RandomUtils;
|
||||
import org.apache.solr.client.solrj.SolrQuery;
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
import com.fangjia.datastore.utils.LoggerFactory;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||
import org.apache.solr.common.params.ShardParams;
|
||||
import java.io.IOException;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.lang.ref.WeakReference;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.Executors;
|
||||
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import java.util.Iterator;
|
||||
import org.apache.commons.lang.math.RandomUtils;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import org.apache.solr.client.solrj.SolrQuery;
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import com.fangjia.datastore.utils.LoggerFactory;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author velna
|
||||
* @see org.apache.solr.client.solrj.impl.LBHttpSolrServer
|
||||
*/
|
||||
public class LBHttpSolrServer extends HttpSolrServer {
|
||||
private static final long serialVersionUID = -6175098554678462150L;
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LBHttpSolrServer.class);
|
||||
|
||||
private static final SolrQuery SOLR_QUERY = new SolrQuery("*:*");
|
||||
private static final int CHECK_INTERVAL = 10 * 1000;
|
||||
|
||||
private final String coreName;
|
||||
|
||||
private final ReentrantLock checkLock = new ReentrantLock();
|
||||
private ScheduledExecutorService aliveCheckExecutor;
|
||||
private final CopyOnWriteArrayList<WrappedHttpSolrServer> aliveServers = new CopyOnWriteArrayList<WrappedHttpSolrServer>();
|
||||
private final CopyOnWriteArrayList<WrappedHttpSolrServer> zombieServers = new CopyOnWriteArrayList<WrappedHttpSolrServer>();
|
||||
|
||||
public LBHttpSolrServer(HttpSolrServer... httpSolrServers) {
|
||||
if (null == httpSolrServers || httpSolrServers.length == 0) {
|
||||
throw new IllegalArgumentException("length of httpSolrServers must greater than 0");
|
||||
}
|
||||
for (HttpSolrServer httpSolrServer : httpSolrServers) {
|
||||
this.aliveServers.add(new WrappedHttpSolrServer(httpSolrServer));
|
||||
}
|
||||
coreName = httpSolrServers[0].getCoreName();
|
||||
}
|
||||
|
||||
private WrappedHttpSolrServer selectRandomServer() {
|
||||
for (;;) {
|
||||
if (aliveServers.isEmpty()) {
|
||||
throw new RuntimeException("no solr server available, all are down!");
|
||||
}
|
||||
try {
|
||||
if (aliveServers.size() == 1) {
|
||||
return aliveServers.get(0);
|
||||
}
|
||||
return aliveServers.get(RandomUtils.nextInt(aliveServers.size()));
|
||||
} catch (IndexOutOfBoundsException e) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*@Override
|
||||
public NamedList<Object> request(final SolrRequest request) throws SolrServerException, IOException {
|
||||
int attempts = 0;
|
||||
int startSize = aliveServers.size();
|
||||
Exception ex;
|
||||
String zombieHost = null;
|
||||
for (;;) {
|
||||
WrappedHttpSolrServer httpSolrServer = selectRandomServer();
|
||||
try {
|
||||
if (null != zombieHost && request.getParams() instanceof SolrQuery) {
|
||||
SolrQuery query = (SolrQuery) request.getParams();
|
||||
String shards = query.get(ShardParams.SHARDS);
|
||||
if (null != shards) {
|
||||
shards = StringUtils.replace(shards, zombieHost, httpSolrServer.httpSolrServer.getHost());
|
||||
query.set(ShardParams.SHARDS, shards);
|
||||
}
|
||||
}
|
||||
return httpSolrServer.httpSolrServer.request(request);
|
||||
} catch (SolrServerException e) {
|
||||
if (e.getRootCause() instanceof IOException) {
|
||||
ex = e;
|
||||
zombieHost = httpSolrServer.httpSolrServer.getHost();
|
||||
LOGGER.error("error request solr server: " + zombieHost, e);
|
||||
moveAliveToDead(httpSolrServer);
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
attempts++;
|
||||
if (attempts >= startSize)
|
||||
throw new SolrServerException("No live SolrServers available to handle this request", ex);
|
||||
}
|
||||
}*/
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
for (WrappedHttpSolrServer server : aliveServers) {
|
||||
LOGGER.info("---"+server.httpSolrServer.getHost());
|
||||
}
|
||||
return selectRandomServer().httpSolrServer.getHost();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getCoreName() {
|
||||
return coreName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reloadCore() throws com.fangjia.datastore.solr.SolrException {
|
||||
// does noting
|
||||
}
|
||||
|
||||
private void moveAliveToDead(WrappedHttpSolrServer solrServer) {
|
||||
checkLock.lock();
|
||||
try {
|
||||
boolean result = aliveServers.remove(solrServer);
|
||||
if (result) {
|
||||
if (zombieServers.addIfAbsent(solrServer)) {
|
||||
if (LOGGER.isWarnEnabled()) {
|
||||
LOGGER.warn("solr server [" + solrServer.httpSolrServer.getHost() + "] is marked as down.");
|
||||
}
|
||||
startAliveCheckExecutor();
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
checkLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private void checkAZombieServer(WrappedHttpSolrServer zombieServer) {
|
||||
checkLock.lock();
|
||||
try {
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("check solr server [" + zombieServer.httpSolrServer.getHost() + "] ...");
|
||||
}
|
||||
QueryResponse resp = zombieServer.httpSolrServer.query(SOLR_QUERY);
|
||||
if (resp.getStatus() == 0) {
|
||||
// server has come back up
|
||||
zombieServers.remove(zombieServer);
|
||||
aliveServers.add(zombieServer);
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("solr server [" + zombieServer.httpSolrServer.getHost() + "] has come back.");
|
||||
}
|
||||
zombieServer.failedPings = 0;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
zombieServer.failedPings++;
|
||||
// Expected . The server is still down
|
||||
} finally {
|
||||
checkLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private void startAliveCheckExecutor() {
|
||||
if (aliveCheckExecutor == null) {
|
||||
synchronized (this) {
|
||||
if (aliveCheckExecutor == null) {
|
||||
aliveCheckExecutor = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory());
|
||||
aliveCheckExecutor.scheduleAtFixedRate(getAliveCheckRunner(new WeakReference<LBHttpSolrServer>(this)),
|
||||
CHECK_INTERVAL, CHECK_INTERVAL, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static Runnable getAliveCheckRunner(final WeakReference<LBHttpSolrServer> lbHttpSolrServer) {
|
||||
return new Runnable() {
|
||||
public void run() {
|
||||
LBHttpSolrServer solrServer = lbHttpSolrServer.get();
|
||||
if (solrServer != null && solrServer.zombieServers != null) {
|
||||
for (WrappedHttpSolrServer zombieServer : solrServer.zombieServers) {
|
||||
solrServer.checkAZombieServer(zombieServer);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The default thread factory
|
||||
*/
|
||||
static class DefaultThreadFactory implements ThreadFactory {
|
||||
static final AtomicInteger poolNumber = new AtomicInteger(1);
|
||||
final ThreadGroup group;
|
||||
final AtomicInteger threadNumber = new AtomicInteger(1);
|
||||
final String namePrefix;
|
||||
|
||||
DefaultThreadFactory() {
|
||||
SecurityManager s = System.getSecurityManager();
|
||||
group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup();
|
||||
namePrefix = "solrcheck-" + poolNumber.getAndIncrement() + "-thread-";
|
||||
}
|
||||
|
||||
public Thread newThread(Runnable r) {
|
||||
Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0);
|
||||
t.setDaemon(true);
|
||||
if (t.getPriority() != Thread.NORM_PRIORITY)
|
||||
t.setPriority(Thread.NORM_PRIORITY);
|
||||
return t;
|
||||
}
|
||||
}
|
||||
|
||||
private class WrappedHttpSolrServer {
|
||||
private final HttpSolrServer httpSolrServer;
|
||||
private int failedPings = 0;
|
||||
|
||||
public WrappedHttpSolrServer(HttpSolrServer httpSolrServer) {
|
||||
this.httpSolrServer = httpSolrServer;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public NamedList<Object> request(SolrRequest paramSolrRequest,
|
||||
String paramString) throws SolrServerException, IOException {
|
||||
int attempts = 0;
|
||||
int startSize = aliveServers.size();
|
||||
Exception ex;
|
||||
String zombieHost = null;
|
||||
for (;;) {
|
||||
WrappedHttpSolrServer httpSolrServer = selectRandomServer();
|
||||
try {
|
||||
if (null != zombieHost && paramSolrRequest.getParams() instanceof SolrQuery) {
|
||||
SolrQuery query = (SolrQuery) paramSolrRequest.getParams();
|
||||
String shards = query.get(ShardParams.SHARDS);
|
||||
if (null != shards) {
|
||||
shards = StringUtils.replace(shards, zombieHost, httpSolrServer.httpSolrServer.getHost());
|
||||
query.set(ShardParams.SHARDS, shards);
|
||||
}
|
||||
}
|
||||
return httpSolrServer.httpSolrServer.request(paramSolrRequest);
|
||||
} catch (SolrServerException e) {
|
||||
if (e.getRootCause() instanceof IOException) {
|
||||
ex = e;
|
||||
zombieHost = httpSolrServer.httpSolrServer.getHost();
|
||||
LOGGER.error("error request solr server: " + zombieHost, e);
|
||||
moveAliveToDead(httpSolrServer);
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
attempts++;
|
||||
if (attempts >= startSize)
|
||||
throw new SolrServerException("No live SolrServers available to handle this request", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void shutdown() {
|
||||
// TODO Auto-generated method stub
|
||||
|
||||
}
|
||||
}
|
||||
public class LBHttpSolrServer extends HttpSolrServer
|
||||
{
|
||||
private static final long serialVersionUID = -6175098554678462150L;
|
||||
private static final Logger LOGGER;
|
||||
private static final SolrQuery SOLR_QUERY;
|
||||
private static final int CHECK_INTERVAL = 10000;
|
||||
private final String coreName;
|
||||
private final ReentrantLock checkLock;
|
||||
private ScheduledExecutorService aliveCheckExecutor;
|
||||
private final CopyOnWriteArrayList<WrappedHttpSolrServer> aliveServers;
|
||||
private final CopyOnWriteArrayList<WrappedHttpSolrServer> zombieServers;
|
||||
|
||||
public LBHttpSolrServer(final HttpSolrServer... httpSolrServers) {
|
||||
this.checkLock = new ReentrantLock();
|
||||
this.aliveServers = new CopyOnWriteArrayList<WrappedHttpSolrServer>();
|
||||
this.zombieServers = new CopyOnWriteArrayList<WrappedHttpSolrServer>();
|
||||
if (null == httpSolrServers || httpSolrServers.length == 0) {
|
||||
throw new IllegalArgumentException("length of httpSolrServers must greater than 0");
|
||||
}
|
||||
for (final HttpSolrServer httpSolrServer : httpSolrServers) {
|
||||
this.aliveServers.add(new WrappedHttpSolrServer(httpSolrServer));
|
||||
}
|
||||
this.coreName = httpSolrServers[0].getCoreName();
|
||||
}
|
||||
|
||||
private WrappedHttpSolrServer selectRandomServer() {
|
||||
while (!this.aliveServers.isEmpty()) {
|
||||
try {
|
||||
if (this.aliveServers.size() == 1) {
|
||||
return this.aliveServers.get(0);
|
||||
}
|
||||
return this.aliveServers.get(RandomUtils.nextInt(this.aliveServers.size()));
|
||||
}
|
||||
catch (IndexOutOfBoundsException e) {}
|
||||
}
|
||||
throw new RuntimeException("no solr server available, all are down!");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
for (final WrappedHttpSolrServer server : this.aliveServers) {
|
||||
LBHttpSolrServer.LOGGER.info("---" + server.httpSolrServer.getHost());
|
||||
}
|
||||
return this.selectRandomServer().httpSolrServer.getHost();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getCoreName() {
|
||||
return this.coreName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reloadCore() throws SolrException {
|
||||
}
|
||||
|
||||
private void moveAliveToDead(final WrappedHttpSolrServer solrServer) {
|
||||
this.checkLock.lock();
|
||||
try {
|
||||
final boolean result = this.aliveServers.remove(solrServer);
|
||||
if (result && this.zombieServers.addIfAbsent(solrServer)) {
|
||||
if (LBHttpSolrServer.LOGGER.isWarnEnabled()) {
|
||||
LBHttpSolrServer.LOGGER.warn("solr server [" + solrServer.httpSolrServer.getHost() + "] is marked as down.");
|
||||
}
|
||||
this.startAliveCheckExecutor();
|
||||
}
|
||||
}
|
||||
finally {
|
||||
this.checkLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private void checkAZombieServer(final WrappedHttpSolrServer zombieServer) {
|
||||
this.checkLock.lock();
|
||||
try {
|
||||
if (LBHttpSolrServer.LOGGER.isInfoEnabled()) {
|
||||
LBHttpSolrServer.LOGGER.info("check solr server [" + zombieServer.httpSolrServer.getHost() + "] ...");
|
||||
}
|
||||
final QueryResponse resp = zombieServer.httpSolrServer.query((SolrParams)LBHttpSolrServer.SOLR_QUERY);
|
||||
if (resp.getStatus() == 0) {
|
||||
this.zombieServers.remove(zombieServer);
|
||||
this.aliveServers.add(zombieServer);
|
||||
if (LBHttpSolrServer.LOGGER.isInfoEnabled()) {
|
||||
LBHttpSolrServer.LOGGER.info("solr server [" + zombieServer.httpSolrServer.getHost() + "] has come back.");
|
||||
}
|
||||
zombieServer.failedPings = 0;
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
zombieServer.failedPings++;
|
||||
}
|
||||
finally {
|
||||
this.checkLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private void startAliveCheckExecutor() {
|
||||
if (this.aliveCheckExecutor == null) {
|
||||
synchronized (this) {
|
||||
if (this.aliveCheckExecutor == null) {
|
||||
(this.aliveCheckExecutor = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory())).scheduleAtFixedRate(getAliveCheckRunner(new WeakReference<LBHttpSolrServer>(this)), 10000L, 10000L, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static Runnable getAliveCheckRunner(final WeakReference<LBHttpSolrServer> lbHttpSolrServer) {
|
||||
return new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
final LBHttpSolrServer solrServer = (LBHttpSolrServer)lbHttpSolrServer.get();
|
||||
if (solrServer != null && solrServer.zombieServers != null) {
|
||||
for (final WrappedHttpSolrServer zombieServer : solrServer.zombieServers) {
|
||||
solrServer.checkAZombieServer(zombieServer);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public NamedList<Object> request(final SolrRequest paramSolrRequest, final String paramString) throws SolrServerException, IOException {
|
||||
int attempts = 0;
|
||||
final int startSize = this.aliveServers.size();
|
||||
String zombieHost = null;
|
||||
while (true) {
|
||||
final WrappedHttpSolrServer httpSolrServer = this.selectRandomServer();
|
||||
try {
|
||||
if (null != zombieHost && paramSolrRequest.getParams() instanceof SolrQuery) {
|
||||
final SolrQuery query = (SolrQuery)paramSolrRequest.getParams();
|
||||
String shards = query.get("shards");
|
||||
if (null != shards) {
|
||||
shards = StringUtils.replace(shards, zombieHost, httpSolrServer.httpSolrServer.getHost());
|
||||
query.set("shards", shards);
|
||||
}
|
||||
}
|
||||
return httpSolrServer.httpSolrServer.request(paramSolrRequest);
|
||||
}
|
||||
catch (SolrServerException e) {
|
||||
if (!(e.getRootCause() instanceof IOException)) {
|
||||
throw e;
|
||||
}
|
||||
final Exception ex = (Exception)e;
|
||||
zombieHost = httpSolrServer.httpSolrServer.getHost();
|
||||
LBHttpSolrServer.LOGGER.error("error request solr server: " + zombieHost, (Throwable)e);
|
||||
this.moveAliveToDead(httpSolrServer);
|
||||
if (++attempts >= startSize) {
|
||||
throw new SolrServerException("No live SolrServers available to handle this request", (Throwable)ex);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void shutdown() {
|
||||
}
|
||||
|
||||
static {
|
||||
LOGGER = LoggerFactory.getLogger(LBHttpSolrServer.class);
|
||||
SOLR_QUERY = new SolrQuery("*:*");
|
||||
}
|
||||
|
||||
static class DefaultThreadFactory implements ThreadFactory
|
||||
{
|
||||
static final AtomicInteger poolNumber;
|
||||
final ThreadGroup group;
|
||||
final AtomicInteger threadNumber;
|
||||
final String namePrefix;
|
||||
|
||||
DefaultThreadFactory() {
|
||||
this.threadNumber = new AtomicInteger(1);
|
||||
final SecurityManager s = System.getSecurityManager();
|
||||
this.group = ((s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup());
|
||||
this.namePrefix = "solrcheck-" + DefaultThreadFactory.poolNumber.getAndIncrement() + "-thread-";
|
||||
}
|
||||
|
||||
@Override
|
||||
public Thread newThread(final Runnable r) {
|
||||
final Thread t = new Thread(this.group, r, this.namePrefix + this.threadNumber.getAndIncrement(), 0L);
|
||||
t.setDaemon(true);
|
||||
if (t.getPriority() != 5) {
|
||||
t.setPriority(5);
|
||||
}
|
||||
return t;
|
||||
}
|
||||
|
||||
static {
|
||||
poolNumber = new AtomicInteger(1);
|
||||
}
|
||||
}
|
||||
|
||||
private class WrappedHttpSolrServer
|
||||
{
|
||||
private final HttpSolrServer httpSolrServer;
|
||||
private int failedPings;
|
||||
|
||||
public WrappedHttpSolrServer(final HttpSolrServer httpSolrServer) {
|
||||
this.failedPings = 0;
|
||||
this.httpSolrServer = httpSolrServer;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+18
-20
@@ -1,23 +1,21 @@
|
||||
package com.fangjia.datastore.solr;
|
||||
|
||||
public class SolrException extends Exception {
|
||||
|
||||
private static final long serialVersionUID = -1105081175581071875L;
|
||||
|
||||
public SolrException() {
|
||||
super();
|
||||
}
|
||||
|
||||
public SolrException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
|
||||
public SolrException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public SolrException(Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
|
||||
public class SolrException extends Exception
|
||||
{
|
||||
private static final long serialVersionUID = -1105081175581071875L;
|
||||
|
||||
public SolrException() {
|
||||
}
|
||||
|
||||
public SolrException(final String message, final Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
|
||||
public SolrException(final String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public SolrException(final Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
}
|
||||
|
||||
+170
-171
@@ -1,177 +1,176 @@
|
||||
package com.fangjia.datastore.solr;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.TimeZone;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.commons.lang.time.FastDateFormat;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.queryParser.QueryParser;
|
||||
import org.apache.lucene.search.BooleanClause;
|
||||
import org.apache.lucene.search.BooleanQuery;
|
||||
import org.apache.lucene.search.NumericRangeQuery;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.TermQuery;
|
||||
import org.apache.lucene.search.TermRangeQuery;
|
||||
import java.util.ArrayList;
|
||||
import org.apache.solr.client.solrj.util.ClientUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import org.apache.lucene.queryParser.QueryParser;
|
||||
import org.apache.lucene.search.TermQuery;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.TermRangeQuery;
|
||||
import java.util.Date;
|
||||
import org.apache.lucene.search.NumericRangeQuery;
|
||||
import org.apache.lucene.search.BooleanClause;
|
||||
import org.apache.commons.lang.time.FastDateFormat;
|
||||
import org.apache.lucene.search.BooleanQuery;
|
||||
|
||||
public class SolrQueryBuilder {
|
||||
private BooleanQuery booleanQuery = new BooleanQuery();
|
||||
public final static FastDateFormat DATE_FORMAT = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", TimeZone
|
||||
.getTimeZone("UTC"));
|
||||
/**
|
||||
* 约定:以$$开头的字符串不需要特殊字符转义
|
||||
*/
|
||||
public final static String NO_ESCAPSE = "$$";
|
||||
|
||||
public SolrQueryBuilder range(String field, Object min, Object max, boolean minInclusive, boolean maxInclusive,
|
||||
BooleanClause.Occur occur) {
|
||||
if (min != null || max != null) {
|
||||
Object obj = null == min ? max : min;
|
||||
Query query = null;
|
||||
if (Integer.class.isInstance(obj)) {
|
||||
query = NumericRangeQuery.newIntRange(field, (Integer) min, (Integer) max, minInclusive, maxInclusive);
|
||||
} else if (Long.class.isInstance(obj)) {
|
||||
query = NumericRangeQuery.newLongRange(field, (Long) min, (Long) max, minInclusive, maxInclusive);
|
||||
} else if (Double.class.isInstance(obj)) {
|
||||
query = NumericRangeQuery.newDoubleRange(field, (Double) min, (Double) max, minInclusive, maxInclusive);
|
||||
} else if (Float.class.isInstance(obj)) {
|
||||
query = NumericRangeQuery.newFloatRange(field, (Float) min, (Float) max, minInclusive, maxInclusive);
|
||||
} else if (Date.class.isInstance(obj)) {
|
||||
query = new TermRangeQuery(field, null == min ? null : DATE_FORMAT.format((Date) min), null == max ? null
|
||||
: DATE_FORMAT.format((Date) max), minInclusive, maxInclusive);
|
||||
} else {
|
||||
query = new TermRangeQuery(field, null == min ? (String) min : min.toString(), null == max ? (String) max : max
|
||||
.toString(), minInclusive, maxInclusive);
|
||||
}
|
||||
booleanQuery.add(query, occur);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public SolrQueryBuilder range(String field, Object min, Object max, boolean minInclusive, boolean maxInclusive) {
|
||||
return range(field, min, max, minInclusive, maxInclusive, BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder range(String field, Object min, Object max) {
|
||||
return range(field, min, max, true, true, BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder range(String field, Object min, Object max, BooleanClause.Occur occur) {
|
||||
return range(field, min, max, true, true, occur);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder gt(String field, Object value) {
|
||||
return range(field, value, null, false, true, BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder gt(String field, Object value, BooleanClause.Occur occur) {
|
||||
return range(field, value, null, false, true, occur);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder gte(String field, Object value) {
|
||||
return range(field, value, null, true, true, BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder gte(String field, Object value, BooleanClause.Occur occur) {
|
||||
return range(field, value, null, true, true, occur);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder lt(String field, Object value) {
|
||||
return range(field, null, value, true, false, BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder lt(String field, Object value, BooleanClause.Occur occur) {
|
||||
return range(field, null, value, true, false, occur);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder lte(String field, Object value) {
|
||||
return range(field, null, value, true, true, BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder lte(String field, Object value, BooleanClause.Occur occur) {
|
||||
return range(field, null, value, true, true, occur);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder eq(String field, Object value, BooleanClause.Occur occur) {
|
||||
if (null != value) {
|
||||
Query query = new TermQuery(new Term(field, value.toString()));
|
||||
booleanQuery.add(query, occur);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public SolrQueryBuilder eq(String field, Object value) {
|
||||
return eq(field, getRealValue(value), BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder eq(String field, Object value, boolean escapse) {
|
||||
return eq(field, escapse ? QueryParser.escape(value.toString()) : value.toString(), BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder ne(String field, Object value) {
|
||||
return eq(field, getRealValue(value), BooleanClause.Occur.MUST_NOT);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder in(String field, List<Object> value, boolean escapse, BooleanClause.Occur occur) {
|
||||
if (null == value || value.isEmpty()) {
|
||||
throw new IllegalArgumentException("in query with null or empty list!!");
|
||||
}
|
||||
BooleanQuery bQuery = new BooleanQuery();
|
||||
for (Object v : value) {
|
||||
Query query = new TermQuery(new Term(field, escapse ? QueryParser.escape(v.toString()) : v.toString()));
|
||||
bQuery.add(query, BooleanClause.Occur.SHOULD);
|
||||
}
|
||||
booleanQuery.add(bQuery, occur);
|
||||
return this;
|
||||
}
|
||||
|
||||
public SolrQueryBuilder in(String field, List<Object> value) {
|
||||
return in(field, getRealValue(value), false, BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder in(String field, List<Object> value, BooleanClause.Occur occur) {
|
||||
return in(field, getRealValue(value), false, occur);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder in(String field, List<Object> value, boolean escapse) {
|
||||
return in(field, value, escapse, BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public Query toQuery() {
|
||||
return this.booleanQuery;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取真实value<br/>
|
||||
* 如果是非string的value,返回转义后的原值<br/>
|
||||
* 如果是string型,如果以$$开头,则截取$$后面的字符且不需要转义
|
||||
*
|
||||
* @param value
|
||||
* @return
|
||||
*/
|
||||
private Object getRealValue(Object value) {
|
||||
if (value instanceof String) {
|
||||
if (NO_ESCAPSE.equals(value)) {
|
||||
return value;
|
||||
} else if (StringUtils.startsWith((String) value, NO_ESCAPSE)) {
|
||||
return StringUtils.substring((String) value, NO_ESCAPSE.length());
|
||||
}
|
||||
}
|
||||
return ClientUtils.escapeQueryChars(value.toString());
|
||||
// return QueryParser.escape(value.toString());
|
||||
}
|
||||
|
||||
private List<Object> getRealValue(List<Object> value) {
|
||||
if (null == value || value.isEmpty()) {
|
||||
return value;
|
||||
}
|
||||
List<Object> ret = new ArrayList<Object>(value.size());
|
||||
for (Object v : value) {
|
||||
ret.add(getRealValue(v));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
public class SolrQueryBuilder
|
||||
{
|
||||
private BooleanQuery booleanQuery;
|
||||
public static final FastDateFormat DATE_FORMAT;
|
||||
public static final String NO_ESCAPSE = "$$";
|
||||
|
||||
public SolrQueryBuilder() {
|
||||
this.booleanQuery = new BooleanQuery();
|
||||
}
|
||||
|
||||
public SolrQueryBuilder range(final String field, final Object min, final Object max, final boolean minInclusive, final boolean maxInclusive, final BooleanClause.Occur occur) {
|
||||
if (min != null || max != null) {
|
||||
final Object obj = (null == min) ? max : min;
|
||||
Query query = null;
|
||||
if (Integer.class.isInstance(obj)) {
|
||||
query = (Query)NumericRangeQuery.newIntRange(field, (Integer)min, (Integer)max, minInclusive, maxInclusive);
|
||||
}
|
||||
else if (Long.class.isInstance(obj)) {
|
||||
query = (Query)NumericRangeQuery.newLongRange(field, (Long)min, (Long)max, minInclusive, maxInclusive);
|
||||
}
|
||||
else if (Double.class.isInstance(obj)) {
|
||||
query = (Query)NumericRangeQuery.newDoubleRange(field, (Double)min, (Double)max, minInclusive, maxInclusive);
|
||||
}
|
||||
else if (Float.class.isInstance(obj)) {
|
||||
query = (Query)NumericRangeQuery.newFloatRange(field, (Float)min, (Float)max, minInclusive, maxInclusive);
|
||||
}
|
||||
else if (Date.class.isInstance(obj)) {
|
||||
query = (Query)new TermRangeQuery(field, (null == min) ? null : SolrQueryBuilder.DATE_FORMAT.format((Date)min), (null == max) ? null : SolrQueryBuilder.DATE_FORMAT.format((Date)max), minInclusive, maxInclusive);
|
||||
}
|
||||
else {
|
||||
query = (Query)new TermRangeQuery(field, (null == min) ? ((String)min) : min.toString(), (null == max) ? ((String)max) : max.toString(), minInclusive, maxInclusive);
|
||||
}
|
||||
this.booleanQuery.add(query, occur);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public SolrQueryBuilder range(final String field, final Object min, final Object max, final boolean minInclusive, final boolean maxInclusive) {
|
||||
return this.range(field, min, max, minInclusive, maxInclusive, BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder range(final String field, final Object min, final Object max) {
|
||||
return this.range(field, min, max, true, true, BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder range(final String field, final Object min, final Object max, final BooleanClause.Occur occur) {
|
||||
return this.range(field, min, max, true, true, occur);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder gt(final String field, final Object value) {
|
||||
return this.range(field, value, null, false, true, BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder gt(final String field, final Object value, final BooleanClause.Occur occur) {
|
||||
return this.range(field, value, null, false, true, occur);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder gte(final String field, final Object value) {
|
||||
return this.range(field, value, null, true, true, BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder gte(final String field, final Object value, final BooleanClause.Occur occur) {
|
||||
return this.range(field, value, null, true, true, occur);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder lt(final String field, final Object value) {
|
||||
return this.range(field, null, value, true, false, BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder lt(final String field, final Object value, final BooleanClause.Occur occur) {
|
||||
return this.range(field, null, value, true, false, occur);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder lte(final String field, final Object value) {
|
||||
return this.range(field, null, value, true, true, BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder lte(final String field, final Object value, final BooleanClause.Occur occur) {
|
||||
return this.range(field, null, value, true, true, occur);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder eq(final String field, final Object value, final BooleanClause.Occur occur) {
|
||||
if (null != value) {
|
||||
final Query query = (Query)new TermQuery(new Term(field, value.toString()));
|
||||
this.booleanQuery.add(query, occur);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public SolrQueryBuilder eq(final String field, final Object value) {
|
||||
return this.eq(field, this.getRealValue(value), BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder eq(final String field, final Object value, final boolean escapse) {
|
||||
return this.eq(field, escapse ? QueryParser.escape(value.toString()) : value.toString(), BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder ne(final String field, final Object value) {
|
||||
return this.eq(field, this.getRealValue(value), BooleanClause.Occur.MUST_NOT);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder in(final String field, final List<Object> value, final boolean escapse, final BooleanClause.Occur occur) {
|
||||
if (null == value || value.isEmpty()) {
|
||||
throw new IllegalArgumentException("in query with null or empty list!!");
|
||||
}
|
||||
final BooleanQuery bQuery = new BooleanQuery();
|
||||
for (final Object v : value) {
|
||||
final Query query = (Query)new TermQuery(new Term(field, escapse ? QueryParser.escape(v.toString()) : v.toString()));
|
||||
bQuery.add(query, BooleanClause.Occur.SHOULD);
|
||||
}
|
||||
this.booleanQuery.add((Query)bQuery, occur);
|
||||
return this;
|
||||
}
|
||||
|
||||
public SolrQueryBuilder in(final String field, final List<Object> value) {
|
||||
return this.in(field, this.getRealValue(value), false, BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder in(final String field, final List<Object> value, final BooleanClause.Occur occur) {
|
||||
return this.in(field, this.getRealValue(value), false, occur);
|
||||
}
|
||||
|
||||
public SolrQueryBuilder in(final String field, final List<Object> value, final boolean escapse) {
|
||||
return this.in(field, value, escapse, BooleanClause.Occur.MUST);
|
||||
}
|
||||
|
||||
public Query toQuery() {
|
||||
return (Query)this.booleanQuery;
|
||||
}
|
||||
|
||||
private Object getRealValue(final Object value) {
|
||||
if (value instanceof String) {
|
||||
if ("$$".equals(value)) {
|
||||
return value;
|
||||
}
|
||||
if (StringUtils.startsWith((String)value, "$$")) {
|
||||
return StringUtils.substring((String)value, "$$".length());
|
||||
}
|
||||
}
|
||||
return ClientUtils.escapeQueryChars(value.toString());
|
||||
}
|
||||
|
||||
private List<Object> getRealValue(final List<Object> value) {
|
||||
if (null == value || value.isEmpty()) {
|
||||
return value;
|
||||
}
|
||||
final List<Object> ret = new ArrayList<Object>(value.size());
|
||||
for (final Object v : value) {
|
||||
ret.add(this.getRealValue(v));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
static {
|
||||
DATE_FORMAT = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", TimeZone.getTimeZone("UTC"));
|
||||
}
|
||||
}
|
||||
|
||||
+20
-20
@@ -1,26 +1,26 @@
|
||||
package com.fangjia.datastore.solr;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Collection;
|
||||
import java.util.Set;
|
||||
|
||||
import java.util.Collection;
|
||||
import com.fangjia.datastore.config.Reloadable;
|
||||
|
||||
public interface SolrSchema extends Reloadable {
|
||||
public final static String VERTION = "_version";
|
||||
public final static String ID = "_id";
|
||||
|
||||
int saveIndex(Collection<Index> indexes) throws SolrException;
|
||||
|
||||
boolean removeIndex(String name) throws SolrException;
|
||||
|
||||
Index getIndex(String name);
|
||||
|
||||
Collection<Index> getIndexes();
|
||||
|
||||
Set<String> getIndexNames();
|
||||
|
||||
File getSchemaFile();
|
||||
|
||||
void setSchemaFile(File schemaFile);
|
||||
}
|
||||
public interface SolrSchema extends Reloadable
|
||||
{
|
||||
public static final String VERTION = "_version";
|
||||
public static final String ID = "_id";
|
||||
|
||||
int saveIndex(final Collection<Index> p0) throws SolrException;
|
||||
|
||||
boolean removeIndex(final String p0) throws SolrException;
|
||||
|
||||
Index getIndex(final String p0);
|
||||
|
||||
Collection<Index> getIndexes();
|
||||
|
||||
Set<String> getIndexNames();
|
||||
|
||||
File getSchemaFile();
|
||||
|
||||
void setSchemaFile(final File p0);
|
||||
}
|
||||
|
||||
+186
-181
@@ -1,189 +1,194 @@
|
||||
package com.fangjia.datastore.solr;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.dom4j.Document;
|
||||
import java.io.OutputStream;
|
||||
import org.dom4j.io.XMLWriter;
|
||||
import java.io.FileOutputStream;
|
||||
import org.dom4j.DocumentException;
|
||||
import java.util.Set;
|
||||
import java.util.List;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.dom4j.Node;
|
||||
import java.util.Iterator;
|
||||
import org.dom4j.DocumentHelper;
|
||||
import org.dom4j.Element;
|
||||
import org.dom4j.Node;
|
||||
import org.dom4j.io.SAXReader;
|
||||
import org.dom4j.io.XMLWriter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.Collection;
|
||||
import com.fangjia.datastore.ConfigurationException;
|
||||
import com.fangjia.datastore.solr.Index.IndexTypeEnum;
|
||||
import java.util.HashMap;
|
||||
import org.slf4j.Logger;
|
||||
import org.dom4j.Document;
|
||||
import java.util.Map;
|
||||
import java.io.File;
|
||||
import org.dom4j.io.SAXReader;
|
||||
|
||||
public class SolrSchemaImpl implements SolrSchema {
|
||||
private final SAXReader reader = new SAXReader();
|
||||
private File schemaFile;
|
||||
private final Map<String, Index> indexes = new HashMap<String, Index>();
|
||||
private Document document;
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(SolrSchemaImpl.class);
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
new SolrSchemaImpl(new File("D:\\viewstore\\solrhome\\house\\conf\\schema.xml"));
|
||||
}
|
||||
|
||||
public SolrSchemaImpl(File schemaFile) throws ConfigurationException {
|
||||
if (!schemaFile.exists() || schemaFile.isDirectory()) {
|
||||
throw new IllegalArgumentException("schema file not exists or is a directory:" + schemaFile);
|
||||
}
|
||||
this.schemaFile = schemaFile;
|
||||
reload();
|
||||
}
|
||||
|
||||
public int saveIndex(Collection<Index> newIndexes) throws SolrException {
|
||||
int ret = 0;
|
||||
try {
|
||||
for (Index index : newIndexes) {
|
||||
Index i = indexes.get(index.getName());
|
||||
if (index.equals(i)) {
|
||||
continue;
|
||||
}
|
||||
ret++;
|
||||
Element e;
|
||||
if (indexes.containsKey(index.getName())) {
|
||||
String xpath = "//schema//fields//" + (index.isDynamic() ? "dynamicField" : "field") + "[name=\""
|
||||
+ index.getName() + "\"]";
|
||||
e = (Element) this.document.selectSingleNode(xpath);
|
||||
} else {
|
||||
e = DocumentHelper.createElement(index.isDynamic() ? "dynamicField" : "field");
|
||||
Element fields = (Element) document.selectSingleNode("//schema//fields");
|
||||
fields.add(e);
|
||||
}
|
||||
e.addAttribute("name", index.getName());
|
||||
e.addAttribute("type", index.getType().getType());
|
||||
e.addAttribute("stored", String.valueOf(index.isStored()));
|
||||
e.addAttribute("indexed", String.valueOf(index.isIndexed()));
|
||||
e.addAttribute("multiValued", String.valueOf(index.isMultiValued()));
|
||||
e.addAttribute("comments", String.valueOf(index.getComments()));
|
||||
this.indexes.put(index.getName(), index);
|
||||
}
|
||||
} finally {
|
||||
write();
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public synchronized boolean removeIndex(String name) throws SolrException {
|
||||
Index index = this.indexes.get(name);
|
||||
if (null == index) {
|
||||
return false;
|
||||
}
|
||||
if (ID.equals(name) || VERTION.equals(name)) {
|
||||
throw new SolrException("delete of field \"" + name + "\" is not allowed!");
|
||||
}
|
||||
String xpath = "//schema//fields//" + (index.isDynamic() ? "dynamicField" : "field") + "[name=\"" + index.getName()
|
||||
+ "\"]";
|
||||
Element e = (Element) this.document.selectSingleNode(xpath);
|
||||
if (null != e) {
|
||||
e.detach();
|
||||
}
|
||||
write();
|
||||
this.indexes.remove(name);
|
||||
return true;
|
||||
}
|
||||
|
||||
public Collection<Index> getIndexes() {
|
||||
return indexes.values();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public void reload() throws ConfigurationException {
|
||||
if (LOGGER.isDebugEnabled()) {
|
||||
LOGGER.debug("refresh solr schema: {}", this.schemaFile);
|
||||
}
|
||||
Map<String, Index> newIndexes = new HashMap<String, Index>();
|
||||
|
||||
Document newDocument = read();
|
||||
|
||||
List<Node> list = newDocument.selectNodes("//schema");
|
||||
if (null == list || list.size() > 1) {
|
||||
throw new ConfigurationException("zero or more than one \"fields\" tag found, expect only one \"fields\" tag");
|
||||
}
|
||||
Node fieldsNode = list.get(0);
|
||||
list = fieldsNode.selectNodes("field");
|
||||
for (Node node : list) {
|
||||
Element e = (Element) node;
|
||||
|
||||
String name = e.attributeValue("name");
|
||||
if (StringUtils.isEmpty(name)) {
|
||||
throw new ConfigurationException("invalid field name \"" + name + "\" in solr schema: " + this.schemaFile);
|
||||
}
|
||||
|
||||
IndexTypeEnum type = IndexTypeEnum.parse(e.attributeValue("type"));
|
||||
if (null == type) {
|
||||
throw new ConfigurationException("invalid field type \"" + e.attributeValue("type") + "\" in solr schema: "
|
||||
+ this.schemaFile);
|
||||
}
|
||||
|
||||
boolean stored = Boolean.parseBoolean(e.attributeValue("stored"));
|
||||
boolean indexed = Boolean.parseBoolean(e.attributeValue("indexed"));
|
||||
boolean multiValued = Boolean.parseBoolean(e.attributeValue("multiValued"));
|
||||
String comments = e.attributeValue("comments");
|
||||
boolean dynamic = e.getName().equals("dynamicField");
|
||||
|
||||
Index index = new Index(name, type, stored, indexed, multiValued, dynamic, comments);
|
||||
newIndexes.put(name, index);
|
||||
}
|
||||
if (!newIndexes.containsKey(ID)) {
|
||||
throw new ConfigurationException("no field names \"" + ID + "\" find in schema: " + this.schemaFile);
|
||||
}
|
||||
if (!newIndexes.containsKey(VERTION)) {
|
||||
throw new ConfigurationException("no field names \"" + VERTION + "\" find in schema: " + this.schemaFile);
|
||||
}
|
||||
indexes.clear();
|
||||
indexes.putAll(newIndexes);
|
||||
this.document = newDocument;
|
||||
|
||||
}
|
||||
|
||||
public Set<String> getIndexNames() {
|
||||
return this.indexes.keySet();
|
||||
}
|
||||
|
||||
private Document read() throws ConfigurationException {
|
||||
try {
|
||||
return reader.read(schemaFile);
|
||||
} catch (DocumentException e) {
|
||||
throw new ConfigurationException("error read solr xml file: " + this, e);
|
||||
}
|
||||
}
|
||||
|
||||
private void write() throws SolrException {
|
||||
if (LOGGER.isDebugEnabled()) {
|
||||
LOGGER.debug("write solr schema: {}" + this.schemaFile);
|
||||
}
|
||||
XMLWriter writer;
|
||||
try {
|
||||
writer = new XMLWriter(new FileOutputStream(schemaFile));
|
||||
writer.write(document);
|
||||
writer.close();
|
||||
} catch (IOException e) {
|
||||
throw new SolrException("error write solr xml file: " + this, e);
|
||||
}
|
||||
}
|
||||
|
||||
public Index getIndex(String name) {
|
||||
return this.indexes.get(name);
|
||||
}
|
||||
|
||||
public File getSchemaFile() {
|
||||
return schemaFile;
|
||||
}
|
||||
|
||||
public void setSchemaFile(File schemaFile) {
|
||||
this.schemaFile = schemaFile;
|
||||
}
|
||||
|
||||
}
|
||||
public class SolrSchemaImpl implements SolrSchema
|
||||
{
|
||||
private final SAXReader reader;
|
||||
private File schemaFile;
|
||||
private final Map<String, Index> indexes;
|
||||
private Document document;
|
||||
private static final Logger LOGGER;
|
||||
|
||||
public static void main(final String[] args) throws Exception {
|
||||
new SolrSchemaImpl(new File("D:\\viewstore\\solrhome\\house\\conf\\schema.xml"));
|
||||
}
|
||||
|
||||
public SolrSchemaImpl(final File schemaFile) throws ConfigurationException {
|
||||
this.reader = new SAXReader();
|
||||
this.indexes = new HashMap<String, Index>();
|
||||
if (!schemaFile.exists() || schemaFile.isDirectory()) {
|
||||
throw new IllegalArgumentException("schema file not exists or is a directory:" + schemaFile);
|
||||
}
|
||||
this.schemaFile = schemaFile;
|
||||
this.reload();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int saveIndex(final Collection<Index> newIndexes) throws SolrException {
|
||||
int ret = 0;
|
||||
try {
|
||||
for (final Index index : newIndexes) {
|
||||
final Index i = this.indexes.get(index.getName());
|
||||
if (index.equals(i)) {
|
||||
continue;
|
||||
}
|
||||
++ret;
|
||||
Element e;
|
||||
if (this.indexes.containsKey(index.getName())) {
|
||||
final String xpath = "//schema//fields//" + (index.isDynamic() ? "dynamicField" : "field") + "[name=\"" + index.getName() + "\"]";
|
||||
e = (Element)this.document.selectSingleNode(xpath);
|
||||
}
|
||||
else {
|
||||
e = DocumentHelper.createElement(index.isDynamic() ? "dynamicField" : "field");
|
||||
final Element fields = (Element)this.document.selectSingleNode("//schema//fields");
|
||||
fields.add(e);
|
||||
}
|
||||
e.addAttribute("name", index.getName());
|
||||
e.addAttribute("type", index.getType().getType());
|
||||
e.addAttribute("stored", String.valueOf(index.isStored()));
|
||||
e.addAttribute("indexed", String.valueOf(index.isIndexed()));
|
||||
e.addAttribute("multiValued", String.valueOf(index.isMultiValued()));
|
||||
e.addAttribute("comments", String.valueOf(index.getComments()));
|
||||
this.indexes.put(index.getName(), index);
|
||||
}
|
||||
}
|
||||
finally {
|
||||
this.write();
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized boolean removeIndex(final String name) throws SolrException {
|
||||
final Index index = this.indexes.get(name);
|
||||
if (null == index) {
|
||||
return false;
|
||||
}
|
||||
if ("_id".equals(name) || "_version".equals(name)) {
|
||||
throw new SolrException("delete of field \"" + name + "\" is not allowed!");
|
||||
}
|
||||
final String xpath = "//schema//fields//" + (index.isDynamic() ? "dynamicField" : "field") + "[name=\"" + index.getName() + "\"]";
|
||||
final Element e = (Element)this.document.selectSingleNode(xpath);
|
||||
if (null != e) {
|
||||
e.detach();
|
||||
}
|
||||
this.write();
|
||||
this.indexes.remove(name);
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<Index> getIndexes() {
|
||||
return this.indexes.values();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reload() throws ConfigurationException {
|
||||
if (SolrSchemaImpl.LOGGER.isDebugEnabled()) {
|
||||
SolrSchemaImpl.LOGGER.debug("refresh solr schema: {}", (Object)this.schemaFile);
|
||||
}
|
||||
final Map<String, Index> newIndexes = new HashMap<String, Index>();
|
||||
final Document newDocument = this.read();
|
||||
List<Node> list = (List<Node>)newDocument.selectNodes("//schema");
|
||||
if (null == list || list.size() > 1) {
|
||||
throw new ConfigurationException("zero or more than one \"fields\" tag found, expect only one \"fields\" tag");
|
||||
}
|
||||
final Node fieldsNode = list.get(0);
|
||||
list = (List<Node>)fieldsNode.selectNodes("field");
|
||||
for (final Node node : list) {
|
||||
final Element e = (Element)node;
|
||||
final String name = e.attributeValue("name");
|
||||
if (StringUtils.isEmpty(name)) {
|
||||
throw new ConfigurationException("invalid field name \"" + name + "\" in solr schema: " + this.schemaFile);
|
||||
}
|
||||
final Index.IndexTypeEnum type = Index.IndexTypeEnum.parse(e.attributeValue("type"));
|
||||
if (null == type) {
|
||||
throw new ConfigurationException("invalid field type \"" + e.attributeValue("type") + "\" in solr schema: " + this.schemaFile);
|
||||
}
|
||||
final boolean stored = Boolean.parseBoolean(e.attributeValue("stored"));
|
||||
final boolean indexed = Boolean.parseBoolean(e.attributeValue("indexed"));
|
||||
final boolean multiValued = Boolean.parseBoolean(e.attributeValue("multiValued"));
|
||||
final String comments = e.attributeValue("comments");
|
||||
final boolean dynamic = e.getName().equals("dynamicField");
|
||||
final Index index = new Index(name, type, stored, indexed, multiValued, dynamic, comments);
|
||||
newIndexes.put(name, index);
|
||||
}
|
||||
if (!newIndexes.containsKey("_id")) {
|
||||
throw new ConfigurationException("no field names \"_id\" find in schema: " + this.schemaFile);
|
||||
}
|
||||
if (!newIndexes.containsKey("_version")) {
|
||||
throw new ConfigurationException("no field names \"_version\" find in schema: " + this.schemaFile);
|
||||
}
|
||||
this.indexes.clear();
|
||||
this.indexes.putAll(newIndexes);
|
||||
this.document = newDocument;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getIndexNames() {
|
||||
return this.indexes.keySet();
|
||||
}
|
||||
|
||||
private Document read() throws ConfigurationException {
|
||||
try {
|
||||
return this.reader.read(this.schemaFile);
|
||||
}
|
||||
catch (DocumentException e) {
|
||||
throw new ConfigurationException("error read solr xml file: " + this, (Throwable)e);
|
||||
}
|
||||
}
|
||||
|
||||
private void write() throws SolrException {
|
||||
if (SolrSchemaImpl.LOGGER.isDebugEnabled()) {
|
||||
SolrSchemaImpl.LOGGER.debug("write solr schema: {}" + this.schemaFile);
|
||||
}
|
||||
try {
|
||||
final XMLWriter writer = new XMLWriter((OutputStream)new FileOutputStream(this.schemaFile));
|
||||
writer.write(this.document);
|
||||
writer.close();
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new SolrException("error write solr xml file: " + this, e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Index getIndex(final String name) {
|
||||
return this.indexes.get(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public File getSchemaFile() {
|
||||
return this.schemaFile;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setSchemaFile(final File schemaFile) {
|
||||
this.schemaFile = schemaFile;
|
||||
}
|
||||
|
||||
static {
|
||||
LOGGER = LoggerFactory.getLogger((Class)SolrSchemaImpl.class);
|
||||
}
|
||||
}
|
||||
|
||||
+16
-17
@@ -1,23 +1,22 @@
|
||||
package com.fangjia.datastore.solr;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.solr.client.solrj.SolrQuery;
|
||||
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||
import org.bson.BSONObject;
|
||||
|
||||
import java.util.List;
|
||||
import com.fangjia.datastore.ConfigurationException;
|
||||
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||
import org.apache.solr.client.solrj.SolrQuery;
|
||||
|
||||
public interface SolrServer {
|
||||
QueryResponse query(SolrQuery params, boolean fromMaster, boolean bulk) throws SolrException;
|
||||
|
||||
void deleteByQuery(String query) throws SolrException;
|
||||
|
||||
void reload() throws ConfigurationException;
|
||||
|
||||
void reloadCores() throws SolrException;
|
||||
|
||||
SolrSchema getSolrSchema();
|
||||
|
||||
void add(List<BSONObject> dbObjects) throws SolrException;
|
||||
public interface SolrServer
|
||||
{
|
||||
QueryResponse query(final SolrQuery p0, final boolean p1, final boolean p2) throws SolrException;
|
||||
|
||||
void deleteByQuery(final String p0) throws SolrException;
|
||||
|
||||
void reload() throws ConfigurationException;
|
||||
|
||||
void reloadCores() throws SolrException;
|
||||
|
||||
SolrSchema getSolrSchema();
|
||||
|
||||
void add(final List<BSONObject> p0) throws SolrException;
|
||||
}
|
||||
|
||||
+448
-468
@@ -1,475 +1,455 @@
|
||||
package com.fangjia.datastore.solr;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.Executor;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.commons.lang.math.RandomUtils;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.config.CookieSpecs;
|
||||
import org.apache.http.client.config.RequestConfig;
|
||||
import org.apache.http.impl.client.DefaultHttpClient;
|
||||
import org.apache.http.impl.client.HttpClients;
|
||||
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
|
||||
import org.apache.solr.client.solrj.SolrQuery;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||
import org.apache.solr.common.SolrException.ErrorCode;
|
||||
import org.apache.solr.common.SolrInputDocument;
|
||||
import org.apache.solr.common.SolrInputField;
|
||||
import org.apache.solr.common.params.ShardParams;
|
||||
import org.bson.BSONObject;
|
||||
import org.bson.types.ObjectId;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.sothis.core.util.BlockingThreadPoolExecutor;
|
||||
|
||||
import com.fangjia.datastore.ConfigurationException;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||
import org.apache.solr.client.solrj.SolrQuery;
|
||||
import org.apache.commons.lang.math.RandomUtils;
|
||||
import java.io.IOException;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import java.util.Collections;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.bson.types.ObjectId;
|
||||
import org.apache.solr.common.SolrInputField;
|
||||
import java.util.Iterator;
|
||||
import java.util.Date;
|
||||
import java.util.ArrayList;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import org.bson.BSONObject;
|
||||
import java.util.Collection;
|
||||
import org.apache.solr.common.SolrInputDocument;
|
||||
import java.util.List;
|
||||
import java.util.HashMap;
|
||||
import java.net.MalformedURLException;
|
||||
import com.fangjia.datastore.config.SolrConfig;
|
||||
import com.fangjia.datastore.ConfigurationException;
|
||||
import org.sothis.core.util.BlockingThreadPoolExecutor;
|
||||
import java.io.File;
|
||||
import org.apache.http.impl.client.DefaultHttpClient;
|
||||
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.Executor;
|
||||
import java.util.Map;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import com.fangjia.datastore.config.StoreConfig;
|
||||
import org.slf4j.Logger;
|
||||
|
||||
public class SolrServerImpl implements SolrServer {
|
||||
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(SolrServerImpl.class);
|
||||
private final static int SOLR_COMMIT_MAX_RETRY_COUNT = 3;
|
||||
private final static int SOLR_QUERY_MAX_RETRY_COUNT = 3;
|
||||
private final static int MAX_NONE_BULK_SKIP = 100000;
|
||||
|
||||
private final String name;
|
||||
private final StoreConfig storeConfig;
|
||||
private final HttpClient httpClient;
|
||||
private int masterCommitSolrServerCount;
|
||||
private final Map<String, HttpSolrServer> allSolrServers;
|
||||
private String[] masterCommitFilters;
|
||||
private HttpSolrServer[][] masterCommitSolrServers;
|
||||
private HttpSolrServer[] masterSolrServers;
|
||||
private HttpSolrServer[] slaveSolrServers;
|
||||
private SolrSchema[] solrSchemas;
|
||||
private final SolrSchema solrSchema;
|
||||
private final Executor solrAddExecutorService;
|
||||
private final AtomicBoolean filtering = new AtomicBoolean(false);
|
||||
private long lastFilterTime = 0;
|
||||
|
||||
public SolrServerImpl(StoreConfig storeConfig) throws ConfigurationException {
|
||||
this.storeConfig = storeConfig;
|
||||
this.name = storeConfig.getName();
|
||||
|
||||
/*MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
|
||||
connectionManager.getParams().setTcpNoDelay(true);
|
||||
connectionManager.getParams().setStaleCheckingEnabled(true);
|
||||
this.httpClient = new HttpClient(connectionManager);
|
||||
httpClient.getParams().setConnectionManagerTimeout(0);
|
||||
DefaultHttpMethodRetryHandler retryhandler = new DefaultHttpMethodRetryHandler(0, false);
|
||||
this.httpClient.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryhandler);
|
||||
*/
|
||||
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
|
||||
cm.setMaxTotal(5);
|
||||
/* RequestConfig requestConfig = RequestConfig.custom()
|
||||
.setConnectTimeout(1000 * 10)
|
||||
.setSocketTimeout(1000 * 10)
|
||||
.setCookieSpec(CookieSpecs.IGNORE_COOKIES)
|
||||
.build();
|
||||
|
||||
this.httpClient = HttpClients.custom()
|
||||
.setDefaultRequestConfig(requestConfig)
|
||||
.setConnectionManager(cm)
|
||||
.build();*/
|
||||
this.httpClient = new DefaultHttpClient();
|
||||
this.allSolrServers = initSolrServers(this.storeConfig, httpClient);
|
||||
this.solrSchema = new SolrSchemaImpl(new File(storeConfig.getMasterSolrConfig().getHome() + "/" + name
|
||||
+ "/conf/managed-schema"));
|
||||
int size = (masterCommitSolrServerCount + 1) * 2;
|
||||
this.solrAddExecutorService = new BlockingThreadPoolExecutor(name + "-SolrAdd", 1, size, size, 60000);
|
||||
}
|
||||
|
||||
private CommonsHttpSolrServer createSolrServer(String coreName, SolrConfig solronfig) throws ConfigurationException {
|
||||
try {
|
||||
CommonsHttpSolrServer solrServer = new CommonsHttpSolrServer(solronfig.getHost(), coreName, httpClient);
|
||||
return solrServer;
|
||||
} catch (MalformedURLException e) {
|
||||
throw new ConfigurationException("[" + this.name + "] error create solr server: ", e);
|
||||
}
|
||||
}
|
||||
|
||||
private HttpSolrServer[] initSolrServer(Map<String, HttpSolrServer> allSolrServers, SolrConfig solrConfig)
|
||||
throws ConfigurationException {
|
||||
SolrConfig[] shardsConfigs = solrConfig.getShardsConfigs();
|
||||
if (shardsConfigs.length == 0) {
|
||||
shardsConfigs = new SolrConfig[] { solrConfig };
|
||||
}
|
||||
HttpSolrServer[] shards = new HttpSolrServer[shardsConfigs.length];
|
||||
for (int i = 0; i < shardsConfigs.length; i++) {
|
||||
SolrConfig shardConfig = shardsConfigs[i];
|
||||
if (!allSolrServers.containsKey(shardConfig.getId())) {
|
||||
SolrConfig[] slaveConfigs = shardConfig.getSlaveConfigs();
|
||||
if (slaveConfigs.length == 0) {
|
||||
allSolrServers.put(shardConfig.getId(), createSolrServer(name, shardConfig));
|
||||
} else {
|
||||
CommonsHttpSolrServer[] slaveServers = new CommonsHttpSolrServer[slaveConfigs.length];
|
||||
for (int j = 0; j < slaveConfigs.length; j++) {
|
||||
SolrConfig slaveConfig = slaveConfigs[j];
|
||||
if (!allSolrServers.containsKey(slaveConfig.getId())) {
|
||||
allSolrServers.put(slaveConfig.getId(), createSolrServer(name, slaveConfig));
|
||||
}
|
||||
slaveServers[j] = (CommonsHttpSolrServer) allSolrServers.get(slaveConfig.getId());
|
||||
}
|
||||
allSolrServers.put(shardConfig.getId(), new LBHttpSolrServer(slaveServers));
|
||||
}
|
||||
}
|
||||
shards[i] = allSolrServers.get(shardConfig.getId());
|
||||
}
|
||||
return shards;
|
||||
}
|
||||
|
||||
private synchronized Map<String, HttpSolrServer> initSolrServers(StoreConfig storeConfig, HttpClient httpClient)
|
||||
throws ConfigurationException {
|
||||
// http connections
|
||||
//httpClient.getHttpConnectionManager().getParams().setDefaultMaxConnectionsPerHost(
|
||||
//storeConfig.getSolrMaxConnectionsPerHost());
|
||||
//httpClient.getHttpConnectionManager().getParams().setMaxTotalConnections(storeConfig.getSolrMaxTotalConnections());
|
||||
|
||||
Map<String, HttpSolrServer> solrServers = new HashMap<String, HttpSolrServer>();
|
||||
SolrConfig[] masterConfigs = storeConfig.getMasterSolrConfigs();
|
||||
masterCommitSolrServers = new HttpSolrServer[masterConfigs.length][];
|
||||
masterCommitFilters = new String[masterConfigs.length];
|
||||
solrSchemas = new SolrSchema[masterConfigs.length];
|
||||
masterCommitSolrServerCount = 0;
|
||||
for (int i = 0; i < masterConfigs.length; i++) {
|
||||
masterCommitSolrServers[i] = initSolrServer(solrServers, masterConfigs[i]);
|
||||
masterCommitFilters[i] = masterConfigs[i].getFilter();
|
||||
solrSchemas[i] = new SolrSchemaImpl(new File(masterConfigs[i].getHome() + "/" + name + "/conf/managed-schema"));
|
||||
masterCommitSolrServerCount += masterCommitSolrServers[i].length;
|
||||
}
|
||||
masterSolrServers = initSolrServer(solrServers, storeConfig.getMasterSolrConfig());
|
||||
slaveSolrServers = initSolrServer(solrServers, storeConfig.getSlaveSolrConfig());
|
||||
|
||||
return solrServers;
|
||||
}
|
||||
|
||||
private void addToSolr0(HttpSolrServer solrServer, List<SolrInputDocument> documents, int errorCount) {
|
||||
try {
|
||||
if (null != documents && !documents.isEmpty()) {
|
||||
solrServer.add(documents);
|
||||
solrServer.commit();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
if (errorCount <= SOLR_COMMIT_MAX_RETRY_COUNT) {
|
||||
errorCount++;
|
||||
long sleepTime = errorCount * errorCount;
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("[" + this.name + "] error commit to solr[" + solrServer.getHost() + "], sleep " + sleepTime
|
||||
+ "s and retry(" + e.getMessage() + ")");
|
||||
}
|
||||
try {
|
||||
Thread.sleep(1000 * sleepTime);
|
||||
} catch (InterruptedException e1) {
|
||||
LOGGER.error("[" + this.name + "] error save to solr: ", e1);
|
||||
}
|
||||
addToSolr0(solrServer, documents, errorCount);
|
||||
} else {
|
||||
if (LOGGER.isErrorEnabled()) {
|
||||
LOGGER.error("[" + this.name + "] error save to solr[" + solrServer.getHost() + "] after "
|
||||
+ SOLR_COMMIT_MAX_RETRY_COUNT + " times retry: ", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void add(List<BSONObject> dbObjects) throws SolrException {
|
||||
if (null == dbObjects || dbObjects.size() == 0) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
long time = System.currentTimeMillis();
|
||||
checkFilter();
|
||||
final CountDownLatch countDownLatch = new CountDownLatch(masterCommitSolrServerCount);
|
||||
for (int j = 0; j < this.masterCommitSolrServers.length; j++) {
|
||||
final int jndex = j;
|
||||
final Map<Integer, List<BSONObject>> shardedObjectMap = shard(dbObjects, this.masterCommitSolrServers[j].length);
|
||||
for (int i = 0; i < this.masterCommitSolrServers[j].length; i++) {
|
||||
final int index = i;
|
||||
this.solrAddExecutorService.execute(new Runnable() {
|
||||
public void run() {
|
||||
try {
|
||||
List<BSONObject> sharedObjects = shardedObjectMap.get(index);
|
||||
if (null == sharedObjects || sharedObjects.size() == 0) {
|
||||
return;
|
||||
}
|
||||
final List<SolrInputDocument> documents = new ArrayList<SolrInputDocument>(sharedObjects.size());
|
||||
for (BSONObject obj : sharedObjects) {
|
||||
SolrInputDocument document = toSolrInputDocument(obj, solrSchemas[jndex]);
|
||||
if (!document.isEmpty()) {
|
||||
document.addField(SolrSchema.VERTION, Core.VERSION_DATE_FORMAT.format(new Date()));
|
||||
System.out.println("document:" + document.toString());
|
||||
documents.add(document);
|
||||
} else if (LOGGER.isDebugEnabled()) {
|
||||
LOGGER.debug("solr input document is empty:" + document);
|
||||
}
|
||||
}
|
||||
addToSolr0(masterCommitSolrServers[jndex][index], documents, 0);
|
||||
} catch (Exception e) {
|
||||
if (LOGGER.isErrorEnabled()) {
|
||||
LOGGER.error("[" + name + "] error save to solr: ", e);
|
||||
}
|
||||
} finally {
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
countDownLatch.await();
|
||||
time = System.currentTimeMillis() - time;
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("[" + this.name + "] add {} documents to solr, {}ms, {}/s.", new Object[] { dbObjects.size(), time,
|
||||
dbObjects.size() * 1000 / (time) });
|
||||
}
|
||||
} catch (Exception e) {
|
||||
if (LOGGER.isErrorEnabled()) {
|
||||
LOGGER.error("[" + this.name + "] error save to solr: ", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private SolrInputDocument toSolrInputDocument(BSONObject doc, SolrSchema solrSchema) {
|
||||
Collection<Index> indexes = solrSchema.getIndexes();
|
||||
SolrInputDocument document = new SolrInputDocument();
|
||||
Map<String, Float> boostsMap = null;
|
||||
if (doc.get("$boosts") != null) {
|
||||
if (doc.get("$boosts") instanceof Map) {
|
||||
boostsMap = (Map<String, Float>) doc.get("$boosts");
|
||||
if (boostsMap.containsKey("$doc_boost")) {
|
||||
document.setDocumentBoost(((Number) boostsMap.get("$doc_boost")).floatValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (Index index : indexes) {
|
||||
String key = index.getName();
|
||||
if (doc.containsField(key)) {
|
||||
Object value = doc.get(key);
|
||||
SolrInputField field = new SolrInputField(key);
|
||||
if (value instanceof ObjectId) {
|
||||
field.setValue(value.toString(), 1.0f);
|
||||
} else {
|
||||
if (boostsMap != null && boostsMap.containsKey(key)) {
|
||||
field.setValue(value, ((Number) boostsMap.get(key)).floatValue());
|
||||
} else {
|
||||
field.setValue(value, 1.0f);
|
||||
}
|
||||
}
|
||||
document.put(key, field);
|
||||
}
|
||||
}
|
||||
return document;
|
||||
}
|
||||
|
||||
private Map<Integer, List<BSONObject>> shard(final List<BSONObject> dbObjects, int shards) {
|
||||
if (CollectionUtils.isEmpty(dbObjects)) {
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
Map<Integer, List<BSONObject>> ret = new HashMap<Integer, List<BSONObject>>();
|
||||
for (BSONObject object : dbObjects) {
|
||||
int hashMod = Math.abs(object.get("_id").hashCode()) % shards;
|
||||
List<BSONObject> modObjects = ret.get(hashMod);
|
||||
if (null == modObjects) {
|
||||
modObjects = new ArrayList<BSONObject>();
|
||||
ret.put(hashMod, modObjects);
|
||||
}
|
||||
modObjects.add(object);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
private void checkFilter() {
|
||||
// 5·ÖÖÓÒ»´Î
|
||||
if (System.currentTimeMillis() - lastFilterTime < 300000) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (this.filtering.compareAndSet(false, true)) {
|
||||
final long start = System.currentTimeMillis();
|
||||
for (int i = 0; i < masterCommitFilters.length; i++) {
|
||||
if (StringUtils.isEmpty(masterCommitFilters[i])) {
|
||||
continue;
|
||||
}
|
||||
for (HttpSolrServer server : masterCommitSolrServers[i]) {
|
||||
try {
|
||||
server.deleteByQuery(masterCommitFilters[i]);
|
||||
} catch (SolrServerException e) {
|
||||
if (LOGGER.isErrorEnabled()) {
|
||||
LOGGER.error("[" + this.name + "] error execute filter delete: ", e);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
if (LOGGER.isErrorEnabled()) {
|
||||
LOGGER.error("[" + this.name + "] error execute filter delete: ", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("[" + this.name + "] execute filter delete, " + (System.currentTimeMillis() - start) / 1000
|
||||
+ "s.");
|
||||
}
|
||||
lastFilterTime = System.currentTimeMillis();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
if (LOGGER.isErrorEnabled()) {
|
||||
LOGGER.error("[" + this.name + "] error execute filter delete: ", e);
|
||||
}
|
||||
} finally {
|
||||
this.filtering.compareAndSet(true, false);
|
||||
}
|
||||
}
|
||||
|
||||
public void deleteByQuery(String query) throws SolrException {
|
||||
try {
|
||||
for (HttpSolrServer[] servers : this.masterCommitSolrServers) {
|
||||
for (HttpSolrServer server : servers) {
|
||||
server.deleteByQuery(query);
|
||||
}
|
||||
}
|
||||
} catch (SolrServerException e) {
|
||||
throw new SolrException("[" + this.name + "] error execute delete[" + query + "]: ", e);
|
||||
} catch (IOException e) {
|
||||
throw new SolrException("[" + this.name + "] error execute delete[" + query + "]: ", e);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
public synchronized void reload() throws ConfigurationException {
|
||||
this.allSolrServers.clear();
|
||||
this.allSolrServers.putAll(initSolrServers(this.storeConfig, httpClient));
|
||||
String schema = this.storeConfig.getMasterSolrConfig().getHome() + "/" + name + "/conf/managed-schema";
|
||||
this.solrSchema.setSchemaFile(new File(schema));
|
||||
this.solrSchema.reload();
|
||||
}
|
||||
|
||||
public void reloadCores() throws SolrException {
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("[" + this.name + "] reload core...");
|
||||
}
|
||||
for (String id : this.allSolrServers.keySet()) {
|
||||
HttpSolrServer solrServer = this.allSolrServers.get(id);
|
||||
solrServer.reloadCore();
|
||||
}
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("[" + this.name + "] core reload success");
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isShardedServer() {
|
||||
return this.slaveSolrServers.length > 1;
|
||||
}
|
||||
|
||||
private HttpSolrServer selectRandomServer(HttpSolrServer[] shards) {
|
||||
if (null == shards || shards.length == 0) {
|
||||
throw new IllegalArgumentException("no shards");
|
||||
}
|
||||
if (shards.length == 1) {
|
||||
LOGGER.info("Choice:" + shards[0].getHost());
|
||||
return shards[0];
|
||||
}
|
||||
HttpSolrServer server = shards[RandomUtils.nextInt(shards.length)];
|
||||
LOGGER.info("Choice:" + server.getHost());
|
||||
return server;
|
||||
}
|
||||
|
||||
private String buildShardsParam(HttpSolrServer[] solrServers) {
|
||||
StringBuilder ret = new StringBuilder();
|
||||
for (int i = 0; i < solrServers.length; i++) {
|
||||
ret.append(solrServers[i].getHost()).append('/').append(this.name);
|
||||
if (i < solrServers.length - 1) {
|
||||
ret.append(',');
|
||||
}
|
||||
}
|
||||
LOGGER.info("slaveAddress:" + ret.toString());
|
||||
return ret.toString();
|
||||
}
|
||||
|
||||
public QueryResponse query(SolrQuery solrQuery, boolean fromMaster, boolean bulk) throws SolrException {
|
||||
if (isShardedServer() && !bulk && solrQuery.getStart() > MAX_NONE_BULK_SKIP) {
|
||||
throw new SolrException("[" + this.name + "] skip of none bulk query must not greater than " + MAX_NONE_BULK_SKIP
|
||||
+ ", try set $bulk to true in the query !");
|
||||
}
|
||||
return queryWithRetry(solrQuery, fromMaster, bulk, 0);
|
||||
}
|
||||
|
||||
private QueryResponse queryWithRetry(SolrQuery solrQuery, boolean fromMaster, boolean bulk, int retryCount)
|
||||
throws SolrException {
|
||||
// shards params
|
||||
LOGGER.info("isShardedServer:"+isShardedServer());
|
||||
LOGGER.info("fromMaster:"+fromMaster);
|
||||
if (isShardedServer()) {
|
||||
if (fromMaster) {
|
||||
solrQuery.set(ShardParams.SHARDS, buildShardsParam(this.masterSolrServers));
|
||||
} else {
|
||||
solrQuery.set(ShardParams.SHARDS, buildShardsParam(this.slaveSolrServers));
|
||||
}
|
||||
// bulk query
|
||||
if (bulk) {
|
||||
int numberShards = this.masterSolrServers.length;
|
||||
solrQuery.set(ShardParams.SHARDS_START, solrQuery.getStart() / numberShards);
|
||||
solrQuery.set(ShardParams.SHARDS_ROWS, Math.max(solrQuery.getRows() / numberShards, 1));
|
||||
solrQuery.setStart(0);
|
||||
}
|
||||
}
|
||||
LOGGER.info("solrQuery:"+solrQuery.toString());
|
||||
QueryResponse queryResponse = null;
|
||||
HttpSolrServer httpSolrServer = null;
|
||||
try {
|
||||
if (fromMaster) {
|
||||
httpSolrServer = selectRandomServer(this.masterSolrServers);
|
||||
try {
|
||||
queryResponse = httpSolrServer.query(solrQuery);
|
||||
} catch (IOException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
} else {
|
||||
httpSolrServer = selectRandomServer(this.slaveSolrServers);
|
||||
try {
|
||||
queryResponse = httpSolrServer.query(solrQuery);
|
||||
} catch (IOException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
} catch (SolrServerException ex) {
|
||||
if (ex.getCause() instanceof org.apache.solr.common.SolrException) {
|
||||
org.apache.solr.common.SolrException e = (org.apache.solr.common.SolrException) ex.getCause();
|
||||
if (retryCount < SOLR_QUERY_MAX_RETRY_COUNT && ErrorCode.getErrorCode(e.code()) == ErrorCode.SERVER_ERROR) {
|
||||
retryCount++;
|
||||
if (LOGGER.isErrorEnabled()) {
|
||||
LOGGER.error("#[" + this.name + "]# error query solr[" + httpSolrServer.getHost() + "], code:"
|
||||
+ ErrorCode.SERVER_ERROR + "(" + ErrorCode.SERVER_ERROR.code + "), retry for the " + retryCount
|
||||
+ "th time...");
|
||||
}
|
||||
return queryWithRetry(solrQuery, fromMaster, bulk, retryCount);
|
||||
} else {
|
||||
throw new SolrException("=[" + this.name + "]= error query solr[" + httpSolrServer.getHost() + "]:", e);
|
||||
}
|
||||
}
|
||||
throw new SolrException("*[" + this.name + "]* error query solr:", ex);
|
||||
}
|
||||
return queryResponse;
|
||||
}
|
||||
|
||||
public SolrSchema getSolrSchema() {
|
||||
return solrSchema;
|
||||
}
|
||||
|
||||
public class SolrServerImpl implements SolrServer
|
||||
{
|
||||
private static final Logger LOGGER;
|
||||
private static final int SOLR_COMMIT_MAX_RETRY_COUNT = 3;
|
||||
private static final int SOLR_QUERY_MAX_RETRY_COUNT = 3;
|
||||
private static final int MAX_NONE_BULK_SKIP = 100000;
|
||||
private final String name;
|
||||
private final StoreConfig storeConfig;
|
||||
private final HttpClient httpClient;
|
||||
private int masterCommitSolrServerCount;
|
||||
private final Map<String, HttpSolrServer> allSolrServers;
|
||||
private String[] masterCommitFilters;
|
||||
private HttpSolrServer[][] masterCommitSolrServers;
|
||||
private HttpSolrServer[] masterSolrServers;
|
||||
private HttpSolrServer[] slaveSolrServers;
|
||||
private SolrSchema[] solrSchemas;
|
||||
private final SolrSchema solrSchema;
|
||||
private final Executor solrAddExecutorService;
|
||||
private final AtomicBoolean filtering;
|
||||
private long lastFilterTime;
|
||||
|
||||
public SolrServerImpl(final StoreConfig storeConfig) throws ConfigurationException {
|
||||
this.filtering = new AtomicBoolean(false);
|
||||
this.lastFilterTime = 0L;
|
||||
this.storeConfig = storeConfig;
|
||||
this.name = storeConfig.getName();
|
||||
final PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
|
||||
cm.setMaxTotal(50);
|
||||
this.httpClient = (HttpClient)new DefaultHttpClient();
|
||||
this.allSolrServers = this.initSolrServers(this.storeConfig, this.httpClient);
|
||||
this.solrSchema = new SolrSchemaImpl(new File(storeConfig.getMasterSolrConfig().getHome() + "/" + this.name + "/conf/managed-schema"));
|
||||
final int size = (this.masterCommitSolrServerCount + 1) * 2;
|
||||
this.solrAddExecutorService = (Executor)new BlockingThreadPoolExecutor(this.name + "-SolrAdd", 1, size, size, 60000L);
|
||||
}
|
||||
|
||||
private CommonsHttpSolrServer createSolrServer(final String coreName, final SolrConfig solronfig) throws ConfigurationException {
|
||||
try {
|
||||
final CommonsHttpSolrServer solrServer = new CommonsHttpSolrServer(solronfig.getHost(), coreName, this.httpClient);
|
||||
return solrServer;
|
||||
}
|
||||
catch (MalformedURLException e) {
|
||||
throw new ConfigurationException("[" + this.name + "] error create solr server: ", e);
|
||||
}
|
||||
}
|
||||
|
||||
private HttpSolrServer[] initSolrServer(final Map<String, HttpSolrServer> allSolrServers, final SolrConfig solrConfig) throws ConfigurationException {
|
||||
SolrConfig[] shardsConfigs = solrConfig.getShardsConfigs();
|
||||
if (shardsConfigs.length == 0) {
|
||||
shardsConfigs = new SolrConfig[] { solrConfig };
|
||||
}
|
||||
final HttpSolrServer[] shards = new HttpSolrServer[shardsConfigs.length];
|
||||
for (int i = 0; i < shardsConfigs.length; ++i) {
|
||||
final SolrConfig shardConfig = shardsConfigs[i];
|
||||
if (!allSolrServers.containsKey(shardConfig.getId())) {
|
||||
final SolrConfig[] slaveConfigs = shardConfig.getSlaveConfigs();
|
||||
if (slaveConfigs.length == 0) {
|
||||
allSolrServers.put(shardConfig.getId(), this.createSolrServer(this.name, shardConfig));
|
||||
}
|
||||
else {
|
||||
final CommonsHttpSolrServer[] slaveServers = new CommonsHttpSolrServer[slaveConfigs.length];
|
||||
for (int j = 0; j < slaveConfigs.length; ++j) {
|
||||
final SolrConfig slaveConfig = slaveConfigs[j];
|
||||
if (!allSolrServers.containsKey(slaveConfig.getId())) {
|
||||
allSolrServers.put(slaveConfig.getId(), this.createSolrServer(this.name, slaveConfig));
|
||||
}
|
||||
slaveServers[j] = (CommonsHttpSolrServer)allSolrServers.get(slaveConfig.getId());
|
||||
}
|
||||
allSolrServers.put(shardConfig.getId(), new LBHttpSolrServer((HttpSolrServer[])slaveServers));
|
||||
}
|
||||
}
|
||||
shards[i] = allSolrServers.get(shardConfig.getId());
|
||||
}
|
||||
return shards;
|
||||
}
|
||||
|
||||
private synchronized Map<String, HttpSolrServer> initSolrServers(final StoreConfig storeConfig, final HttpClient httpClient) throws ConfigurationException {
|
||||
final Map<String, HttpSolrServer> solrServers = new HashMap<String, HttpSolrServer>();
|
||||
final SolrConfig[] masterConfigs = storeConfig.getMasterSolrConfigs();
|
||||
this.masterCommitSolrServers = new HttpSolrServer[masterConfigs.length][];
|
||||
this.masterCommitFilters = new String[masterConfigs.length];
|
||||
this.solrSchemas = new SolrSchema[masterConfigs.length];
|
||||
this.masterCommitSolrServerCount = 0;
|
||||
for (int i = 0; i < masterConfigs.length; ++i) {
|
||||
this.masterCommitSolrServers[i] = this.initSolrServer(solrServers, masterConfigs[i]);
|
||||
this.masterCommitFilters[i] = masterConfigs[i].getFilter();
|
||||
this.solrSchemas[i] = new SolrSchemaImpl(new File(masterConfigs[i].getHome() + "/" + this.name + "/conf/managed-schema"));
|
||||
this.masterCommitSolrServerCount += this.masterCommitSolrServers[i].length;
|
||||
}
|
||||
this.masterSolrServers = this.initSolrServer(solrServers, storeConfig.getMasterSolrConfig());
|
||||
this.slaveSolrServers = this.initSolrServer(solrServers, storeConfig.getSlaveSolrConfig());
|
||||
return solrServers;
|
||||
}
|
||||
|
||||
private void addToSolr0(final HttpSolrServer solrServer, final List<SolrInputDocument> documents, int errorCount) {
|
||||
try {
|
||||
if (null != documents && !documents.isEmpty()) {
|
||||
solrServer.add((Collection)documents);
|
||||
solrServer.commit();
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
if (errorCount <= 3) {
|
||||
final long sleepTime = ++errorCount * errorCount;
|
||||
if (SolrServerImpl.LOGGER.isInfoEnabled()) {
|
||||
SolrServerImpl.LOGGER.info("[" + this.name + "] error commit to solr[" + solrServer.getHost() + "], sleep " + sleepTime + "s and retry(" + e.getMessage() + ")");
|
||||
}
|
||||
try {
|
||||
Thread.sleep(1000L * sleepTime);
|
||||
}
|
||||
catch (InterruptedException e2) {
|
||||
SolrServerImpl.LOGGER.error("[" + this.name + "] error save to solr: ", (Throwable)e2);
|
||||
}
|
||||
this.addToSolr0(solrServer, documents, errorCount);
|
||||
}
|
||||
else if (SolrServerImpl.LOGGER.isErrorEnabled()) {
|
||||
SolrServerImpl.LOGGER.error("[" + this.name + "] error save to solr[" + solrServer.getHost() + "] after " + 3 + " times retry: ", (Throwable)e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void add(final List<BSONObject> dbObjects) throws SolrException {
|
||||
if (null == dbObjects || dbObjects.size() == 0) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
long time = System.currentTimeMillis();
|
||||
this.checkFilter();
|
||||
final CountDownLatch countDownLatch = new CountDownLatch(this.masterCommitSolrServerCount);
|
||||
for (int j = 0; j < this.masterCommitSolrServers.length; ++j) {
|
||||
final int jndex = j;
|
||||
final Map<Integer, List<BSONObject>> shardedObjectMap = this.shard(dbObjects, this.masterCommitSolrServers[j].length);
|
||||
for (int i = 0; i < this.masterCommitSolrServers[j].length; ++i) {
|
||||
final int index = i;
|
||||
this.solrAddExecutorService.execute(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
final List<BSONObject> sharedObjects = shardedObjectMap.get(index);
|
||||
if (null == sharedObjects || sharedObjects.size() == 0) {
|
||||
return;
|
||||
}
|
||||
final List<SolrInputDocument> documents = new ArrayList<SolrInputDocument>(sharedObjects.size());
|
||||
for (final BSONObject obj : sharedObjects) {
|
||||
final SolrInputDocument document = SolrServerImpl.this.toSolrInputDocument(obj, SolrServerImpl.this.solrSchemas[jndex]);
|
||||
if (!document.isEmpty()) {
|
||||
document.addField("_version", (Object)Core.VERSION_DATE_FORMAT.format(new Date()));
|
||||
System.out.println("document:" + document.toString());
|
||||
documents.add(document);
|
||||
}
|
||||
else {
|
||||
if (!SolrServerImpl.LOGGER.isDebugEnabled()) {
|
||||
continue;
|
||||
}
|
||||
SolrServerImpl.LOGGER.debug("solr input document is empty:" + document);
|
||||
}
|
||||
}
|
||||
SolrServerImpl.this.addToSolr0(SolrServerImpl.this.masterCommitSolrServers[jndex][index], documents, 0);
|
||||
}
|
||||
catch (Exception e) {
|
||||
if (SolrServerImpl.LOGGER.isErrorEnabled()) {
|
||||
SolrServerImpl.LOGGER.error("[" + SolrServerImpl.this.name + "] error save to solr: ", (Throwable)e);
|
||||
}
|
||||
}
|
||||
finally {
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
countDownLatch.await();
|
||||
time = System.currentTimeMillis() - time;
|
||||
if (SolrServerImpl.LOGGER.isInfoEnabled()) {
|
||||
SolrServerImpl.LOGGER.info("[" + this.name + "] add {} documents to solr, {}ms, {}/s.", new Object[] { dbObjects.size(), time, dbObjects.size() * 1000 / time });
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
if (SolrServerImpl.LOGGER.isErrorEnabled()) {
|
||||
SolrServerImpl.LOGGER.error("[" + this.name + "] error save to solr: ", (Throwable)e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private SolrInputDocument toSolrInputDocument(final BSONObject doc, final SolrSchema solrSchema) {
|
||||
final Collection<Index> indexes = solrSchema.getIndexes();
|
||||
final SolrInputDocument document = new SolrInputDocument();
|
||||
Map<String, Float> boostsMap = null;
|
||||
if (doc.get("$boosts") != null && doc.get("$boosts") instanceof Map) {
|
||||
boostsMap = (Map<String, Float>)doc.get("$boosts");
|
||||
if (boostsMap.containsKey("$doc_boost")) {
|
||||
document.setDocumentBoost(boostsMap.get("$doc_boost").floatValue());
|
||||
}
|
||||
}
|
||||
for (final Index index : indexes) {
|
||||
final String key = index.getName();
|
||||
if (doc.containsField(key)) {
|
||||
final Object value = doc.get(key);
|
||||
final SolrInputField field = new SolrInputField(key);
|
||||
if (value instanceof ObjectId) {
|
||||
field.setValue((Object)value.toString(), 1.0f);
|
||||
}
|
||||
else if (boostsMap != null && boostsMap.containsKey(key)) {
|
||||
field.setValue(value, boostsMap.get(key).floatValue());
|
||||
}
|
||||
else {
|
||||
field.setValue(value, 1.0f);
|
||||
}
|
||||
document.put(key, field);
|
||||
}
|
||||
}
|
||||
return document;
|
||||
}
|
||||
|
||||
private Map<Integer, List<BSONObject>> shard(final List<BSONObject> dbObjects, final int shards) {
|
||||
if (CollectionUtils.isEmpty((Collection)dbObjects)) {
|
||||
return Collections.<Integer, List<BSONObject>>emptyMap();
|
||||
}
|
||||
final Map<Integer, List<BSONObject>> ret = new HashMap<Integer, List<BSONObject>>();
|
||||
for (final BSONObject object : dbObjects) {
|
||||
final int hashMod = Math.abs(object.get("_id").hashCode()) % shards;
|
||||
List<BSONObject> modObjects = ret.get(hashMod);
|
||||
if (null == modObjects) {
|
||||
modObjects = new ArrayList<BSONObject>();
|
||||
ret.put(hashMod, modObjects);
|
||||
}
|
||||
modObjects.add(object);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
private void checkFilter() {
|
||||
if (System.currentTimeMillis() - this.lastFilterTime < 300000L) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (this.filtering.compareAndSet(false, true)) {
|
||||
final long start = System.currentTimeMillis();
|
||||
for (int i = 0; i < this.masterCommitFilters.length; ++i) {
|
||||
if (!StringUtils.isEmpty(this.masterCommitFilters[i])) {
|
||||
for (final HttpSolrServer server : this.masterCommitSolrServers[i]) {
|
||||
try {
|
||||
server.deleteByQuery(this.masterCommitFilters[i]);
|
||||
}
|
||||
catch (SolrServerException e) {
|
||||
if (SolrServerImpl.LOGGER.isErrorEnabled()) {
|
||||
SolrServerImpl.LOGGER.error("[" + this.name + "] error execute filter delete: ", (Throwable)e);
|
||||
}
|
||||
}
|
||||
catch (IOException e2) {
|
||||
if (SolrServerImpl.LOGGER.isErrorEnabled()) {
|
||||
SolrServerImpl.LOGGER.error("[" + this.name + "] error execute filter delete: ", (Throwable)e2);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (SolrServerImpl.LOGGER.isInfoEnabled()) {
|
||||
SolrServerImpl.LOGGER.info("[" + this.name + "] execute filter delete, " + (System.currentTimeMillis() - start) / 1000L + "s.");
|
||||
}
|
||||
this.lastFilterTime = System.currentTimeMillis();
|
||||
}
|
||||
}
|
||||
catch (Exception e3) {
|
||||
if (SolrServerImpl.LOGGER.isErrorEnabled()) {
|
||||
SolrServerImpl.LOGGER.error("[" + this.name + "] error execute filter delete: ", (Throwable)e3);
|
||||
}
|
||||
}
|
||||
finally {
|
||||
this.filtering.compareAndSet(true, false);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteByQuery(final String query) throws SolrException {
|
||||
try {
|
||||
for (final HttpSolrServer[] array : this.masterCommitSolrServers) {
|
||||
final HttpSolrServer[] servers = array;
|
||||
for (final HttpSolrServer server : array) {
|
||||
server.deleteByQuery(query);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (SolrServerException e) {
|
||||
throw new SolrException("[" + this.name + "] error execute delete[" + query + "]: ", (Throwable)e);
|
||||
}
|
||||
catch (IOException e2) {
|
||||
throw new SolrException("[" + this.name + "] error execute delete[" + query + "]: ", e2);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void reload() throws ConfigurationException {
|
||||
this.allSolrServers.clear();
|
||||
this.allSolrServers.putAll(this.initSolrServers(this.storeConfig, this.httpClient));
|
||||
final String schema = this.storeConfig.getMasterSolrConfig().getHome() + "/" + this.name + "/conf/managed-schema";
|
||||
this.solrSchema.setSchemaFile(new File(schema));
|
||||
this.solrSchema.reload();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reloadCores() throws SolrException {
|
||||
if (SolrServerImpl.LOGGER.isInfoEnabled()) {
|
||||
SolrServerImpl.LOGGER.info("[" + this.name + "] reload core...");
|
||||
}
|
||||
for (final String id : this.allSolrServers.keySet()) {
|
||||
final HttpSolrServer solrServer = this.allSolrServers.get(id);
|
||||
solrServer.reloadCore();
|
||||
}
|
||||
if (SolrServerImpl.LOGGER.isInfoEnabled()) {
|
||||
SolrServerImpl.LOGGER.info("[" + this.name + "] core reload success");
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isShardedServer() {
|
||||
return this.slaveSolrServers.length > 1;
|
||||
}
|
||||
|
||||
private HttpSolrServer selectRandomServer(final HttpSolrServer[] shards) {
|
||||
if (null == shards || shards.length == 0) {
|
||||
throw new IllegalArgumentException("no shards");
|
||||
}
|
||||
if (shards.length == 1) {
|
||||
SolrServerImpl.LOGGER.info("Choice:" + shards[0].getHost());
|
||||
return shards[0];
|
||||
}
|
||||
final HttpSolrServer server = shards[RandomUtils.nextInt(shards.length)];
|
||||
SolrServerImpl.LOGGER.info("Choice:" + server.getHost());
|
||||
return server;
|
||||
}
|
||||
|
||||
private String buildShardsParam(final HttpSolrServer[] solrServers) {
|
||||
final StringBuilder ret = new StringBuilder();
|
||||
for (int i = 0; i < solrServers.length; ++i) {
|
||||
ret.append(solrServers[i].getHost()).append('/').append(this.name);
|
||||
if (i < solrServers.length - 1) {
|
||||
ret.append(',');
|
||||
}
|
||||
}
|
||||
SolrServerImpl.LOGGER.info("slaveAddress:" + ret.toString());
|
||||
return ret.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResponse query(final SolrQuery solrQuery, final boolean fromMaster, final boolean bulk) throws SolrException {
|
||||
if (this.isShardedServer() && !bulk && solrQuery.getStart() > 100000) {
|
||||
throw new SolrException("[" + this.name + "] skip of none bulk query must not greater than " + 100000 + ", try set $bulk to true in the query !");
|
||||
}
|
||||
return this.queryWithRetry(solrQuery, fromMaster, bulk, 0);
|
||||
}
|
||||
|
||||
private QueryResponse queryWithRetry(final SolrQuery solrQuery, final boolean fromMaster, final boolean bulk, int retryCount) throws SolrException {
|
||||
SolrServerImpl.LOGGER.info("isShardedServer:" + this.isShardedServer());
|
||||
SolrServerImpl.LOGGER.info("fromMaster:" + fromMaster);
|
||||
if (this.isShardedServer()) {
|
||||
if (fromMaster) {
|
||||
solrQuery.set("shards", this.buildShardsParam(this.masterSolrServers));
|
||||
}
|
||||
else {
|
||||
solrQuery.set("shards", this.buildShardsParam(this.slaveSolrServers));
|
||||
}
|
||||
if (bulk) {
|
||||
final int numberShards = this.masterSolrServers.length;
|
||||
solrQuery.set("shards.start", solrQuery.getStart() / numberShards);
|
||||
solrQuery.set("shards.rows", Math.max(solrQuery.getRows() / numberShards, 1));
|
||||
solrQuery.setStart(0);
|
||||
}
|
||||
}
|
||||
SolrServerImpl.LOGGER.info("solrQuery:" + solrQuery.toString());
|
||||
QueryResponse queryResponse = null;
|
||||
HttpSolrServer httpSolrServer = null;
|
||||
try {
|
||||
if (fromMaster) {
|
||||
httpSolrServer = this.selectRandomServer(this.masterSolrServers);
|
||||
try {
|
||||
queryResponse = httpSolrServer.query(solrQuery);
|
||||
}
|
||||
catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
else {
|
||||
httpSolrServer = this.selectRandomServer(this.slaveSolrServers);
|
||||
try {
|
||||
queryResponse = httpSolrServer.query(solrQuery);
|
||||
}
|
||||
catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
solrQuery.add("__host", httpSolrServer.getHost());
|
||||
}
|
||||
catch (SolrServerException ex) {
|
||||
if (!(ex.getCause() instanceof org.apache.solr.common.SolrException)) {
|
||||
throw new SolrException("*[" + this.name + "]* error query solr:", (Throwable)ex);
|
||||
}
|
||||
final org.apache.solr.common.SolrException e2 = (org.apache.solr.common.SolrException)ex.getCause();
|
||||
if (retryCount < 3 && org.apache.solr.common.SolrException.ErrorCode.getErrorCode(e2.code()) == org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR) {
|
||||
++retryCount;
|
||||
if (SolrServerImpl.LOGGER.isErrorEnabled()) {
|
||||
SolrServerImpl.LOGGER.error("#[" + this.name + "]# error query solr[" + httpSolrServer.getHost() + "], code:" + org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR + "(" + org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR.code + "), retry for the " + retryCount + "th time...");
|
||||
}
|
||||
return this.queryWithRetry(solrQuery, fromMaster, bulk, retryCount);
|
||||
}
|
||||
throw new SolrException("=[" + this.name + "]= error query solr[" + httpSolrServer.getHost() + "]:", (Throwable)e2);
|
||||
}
|
||||
return queryResponse;
|
||||
}
|
||||
|
||||
@Override
|
||||
public SolrSchema getSolrSchema() {
|
||||
return this.solrSchema;
|
||||
}
|
||||
|
||||
static {
|
||||
LOGGER = LoggerFactory.getLogger((Class)SolrServerImpl.class);
|
||||
}
|
||||
}
|
||||
|
||||
+29
-48
@@ -1,53 +1,34 @@
|
||||
/**
|
||||
* JMongo is a mongodb driver writtern in java.
|
||||
* Copyright (C) 2010 Xiaohu Huang
|
||||
*
|
||||
* JMongo is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* JMongo is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package com.fangjia.datastore.utils;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class BSONUtils {
|
||||
|
||||
public static long bitSet(long holder, int i, boolean set) {
|
||||
if (i < 0 || i > 63) {
|
||||
throw new IllegalArgumentException("i should from 0 inclusive and 64 exclusive");
|
||||
}
|
||||
if (set) {
|
||||
return holder | (1 << i);
|
||||
} else {
|
||||
int ii = 1 << i;
|
||||
return (holder | ii) ^ ii;
|
||||
}
|
||||
}
|
||||
|
||||
public static boolean isBitSet(long holder, int i) {
|
||||
if (i < 0 || i > 63) {
|
||||
throw new IllegalArgumentException("i should from 0 inclusive and 64 exclusive");
|
||||
}
|
||||
return (holder & (1 << i)) > 0;
|
||||
}
|
||||
|
||||
public static int stringByteLength(String s) throws IOException {
|
||||
byte[] bs = s.getBytes("UTF-8");
|
||||
return bs.length + 5;
|
||||
}
|
||||
|
||||
public static int cstringByteLength(String s) throws IOException {
|
||||
byte[] bs = s.getBytes("UTF-8");
|
||||
return bs.length + 1;
|
||||
}
|
||||
|
||||
public class BSONUtils
|
||||
{
|
||||
public static long bitSet(final long holder, final int i, final boolean set) {
|
||||
if (i < 0 || i > 63) {
|
||||
throw new IllegalArgumentException("i should from 0 inclusive and 64 exclusive");
|
||||
}
|
||||
if (set) {
|
||||
return holder | (long)(1 << i);
|
||||
}
|
||||
final int ii = 1 << i;
|
||||
return (holder | (long)ii) ^ (long)ii;
|
||||
}
|
||||
|
||||
public static boolean isBitSet(final long holder, final int i) {
|
||||
if (i < 0 || i > 63) {
|
||||
throw new IllegalArgumentException("i should from 0 inclusive and 64 exclusive");
|
||||
}
|
||||
return (holder & (long)(1 << i)) > 0L;
|
||||
}
|
||||
|
||||
public static int stringByteLength(final String s) throws IOException {
|
||||
final byte[] bs = s.getBytes("UTF-8");
|
||||
return bs.length + 5;
|
||||
}
|
||||
|
||||
public static int cstringByteLength(final String s) throws IOException {
|
||||
final byte[] bs = s.getBytes("UTF-8");
|
||||
return bs.length + 1;
|
||||
}
|
||||
}
|
||||
|
||||
+101
-270
@@ -1,275 +1,106 @@
|
||||
/**
|
||||
* JMongo is a mongodb driver writtern in java.
|
||||
* Copyright (C) 2010 Xiaohu Huang
|
||||
*
|
||||
* JMongo is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* JMongo is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package com.fangjia.datastore.utils;
|
||||
|
||||
import java.io.PrintStream;
|
||||
|
||||
/**
|
||||
* Hex dump class.
|
||||
*/
|
||||
public final class HexDump {
|
||||
|
||||
public static final void dump(byte[] byt) {
|
||||
dump(byt, byt.length, 0, System.out);
|
||||
}
|
||||
|
||||
/**
|
||||
* Hex dump a byte array
|
||||
*
|
||||
* @param byt
|
||||
* Byte array to dump
|
||||
* @param len
|
||||
* Length of data to dump
|
||||
* @param offset
|
||||
* Offset to start data dump
|
||||
*/
|
||||
public static final void dump(byte[] byt, int len, int offset) {
|
||||
dump(byt, len, offset, System.out);
|
||||
}
|
||||
|
||||
/**
|
||||
* Hex dump a byte array
|
||||
*
|
||||
* @param byt
|
||||
* Byte array to dump
|
||||
* @param len
|
||||
* Length of data to dump
|
||||
* @param offset
|
||||
* Offset to start data dump
|
||||
* @param stream
|
||||
* Output stream to dump the output to.
|
||||
*/
|
||||
|
||||
public static final void dump(byte[] byt, int len, int offset, PrintStream stream) {
|
||||
|
||||
// Create buffers for the ASCII and Hex output
|
||||
|
||||
StringBuffer ascBuf = new StringBuffer();
|
||||
StringBuffer hexBuf = new StringBuffer();
|
||||
|
||||
// dump 16 byte blocks from the array until the length has been
|
||||
// reached
|
||||
|
||||
int dlen = 0;
|
||||
int doff = offset;
|
||||
String posStr = null;
|
||||
|
||||
while (dlen < len) {
|
||||
|
||||
// Reset the ASCII/Hex buffers
|
||||
|
||||
ascBuf.setLength(0);
|
||||
hexBuf.setLength(0);
|
||||
|
||||
posStr = generatePositionString(doff);
|
||||
|
||||
// dump a block of data, update the data offset
|
||||
|
||||
doff = generateLine(byt, doff, ascBuf, hexBuf);
|
||||
|
||||
// Output the current record
|
||||
|
||||
stream.print(posStr);
|
||||
stream.print(hexBuf.toString());
|
||||
stream.println(ascBuf.toString());
|
||||
|
||||
// Update the dump length
|
||||
|
||||
dlen += 16;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a hex string for the specified string
|
||||
*
|
||||
* @param str
|
||||
* String
|
||||
* @return String
|
||||
*/
|
||||
public static final String hexString(String str) {
|
||||
if (str != null)
|
||||
return hexString(str.getBytes());
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a hex string for the specified string
|
||||
*
|
||||
* @param str
|
||||
* String
|
||||
* @param gap
|
||||
* String
|
||||
* @return String
|
||||
*/
|
||||
public static final String hexString(String str, String gap) {
|
||||
if (str != null)
|
||||
return hexString(str.getBytes(), gap);
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a hex string for the specified bytes
|
||||
*
|
||||
* @param buf
|
||||
* byte[]
|
||||
* @return String
|
||||
*/
|
||||
public static final String hexString(byte[] buf) {
|
||||
return hexString(buf, buf.length, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a hex string for the specified bytes
|
||||
*
|
||||
* @param buf
|
||||
* byte[]
|
||||
* @param gap
|
||||
* String
|
||||
* @return String
|
||||
*/
|
||||
public static final String hexString(byte[] buf, String gap) {
|
||||
return hexString(buf, buf.length, gap);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a hex string for the specified bytes
|
||||
*
|
||||
* @param buf
|
||||
* byte[]
|
||||
* @param len
|
||||
* int
|
||||
* @param gap
|
||||
* String
|
||||
* @return String
|
||||
*/
|
||||
public static final String hexString(byte[] buf, int len, String gap) {
|
||||
|
||||
// Check if the buffer is valid
|
||||
|
||||
if (buf == null)
|
||||
return "";
|
||||
|
||||
// Create a string buffer for the hex string
|
||||
|
||||
int buflen = buf.length * 2;
|
||||
if (gap != null)
|
||||
buflen += buf.length * gap.length();
|
||||
|
||||
StringBuffer hex = new StringBuffer(buflen);
|
||||
|
||||
// Convert the bytes to hex-ASCII
|
||||
|
||||
for (int i = 0; i < len; i++) {
|
||||
|
||||
// Get the current byte
|
||||
|
||||
int curbyt = (int) (buf[i] & 0x00FF);
|
||||
|
||||
// Output the hex string
|
||||
|
||||
hex.append(Integer.toHexString((curbyt & 0xF0) >> 4));
|
||||
hex.append(Integer.toHexString(curbyt & 0x0F));
|
||||
|
||||
// Add the gap string, if specified
|
||||
|
||||
if (gap != null && i < (len - 1))
|
||||
hex.append(gap);
|
||||
}
|
||||
|
||||
// Return the hex-ASCII string
|
||||
|
||||
return hex.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a buffer position string
|
||||
*
|
||||
* @param off
|
||||
* int
|
||||
* @return String
|
||||
*/
|
||||
private static final String generatePositionString(int off) {
|
||||
|
||||
// Create a buffer position string
|
||||
|
||||
StringBuffer posStr = new StringBuffer("" + off + ": ");
|
||||
while (posStr.length() < 10)
|
||||
posStr.insert(0, " ");
|
||||
|
||||
// Return the string
|
||||
|
||||
return posStr.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Output a single line of the hex dump to a debug device
|
||||
*
|
||||
* @param byt
|
||||
* Byte array to dump
|
||||
* @param off
|
||||
* Offset to start data dump
|
||||
* @param ascBuf
|
||||
* Buffer for ASCII output
|
||||
* @param hexBuf
|
||||
* Buffer for Hex output
|
||||
* @return New offset value
|
||||
*/
|
||||
|
||||
private static final int generateLine(byte[] byt, int off, StringBuffer ascBuf, StringBuffer hexBuf) {
|
||||
|
||||
// Check if there is enough buffer space to dump 16 bytes
|
||||
|
||||
int dumplen = byt.length - off;
|
||||
if (dumplen > 16)
|
||||
dumplen = 16;
|
||||
|
||||
// dump a 16 byte block of data
|
||||
|
||||
for (int i = 0; i < dumplen; i++) {
|
||||
|
||||
// Get the current byte
|
||||
|
||||
int curbyt = (int) (byt[off++] & 0x00FF);
|
||||
|
||||
// Output the hex string
|
||||
|
||||
hexBuf.append(Integer.toHexString((curbyt & 0xF0) >> 4));
|
||||
hexBuf.append(Integer.toHexString(curbyt & 0x0F));
|
||||
hexBuf.append(" ");
|
||||
|
||||
// Output the character equivalent, if printable
|
||||
|
||||
if (Character.isLetterOrDigit((char) curbyt) || Character.getType((char) curbyt) != Character.CONTROL)
|
||||
ascBuf.append((char) curbyt);
|
||||
else
|
||||
ascBuf.append(".");
|
||||
}
|
||||
|
||||
for (int i = dumplen; i < 16; i++) {
|
||||
hexBuf.append(" ");
|
||||
}
|
||||
|
||||
// Output the hex dump line
|
||||
|
||||
hexBuf.append(" ");
|
||||
|
||||
// Return the new data offset
|
||||
|
||||
return off;
|
||||
}
|
||||
public final class HexDump
|
||||
{
|
||||
public static final void dump(final byte[] byt) {
|
||||
dump(byt, byt.length, 0, System.out);
|
||||
}
|
||||
|
||||
public static final void dump(final byte[] byt, final int len, final int offset) {
|
||||
dump(byt, len, offset, System.out);
|
||||
}
|
||||
|
||||
public static final void dump(final byte[] byt, final int len, final int offset, final PrintStream stream) {
|
||||
final StringBuffer ascBuf = new StringBuffer();
|
||||
final StringBuffer hexBuf = new StringBuffer();
|
||||
int dlen = 0;
|
||||
int doff = offset;
|
||||
String posStr = null;
|
||||
while (dlen < len) {
|
||||
ascBuf.setLength(0);
|
||||
hexBuf.setLength(0);
|
||||
posStr = generatePositionString(doff);
|
||||
doff = generateLine(byt, doff, ascBuf, hexBuf);
|
||||
stream.print(posStr);
|
||||
stream.print(hexBuf.toString());
|
||||
stream.println(ascBuf.toString());
|
||||
dlen += 16;
|
||||
}
|
||||
}
|
||||
|
||||
public static final String hexString(final String str) {
|
||||
if (str != null) {
|
||||
return hexString(str.getBytes());
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
public static final String hexString(final String str, final String gap) {
|
||||
if (str != null) {
|
||||
return hexString(str.getBytes(), gap);
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
public static final String hexString(final byte[] buf) {
|
||||
return hexString(buf, buf.length, null);
|
||||
}
|
||||
|
||||
public static final String hexString(final byte[] buf, final String gap) {
|
||||
return hexString(buf, buf.length, gap);
|
||||
}
|
||||
|
||||
public static final String hexString(final byte[] buf, final int len, final String gap) {
|
||||
if (buf == null) {
|
||||
return "";
|
||||
}
|
||||
int buflen = buf.length * 2;
|
||||
if (gap != null) {
|
||||
buflen += buf.length * gap.length();
|
||||
}
|
||||
final StringBuffer hex = new StringBuffer(buflen);
|
||||
for (int i = 0; i < len; ++i) {
|
||||
final int curbyt = buf[i] & 0xFF;
|
||||
hex.append(Integer.toHexString((curbyt & 0xF0) >> 4));
|
||||
hex.append(Integer.toHexString(curbyt & 0xF));
|
||||
if (gap != null && i < len - 1) {
|
||||
hex.append(gap);
|
||||
}
|
||||
}
|
||||
return hex.toString();
|
||||
}
|
||||
|
||||
private static final String generatePositionString(final int off) {
|
||||
final StringBuffer posStr = new StringBuffer("" + off + ": ");
|
||||
while (posStr.length() < 10) {
|
||||
posStr.insert(0, " ");
|
||||
}
|
||||
return posStr.toString();
|
||||
}
|
||||
|
||||
private static final int generateLine(final byte[] byt, int off, final StringBuffer ascBuf, final StringBuffer hexBuf) {
|
||||
int dumplen = byt.length - off;
|
||||
if (dumplen > 16) {
|
||||
dumplen = 16;
|
||||
}
|
||||
for (int i = 0; i < dumplen; ++i) {
|
||||
final int curbyt = byt[off++] & 0xFF;
|
||||
hexBuf.append(Integer.toHexString((curbyt & 0xF0) >> 4));
|
||||
hexBuf.append(Integer.toHexString(curbyt & 0xF));
|
||||
hexBuf.append(" ");
|
||||
if (Character.isLetterOrDigit((char)curbyt) || Character.getType((char)curbyt) != 15) {
|
||||
ascBuf.append((char)curbyt);
|
||||
}
|
||||
else {
|
||||
ascBuf.append(".");
|
||||
}
|
||||
}
|
||||
for (int i = dumplen; i < 16; ++i) {
|
||||
hexBuf.append(" ");
|
||||
}
|
||||
hexBuf.append(" ");
|
||||
return off;
|
||||
}
|
||||
}
|
||||
|
||||
+13
-13
@@ -2,17 +2,17 @@ package com.fangjia.datastore.utils;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
|
||||
public class LoggerFactory {
|
||||
|
||||
public static Logger getLogger(Class<?> clazz) {
|
||||
return org.slf4j.LoggerFactory.getLogger(clazz);
|
||||
}
|
||||
|
||||
public static Logger getLogger(String name) {
|
||||
return org.slf4j.LoggerFactory.getLogger(name);
|
||||
}
|
||||
|
||||
public static Logger getPerformanceLogger(Class<?> clazz) {
|
||||
return org.slf4j.LoggerFactory.getLogger("performance." + clazz.getName());
|
||||
}
|
||||
public class LoggerFactory
|
||||
{
|
||||
public static Logger getLogger(final Class<?> clazz) {
|
||||
return org.slf4j.LoggerFactory.getLogger((Class)clazz);
|
||||
}
|
||||
|
||||
public static Logger getLogger(final String name) {
|
||||
return org.slf4j.LoggerFactory.getLogger(name);
|
||||
}
|
||||
|
||||
public static Logger getPerformanceLogger(final Class<?> clazz) {
|
||||
return org.slf4j.LoggerFactory.getLogger("performance." + clazz.getName());
|
||||
}
|
||||
}
|
||||
|
||||
+19
-18
@@ -2,22 +2,23 @@ package com.fangjia.datastore.utils;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
public class MongoUtils {
|
||||
private MongoUtils() {
|
||||
}
|
||||
|
||||
public static String[] parseFullCollectionName(String fullCollectionName) {
|
||||
if (StringUtils.isEmpty(fullCollectionName)) {
|
||||
return null;
|
||||
}
|
||||
int i = fullCollectionName.indexOf('.');
|
||||
if (i <= 0 || i >= fullCollectionName.length() - 1) {
|
||||
return null;
|
||||
}
|
||||
return new String[] { fullCollectionName.substring(0, i), fullCollectionName.substring(i + 1) };
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println(parseFullCollectionName(".q"));
|
||||
}
|
||||
public class MongoUtils
|
||||
{
|
||||
private MongoUtils() {
|
||||
}
|
||||
|
||||
public static String[] parseFullCollectionName(final String fullCollectionName) {
|
||||
if (StringUtils.isEmpty(fullCollectionName)) {
|
||||
return null;
|
||||
}
|
||||
final int i = fullCollectionName.indexOf(46);
|
||||
if (i <= 0 || i >= fullCollectionName.length() - 1) {
|
||||
return null;
|
||||
}
|
||||
return new String[] { fullCollectionName.substring(0, i), fullCollectionName.substring(i + 1) };
|
||||
}
|
||||
|
||||
public static void main(final String[] args) {
|
||||
System.out.println(parseFullCollectionName(".q"));
|
||||
}
|
||||
}
|
||||
|
||||
+38
-41
@@ -2,45 +2,42 @@ package com.fangjia.datastore.utils;
|
||||
|
||||
import org.dom4j.Node;
|
||||
|
||||
public class XmlUtils {
|
||||
public static String getNodeValue(Node parentNode, String nodeName, String defaultValue) {
|
||||
Node node = parentNode.selectSingleNode(nodeName);
|
||||
String ret = defaultValue;
|
||||
if (null == node) {
|
||||
return ret;
|
||||
} else {
|
||||
ret = node.getText();
|
||||
if (null == ret) {
|
||||
ret = defaultValue;
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public static int getNodeValueAsInt(Node parentNode, String nodeName, int defaultValue) {
|
||||
String value = getNodeValue(parentNode, nodeName, null);
|
||||
if (null == value) {
|
||||
return defaultValue;
|
||||
} else {
|
||||
return Integer.parseInt(value);
|
||||
}
|
||||
}
|
||||
|
||||
public static long getNodeValueAsLong(Node parentNode, String nodeName, long defaultValue) {
|
||||
String value = getNodeValue(parentNode, nodeName, null);
|
||||
if (null == value) {
|
||||
return defaultValue;
|
||||
} else {
|
||||
return Long.parseLong(value);
|
||||
}
|
||||
}
|
||||
|
||||
public static boolean getNodeValueAsBoolean(Node parentNode, String nodeName, boolean defaultValue) {
|
||||
String value = getNodeValue(parentNode, nodeName, null);
|
||||
if (null == value) {
|
||||
return defaultValue;
|
||||
} else {
|
||||
return Boolean.parseBoolean(value);
|
||||
}
|
||||
}
|
||||
public class XmlUtils
|
||||
{
|
||||
public static String getNodeValue(final Node parentNode, final String nodeName, final String defaultValue) {
|
||||
final Node node = parentNode.selectSingleNode(nodeName);
|
||||
String ret = defaultValue;
|
||||
if (null == node) {
|
||||
return ret;
|
||||
}
|
||||
ret = node.getText();
|
||||
if (null == ret) {
|
||||
ret = defaultValue;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public static int getNodeValueAsInt(final Node parentNode, final String nodeName, final int defaultValue) {
|
||||
final String value = getNodeValue(parentNode, nodeName, null);
|
||||
if (null == value) {
|
||||
return defaultValue;
|
||||
}
|
||||
return Integer.parseInt(value);
|
||||
}
|
||||
|
||||
public static long getNodeValueAsLong(final Node parentNode, final String nodeName, final long defaultValue) {
|
||||
final String value = getNodeValue(parentNode, nodeName, null);
|
||||
if (null == value) {
|
||||
return defaultValue;
|
||||
}
|
||||
return Long.parseLong(value);
|
||||
}
|
||||
|
||||
public static boolean getNodeValueAsBoolean(final Node parentNode, final String nodeName, final boolean defaultValue) {
|
||||
final String value = getNodeValue(parentNode, nodeName, null);
|
||||
if (null == value) {
|
||||
return defaultValue;
|
||||
}
|
||||
return Boolean.parseBoolean(value);
|
||||
}
|
||||
}
|
||||
|
||||
+24
-3
@@ -918,6 +918,19 @@ public class WeiXinEvaluationController {
|
||||
return jsonMmodel;
|
||||
}
|
||||
|
||||
public Object saveBhLoanAction(EvaluationModel model){
|
||||
JsonModel jsonMmodel = new JsonModel();
|
||||
if(StringUtils.isNotBlank(model.getEvaluationId())){
|
||||
FjtEvaluation fjtEvaluation = fjtEvaluationService.findEvaluationById(Long.valueOf(model.getEvaluationId()));
|
||||
if(fjtEvaluation != null && fjtEvaluation.getLoan()!=model.getLoan()){
|
||||
fjtEvaluation.setLoan(model.getLoan());
|
||||
fjtEvaluation.setLastUpdated(new Date());
|
||||
fjtEvaluationService.updateEvaluation(fjtEvaluation);
|
||||
}
|
||||
}
|
||||
return jsonMmodel;
|
||||
}
|
||||
|
||||
//刷新审核价
|
||||
public Object refreshAuditPriceAction(EvaluationModel model){
|
||||
JsonModel jsonMmodel = new JsonModel();
|
||||
@@ -1106,9 +1119,15 @@ public class WeiXinEvaluationController {
|
||||
public Object checkImageAction(EvaluationModel model){
|
||||
JsonModel jsonModel = new JsonModel();
|
||||
if (StringUtils.isNotBlank(model.getEvaluationId())){
|
||||
List<ReportFileAttach> datas = reportFileAttachService.findHouseImage(Integer.valueOf(model.getEvaluationId()), 3);
|
||||
List<ReportFileAttach> datas3 = reportFileAttachService.findHouseImage(Integer.valueOf(model.getEvaluationId()), 3);
|
||||
Map<String, Integer> data = new HashMap<String, Integer>();
|
||||
data.put("count", datas.size());
|
||||
data.put("count_3", datas3.size());
|
||||
|
||||
FjtEvaluation evaluation = fjtEvaluationService.findEvaluationById(Long.valueOf(model.getEvaluationId()));
|
||||
if(evaluation!=null && evaluation.getLoan()!=null && evaluation.getLoan()==1){
|
||||
List<ReportFileAttach> datas5 = reportFileAttachService.findHouseImage(Integer.valueOf(model.getEvaluationId()), 5);
|
||||
data.put("count_5", datas5.size());
|
||||
}
|
||||
jsonModel.setJsonModel(data);
|
||||
}
|
||||
return jsonModel;
|
||||
@@ -1287,7 +1306,7 @@ public class WeiXinEvaluationController {
|
||||
FjtUsers fjtUser = fjtUsersService.findById(Long.valueOf(userId));
|
||||
UsersDetail usersDetail = usersDetailService.findById(fjtUser.getId());
|
||||
try {
|
||||
String id = new ObjectId().toString();
|
||||
|
||||
List<File> uploadfiles = new ArrayList<File>();
|
||||
for (CommonsMultipartFile file : files) {
|
||||
if (file.getSize() == 0 ) {
|
||||
@@ -1319,6 +1338,8 @@ public class WeiXinEvaluationController {
|
||||
} catch (IOException e) {
|
||||
//处理异常信息
|
||||
}
|
||||
String id = new ObjectId().toString();
|
||||
|
||||
reportFileAttach = new ReportFileAttach();
|
||||
reportFileAttach.setEvaluationId(Integer.valueOf(model.getEvaluationId()));
|
||||
reportFileAttach.setQueryId(model.getQueryId());
|
||||
|
||||
@@ -45,6 +45,7 @@ public class EvaluationModel extends BaseModel{
|
||||
private Map<String, Object> loanInfoMap;
|
||||
private Integer hopePrice;
|
||||
private String remark;
|
||||
private Integer loan;
|
||||
// private String preCode;//渤海接口--楼盘编码
|
||||
// private String preName;//渤海接口--楼盘名称
|
||||
// private String countryCode;//渤海接口--区域编码
|
||||
@@ -317,4 +318,12 @@ public class EvaluationModel extends BaseModel{
|
||||
public void setAddress(String address) {
|
||||
this.address = address;
|
||||
}
|
||||
|
||||
public Integer getLoan() {
|
||||
return loan;
|
||||
}
|
||||
|
||||
public void setLoan(Integer loan) {
|
||||
this.loan = loan;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -190,6 +190,36 @@ $('#submitRemark').on('click', function () {
|
||||
}
|
||||
});
|
||||
|
||||
$('#submitLoan').on('click', function () {
|
||||
var opt = $("#submitLoan").text();
|
||||
if(opt == "修改"){
|
||||
$(this).parent().find("span").eq(1).hide()
|
||||
$("input[name=loan]").removeClass("dn");
|
||||
$("#submitLoan").text("保存");
|
||||
}else{
|
||||
var loan = $("input[name=loan]").val();
|
||||
if(loan == ""){
|
||||
$.toast("请输入是否续贷");
|
||||
}else{
|
||||
$.ajax({
|
||||
url:"saveBhLoan",
|
||||
data:{
|
||||
evaluationId:eId,
|
||||
loan:loan
|
||||
},
|
||||
dataType:"json",
|
||||
success:function(data){
|
||||
$("#submitLoan").remove();
|
||||
$("input[name=loan]").remove();
|
||||
$("#refreshPrice0").show();
|
||||
location.href ="bhreport?evaluationId="+eId;
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
$('#refreshPrice,#refreshPrice0').on('click', function () {
|
||||
$.ajax({
|
||||
url:"refreshAuditPrice",
|
||||
@@ -301,21 +331,34 @@ $('.download_btn').on('click', function () {
|
||||
|
||||
/* 下载评估报告 */
|
||||
$('.formal_btn').on('click', function () {
|
||||
$.ajax({
|
||||
url:"checkImage",
|
||||
data:{
|
||||
evaluationId:eId
|
||||
},
|
||||
dataType:"json",
|
||||
success:function(data){
|
||||
if(data && data.count>0){
|
||||
clickFormalReport()
|
||||
}else{
|
||||
$.alert('请上传房屋照片!');
|
||||
if(isLoan!=null) {
|
||||
$.ajax({
|
||||
url:"checkImage",
|
||||
data:{
|
||||
evaluationId:eId
|
||||
},
|
||||
dataType:"json",
|
||||
success:function(data){
|
||||
if(isLoan==1){
|
||||
if(data.count_5==0){
|
||||
$.alert('请上传产调报告或权籍!');
|
||||
}else if(data.count_3==0){
|
||||
$.alert('请上传房屋照片!');
|
||||
}else{
|
||||
clickFormalReport()
|
||||
}
|
||||
}else{
|
||||
if(data.count_3==0){
|
||||
$.alert('请上传房屋照片!');
|
||||
}else{
|
||||
clickFormalReport()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
})
|
||||
}else{
|
||||
$.alert('请选择是否续贷!');
|
||||
}
|
||||
});
|
||||
|
||||
function clickFormalReport(){
|
||||
|
||||
@@ -134,7 +134,7 @@ input::-webkit-input-placeholder{
|
||||
.inputTopic01{margin-bottom:-1px;}
|
||||
.inputTopic01 .topicTitle,.inputTopic02 .topicTitle{margin-right:0;border:none;line-height:2.5rem;}
|
||||
.inputTopic01 .topicTitle .input_price{width:20%;padding-left:5px;}
|
||||
.inputTopic02 .topicTitle .input_lender{width:40%;padding-left:5px;}
|
||||
.inputTopic02 .topicTitle .input_lender{width:39%;padding-left:5px;line-height: 28px;}
|
||||
.inputTopic02 .topicTitle .input_cardId{width:35%;padding-left:5px;}
|
||||
.submit_btn{width:90px;border-left:solid 1px #e7e7e7;float:right;padding-right:0;text-align:center;cursor:pointer;}
|
||||
.left01{float:left;margin-right:8%;}
|
||||
|
||||
@@ -132,6 +132,18 @@
|
||||
</div>
|
||||
</#if>
|
||||
|
||||
<div class="topic inputTopic02" id="inputTopic02b" style="border-bottom:none;margin-bottom:0;">
|
||||
<div class="topicTitle topicTitle01">
|
||||
<span>是否续贷:</span>
|
||||
<input class="input_lender" name="loan" id="loan" type="hidden" required/>
|
||||
<p class="di"><a href="#" class="create-actions01"><#if fjtEvaluation.loan?? && fjtEvaluation.loan==1>是<#elseif fjtEvaluation.loan?? && fjtEvaluation.loan==0>否<#else>请选择</#if></a></p>
|
||||
<a class="submit_btn" id="submitLoan" style="border-left: none;"><#if fjtEvaluation.loan??>修改<#else>提交</#if></a>
|
||||
</div>
|
||||
<div class="topicTitle topicTitle01" style="color: #F4A460;font-size: 12px;line-height: 20px;margin-top: -5px;">
|
||||
注:如出具正式报告时,需上传产调报告或权籍
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<form id="fileForm" enctype="multipart/form-data" method="post" name="fileForm">
|
||||
<input type="hidden" name="evaluationId" value="${fjtEvaluation.id}">
|
||||
<input type="hidden" name="queryId" value="<#if bohaiEvaluationAudit??>${bohaiEvaluationAudit.queryId}</#if>">
|
||||
@@ -139,7 +151,7 @@
|
||||
<#if reportFileMap??>
|
||||
<div class="topicTitle topicTitle01 ">
|
||||
<span>已上传文件:</span>
|
||||
<#if reportFileMap["1"]??><span>询价单(${reportFileMap["1"]?size})</span></#if>
|
||||
<#if reportFileMap["5"]??><span>产调报告或权籍(${reportFileMap["5"]?size})</span></#if>
|
||||
<#if reportFileMap["2"]??><label style="margin-left: 10px;">产证(${reportFileMap["2"]?size})</label></#if>
|
||||
<#if reportFileMap["3"]??><label style="margin-left: 10px;">房屋照片(${reportFileMap["3"]?size})</label></#if>
|
||||
<#if reportFileMap["4"]??><label style="margin-left: 10px;">其它附件(${reportFileMap["4"]?size})</label></#if>
|
||||
@@ -456,6 +468,10 @@
|
||||
<script type="text/javascript">
|
||||
var eId = '${fjtEvaluation.id!}';
|
||||
var role = ${role}
|
||||
var isLoan = null
|
||||
<#if fjtEvaluation?? && fjtEvaluation.loan??>
|
||||
isLoan = ${fjtEvaluation.loan}
|
||||
</#if>
|
||||
showOtherSource(eId);
|
||||
//showTradeReference(eId);
|
||||
showHouseReference(eId);
|
||||
@@ -522,11 +538,11 @@
|
||||
label: true
|
||||
},
|
||||
{
|
||||
text: '询价单',
|
||||
text: '产调报告或权籍(必填)',
|
||||
bold: true,
|
||||
onClick: function () {
|
||||
$("a.create-actions08").empty().text(this.text).addClass("selected")
|
||||
$("#fileClass").val("1")
|
||||
$("#fileClass").val("5")
|
||||
$("input[name=reportFiles]").attr("accept","image/*")
|
||||
}
|
||||
},
|
||||
@@ -576,6 +592,41 @@ function changeFile(d){
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
$(document).on('click', '.create-actions01', function () {
|
||||
var buttons1 = [
|
||||
{
|
||||
text: '请选择是否续贷',
|
||||
label: true
|
||||
},
|
||||
{
|
||||
text: '是',
|
||||
bold: true,
|
||||
onClick: function () {
|
||||
$("#loan").empty().text(this.text).addClass("selected")
|
||||
$(".create-actions01").text(this.text)
|
||||
$("#loan").val("1")
|
||||
}
|
||||
},
|
||||
{
|
||||
text: '否',
|
||||
onClick: function () {
|
||||
$("#loan").empty().text(this.text).addClass("selected")
|
||||
$(".create-actions01").text(this.text)
|
||||
$("#loan").val("0")
|
||||
}
|
||||
}
|
||||
];
|
||||
var buttons2 = [
|
||||
{
|
||||
text: '取消',
|
||||
bg: 'gray'
|
||||
}
|
||||
];
|
||||
var groups = [buttons1, buttons2];
|
||||
$.actions(groups);
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
+1
-8
@@ -23,14 +23,7 @@ class CityInfoEmailSender extends BaseBatch{
|
||||
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
|
||||
final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files')
|
||||
def emails = [
|
||||
"wangsheng@fangjia.com"
|
||||
,"xuyejun@fangjia.com"
|
||||
,"tiki@fangjia.com"
|
||||
,"zhangzx@fangjia.com"
|
||||
,"panyingjiao@fangjia.com"
|
||||
,"yangqixing@fangjia.com"
|
||||
,"james.lin@fangjia.com"
|
||||
,"jhu@fangjia.com"
|
||||
"gaojuan@fangjia.com"
|
||||
]
|
||||
|
||||
|
||||
|
||||
+11
-1
@@ -16,6 +16,7 @@ import com.fangjia.script.util.ServerAreaMap
|
||||
import com.google.common.collect.MapDifference
|
||||
import com.google.common.collect.Maps
|
||||
import com.mongodb.*
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
/**
|
||||
@@ -47,7 +48,16 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
void process(Object args) {
|
||||
def cities = ServerAreaMap.findAllCities()
|
||||
String cityParam = System.getProperty("city");
|
||||
def cities = new ArrayList()
|
||||
if(StringUtils.isNotBlank(cityParam)){
|
||||
cities = new ArrayList()
|
||||
for(String c in cityParam.split(",")){
|
||||
cities.add(c);
|
||||
}
|
||||
}else{
|
||||
cities = ServerAreaMap.findAllCities()
|
||||
}
|
||||
println cities
|
||||
Date weekend = getLastDayOfMonth()
|
||||
println weekend
|
||||
|
||||
-1
@@ -383,7 +383,6 @@ class DealPriceDataProcess {
|
||||
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
|
||||
sqlInstance236.executeUpdate(sql)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+20
-1
@@ -48,13 +48,14 @@ class AnjukeAppCrawl {
|
||||
// task.run();
|
||||
// task.run2()
|
||||
|
||||
task.addThirdPrice()
|
||||
// task.addThirdPrice()
|
||||
// task.saveToEs()
|
||||
// task.saveToEs1()
|
||||
|
||||
// task.groupByCity()
|
||||
// task.selectNoSave()
|
||||
// task.statCount()
|
||||
task.exportFzDist()
|
||||
}
|
||||
def saveToEs1(){
|
||||
def lists = getAllAppDatas()
|
||||
@@ -75,6 +76,24 @@ class AnjukeAppCrawl {
|
||||
})
|
||||
}
|
||||
|
||||
def exportFzDist(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","福州")
|
||||
def dbcursor = detail.find(q).batchSize(5000)
|
||||
def index = 0
|
||||
def rets = []
|
||||
rets << ["city","region","name","_id","id","sale_num","city_short","source"]
|
||||
for(def d : dbcursor){
|
||||
def count = 0
|
||||
if(d.propInfo.saleNum){
|
||||
count = d.propInfo.saleNum.toInteger()
|
||||
}
|
||||
rets << [d.base.cityName, d.base.areaName, d.base.name, d._id.toString(),d.base.id, count, "fz","安居客"]
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\福州\\福州小区.xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
def selectNoSave(){
|
||||
def esData = findAllEsData()
|
||||
def q = new BasicDBObject()
|
||||
|
||||
+16
-5
@@ -25,14 +25,14 @@ class AnjukeHouseCrawl {
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
|
||||
def href = "https://sh.esf.fang.com/loupan/1210043376.htm"
|
||||
def href = "https://fz.anjuke.com/prop/view/S2512363126645770"
|
||||
def doc = getDocument(href, driver)
|
||||
// def doc = ClaABuYun.getResponse(href)
|
||||
// def newUrl = href.substring(0, hrefs.indexOf("?"))
|
||||
sendMq(href, doc, "district_detail_fang")
|
||||
sendMq(href, doc, "sell_anjuke")
|
||||
|
||||
// for(def i=1;i<50;i++){
|
||||
// def url = "https://fz.anjuke.com/sale/guloue/p${i}/"
|
||||
// for(def i=25;i<50;i++){
|
||||
// def url = "https://fz.anjuke.com/sale/jinana/p${i}/"
|
||||
// def doc = getDocument(url, driver)
|
||||
// def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc)
|
||||
// println "Ò³Êý£º"+i+" ¼Ç¼Êý£º"+eles.size()
|
||||
@@ -203,7 +203,18 @@ class AnjukeHouseCrawl {
|
||||
}
|
||||
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".village_info", doc)
|
||||
els = JQueryUtils.select(".detail-content", doc)
|
||||
}
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".content", doc)
|
||||
}
|
||||
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".wrapper", doc)
|
||||
}
|
||||
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".main-wrap", doc)
|
||||
}
|
||||
if(els && els.size()>0){
|
||||
stop = true;
|
||||
|
||||
+2
-2
@@ -23,14 +23,14 @@ class ZhugeFangDistCrawl {
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
def url = "https://fz.esfxiaoqu.zhuge.com/pingtan/"
|
||||
def url = "https://fz.esfxiaoqu.zhuge.com/changle/"
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.manage().deleteAllCookies()
|
||||
getRegions(driver, url)
|
||||
}
|
||||
|
||||
def getRegions(driver, url) {
|
||||
for(def i=1;i<=4;i++){
|
||||
for(def i=1;i<=7;i++){
|
||||
def newUrl = url+"page${i}/"
|
||||
println newUrl
|
||||
def doc = getDocument(newUrl, driver)
|
||||
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class ExportPaimai {
|
||||
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
def task = new ExportPaimai();
|
||||
task.run()
|
||||
// task.addressStandard()
|
||||
// task.addPrice()
|
||||
// task.stat()
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def file = "D:\\数据\\重庆农商行\\法拍\\宿迁法拍数据.xlsx"
|
||||
def datas = ExcelUtils.read(file, 0, 1)
|
||||
def index1 = 0
|
||||
|
||||
def rates1 = []
|
||||
for(def d : datas){
|
||||
if(d['P'] && d['P']=='已成交'){
|
||||
if(d['M'] && d['M']=="住宅")
|
||||
// if(d['K']>=150){
|
||||
// index1++
|
||||
// def sumTradePrice = d['S']
|
||||
// def marketPrice = d['Y']
|
||||
// rates1 << sumTradePrice/marketPrice
|
||||
// }
|
||||
if(d['S']>2000000){
|
||||
index1++
|
||||
def sumTradePrice = d['S']
|
||||
def marketPrice = d['Y']
|
||||
rates1 << sumTradePrice/marketPrice
|
||||
}
|
||||
}
|
||||
}
|
||||
def rate = rates1.sum()/rates1.size()
|
||||
println index1 +" "+ rate
|
||||
}
|
||||
|
||||
def addPrice(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\宿迁法拍数据.xlsx",{d->
|
||||
if(!d['Y'] && d['Z']){
|
||||
d['Y'] = d['Z']
|
||||
}
|
||||
})
|
||||
}
|
||||
def addressStandard(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\重庆法拍数据.xlsx",{d->
|
||||
def formatAddress = AddressUtil.formatAddress(d['C'], d['A'])
|
||||
if(formatAddress){
|
||||
d['F'] = formatAddress
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findPaimaiData()
|
||||
println "总量:"+datas.size()
|
||||
def rows = []
|
||||
def header = ["标地物地址","省","城市","区域","地址","标题提取地址","楼栋号","单元","室号","楼盘名称","建筑面积","竣工日期","房屋用途","总楼层",
|
||||
"拍卖状态","是否成交","拍卖阶段","拍卖方式","成交价","拍卖开始时间","拍卖结束时间","起拍价","保证金","加价幅度","评估价","市场价",
|
||||
"变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数","URL"]
|
||||
rows << header
|
||||
def set = new HashSet()
|
||||
def index = 0
|
||||
for(def d : datas){
|
||||
if(!set.contains(d.title)){
|
||||
index++
|
||||
println index
|
||||
set << d.title
|
||||
def row = []
|
||||
row << d.title
|
||||
row << d.province
|
||||
row << d.city
|
||||
row << d.region
|
||||
row << d.address
|
||||
row << ""
|
||||
// def formatAddress = AddressUtil.formatAddress(d.city, d.title)
|
||||
// if(formatAddress){
|
||||
// row << formatAddress
|
||||
// }else{
|
||||
// row << ""
|
||||
// }
|
||||
row << d.houseNum
|
||||
row << d.unitNum
|
||||
row << d.roomNum
|
||||
row << d.matchName
|
||||
row << d.area
|
||||
row << d.buildYear
|
||||
row << d.estateType
|
||||
row << ""
|
||||
row << ""
|
||||
row << d.status
|
||||
row << d.stage
|
||||
row << ""
|
||||
row << d.tradePrice
|
||||
row << DateUtils.formatDateByPattern(d.startDate, "yyyy-MM-dd")
|
||||
row << DateUtils.formatDateByPattern(d.finishDate, "yyyy-MM-dd")
|
||||
row << d.startPrice
|
||||
row << d.depositPrice
|
||||
row << d.addPrice
|
||||
row << d.evalatePrice
|
||||
row << d.marketPrice
|
||||
row << d.sellOffAdvancePay
|
||||
row << d.pricePkWeekly
|
||||
row << d.delayWeekly
|
||||
row << d.preemptivePerson
|
||||
row << d.enrollCount
|
||||
row << d.lookCount
|
||||
row << d.callCount
|
||||
row << d.delayCount
|
||||
row << d.url
|
||||
|
||||
rows << row
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\重庆法拍原数据.xlsx",rows,"数据")
|
||||
}
|
||||
|
||||
def findPaimaiData(){
|
||||
def pmquery = new BasicDBObject()
|
||||
pmquery.put("city","重庆")
|
||||
pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
|
||||
pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-06-01")))
|
||||
pmquery.put("status","已成交")
|
||||
def dbcursor = paimai43_COLL.find(pmquery).sort(new BasicDBObject("finishDate",-1))
|
||||
def rets = []
|
||||
for(def db : dbcursor){
|
||||
// if(db.estateType && db.estateType=="车库"){
|
||||
// continue
|
||||
// }
|
||||
rets << db
|
||||
}
|
||||
return rets
|
||||
}
|
||||
}
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.alibaba.fastjson.JSON
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.temp.shenjian.util.MessageUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
|
||||
DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ali_fapai_detail", "192.168.0.235", 27018)
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
|
||||
|
||||
//Date date=new Date(1550455200000);
|
||||
//SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
//String str = format.format(date);
|
||||
//System.out.println(str);
|
||||
|
||||
def pmquery = new BasicDBObject()
|
||||
pmquery.put("city","ËÞǨ")
|
||||
pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
|
||||
//pmquery.put("status","Òѳɽ»")
|
||||
//pmquery.put("estateType", null)
|
||||
pmquery.put("url","https://sf-item.taobao.com/sf_item/665574733237.htm")
|
||||
|
||||
def cur = paimai43_COLL.find(pmquery)
|
||||
def urls = []
|
||||
for(def o : cur){
|
||||
urls << o.url
|
||||
}
|
||||
println "´ý½âÎö£º"+urls
|
||||
def index = 0
|
||||
for(def url : urls){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", url)
|
||||
def dbcursor = collection.find(q)
|
||||
index++
|
||||
println index
|
||||
for(def d : dbcursor){
|
||||
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "paimai_taobao")
|
||||
json.put("url", d.url)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", d.content)
|
||||
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
|
||||
result.put("trades", bid_content)
|
||||
result.put("condition", d.condition)
|
||||
result.put("sub_count", d.sub_count)
|
||||
def data_json = net.sf.json.JSONObject.fromObject(d.json)
|
||||
result.put("data_json", data_json)
|
||||
result.put("desc_content", d.desc_content)
|
||||
result.put("item_notice_content", d.item_notice_content)
|
||||
result.put("notice_detail_content", d.notice_detail_content)
|
||||
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+1
-3
@@ -44,9 +44,7 @@ class PdsUseStatisticBatch extends BaseBatch{
|
||||
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
|
||||
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000)
|
||||
def static emails = [
|
||||
"hubingliang@fangjia.com",
|
||||
"james.lin@fangjia.com",
|
||||
"panyingjiao@fangjia.com",
|
||||
|
||||
"duweiliang@fangjia.com",
|
||||
"yaojingmeng@fangjia.com",
|
||||
"xuqi@fangjia.com",
|
||||
|
||||
+1
-2
@@ -34,8 +34,7 @@ class DaiQianGuZhiBatch extends BaseBatch{
|
||||
final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files')
|
||||
def static emails = [
|
||||
"hubingliang@fangjia.com",
|
||||
"james.lin@fangjia.com",
|
||||
"panyingjiao@fangjia.com"
|
||||
"james.lin@fangjia.com"
|
||||
]
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-2
@@ -34,8 +34,7 @@ class KeHuFenLeiBatch extends BaseBatch{
|
||||
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000)
|
||||
def static emails = [
|
||||
"hubingliang@fangjia.com",
|
||||
"james.lin@fangjia.com",
|
||||
"panyingjiao@fangjia.com"
|
||||
"james.lin@fangjia.com"
|
||||
]
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-3
@@ -33,9 +33,7 @@ class KeHuPingJiBatch extends BaseBatch{
|
||||
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
|
||||
final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files')
|
||||
def static emails = [
|
||||
"hubingliang@fangjia.com",
|
||||
"james.lin@fangjia.com",
|
||||
"panyingjiao@fangjia.com"
|
||||
|
||||
]
|
||||
|
||||
static main(args) {
|
||||
|
||||
+2
-2
@@ -34,7 +34,7 @@ sqlInstance.eachRow(sql, { r ->
|
||||
if (year_month == '202205') {
|
||||
fiveMap.put(key, r['avg_price'] + "|" + r['calc_type'])
|
||||
} else if ((year_month == '202206')) {
|
||||
fiveMap.put(key, r['avg_price'] + "|" + r['calc_type'])
|
||||
sixMap.put(key, r['avg_price'] + "|" + r['calc_type'])
|
||||
}
|
||||
})
|
||||
for (String key : sixMap.keySet()) {
|
||||
@@ -47,4 +47,4 @@ for (String key : sixMap.keySet()) {
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("F:\\数据\\价格\\东莞上海价格偏差大.xlsx", result, "统计")
|
||||
ExcelUtils.write("D:\\数据\\价格\\东莞上海价格偏差大.xlsx", result, "统计")
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
package com.fangjia.script.temp.shenjian.poi
|
||||
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.statistics.poi.AmapPoi
|
||||
import com.fangjia.script.statistics.poi.AmapPoiDetail
|
||||
import com.fangjia.script.temp.shenjian.poi.entity.Community
|
||||
import com.fangjia.script.temp.shenjian.util.CommonDtoUtils
|
||||
import com.fangjia.script.temp.shenjian.util.EsUtils
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import org.apache.commons.collections.CollectionUtils
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.stream.Collectors
|
||||
|
||||
/**
|
||||
* 从ES获取POI
|
||||
*/
|
||||
class GetPoiFromCommunity {
|
||||
|
||||
static final ExecutorService executorService = Executors.newFixedThreadPool(10)
|
||||
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail"
|
||||
// 住宅类
|
||||
static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304";
|
||||
|
||||
static void main(String[] args) {
|
||||
AtomicLong atomicLong = new AtomicLong(0)
|
||||
Map<String, String> map = new ConcurrentHashMap<>()
|
||||
def rets = this.findCommunities()
|
||||
CountDownLatch countDownLatch = new CountDownLatch(rets.size())
|
||||
for (def retTemp : rets) {
|
||||
final def ret = retTemp
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
final String id = ret._id // 索引ID,并非小区ID
|
||||
String city = ret.city
|
||||
String region = ret.region
|
||||
String seaweedName = ret.name
|
||||
String address = ret.address
|
||||
List<AmapPoi> pois = new ArrayList<>();
|
||||
List<AmapPoi> namePois = GaoDeUtils.searchPoi(city, seaweedName, region, typecodes);
|
||||
List<AmapPoi> addressPois = GaoDeUtils.searchPoi(city, address, region, typecodes);
|
||||
if (CollectionUtils.isNotEmpty(namePois)) {
|
||||
pois.addAll(namePois)
|
||||
}
|
||||
if (CollectionUtils.isNotEmpty(addressPois)) {
|
||||
pois.addAll(addressPois)
|
||||
}
|
||||
if (CollectionUtils.isNotEmpty(pois)) {
|
||||
List<AmapPoiDetail> detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois);
|
||||
String jsonStr = JSONObject.toJSONString(detailList)
|
||||
HttpUtils.postForJson(URL, jsonStr)
|
||||
}
|
||||
synchronized (map) {
|
||||
map.put(id, id)
|
||||
countDownLatch.countDown()
|
||||
if (map.size() % 100 == 0) {
|
||||
atomicLong.addAndGet(map.size())
|
||||
List<String> ids = map.values().stream().collect(Collectors.toList())
|
||||
EsUtils.bulkUpdateFieldByIds("community", "crawl_flag", 0, ids)
|
||||
map.clear()
|
||||
println(atomicLong.intValue())
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
countDownLatch.await()
|
||||
if (map.size() > 0) {
|
||||
atomicLong.addAndGet(map.size())
|
||||
List<String> ids = map.values().stream().collect(Collectors.toList())
|
||||
EsUtils.bulkUpdateFieldByIds("community", "crawl_flag", 0, ids)
|
||||
map.clear()
|
||||
println(atomicLong.intValue())
|
||||
}
|
||||
System.exit(1)
|
||||
}
|
||||
|
||||
static def findCommunities(){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
||||
.mustNot(QueryBuilders.termQuery("crawl_flag", 0))
|
||||
List<Community> communities = EsUtils.scrollQuery("community", queryBuilder, Community.class)
|
||||
return communities
|
||||
}
|
||||
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package com.fangjia.script.temp.shenjian.poi.entity;
|
||||
|
||||
public class Community extends MyId {
|
||||
private String id;
|
||||
private String city;
|
||||
private String region;
|
||||
private String name;
|
||||
private String address;
|
||||
|
||||
String getId() {
|
||||
return id
|
||||
}
|
||||
|
||||
void setId(String id) {
|
||||
this.id = id
|
||||
}
|
||||
|
||||
String getCity() {
|
||||
return city
|
||||
}
|
||||
|
||||
void setCity(String city) {
|
||||
this.city = city
|
||||
}
|
||||
|
||||
String getRegion() {
|
||||
return region
|
||||
}
|
||||
|
||||
void setRegion(String region) {
|
||||
this.region = region
|
||||
}
|
||||
|
||||
String getName() {
|
||||
return name
|
||||
}
|
||||
|
||||
void setName(String name) {
|
||||
this.name = name
|
||||
}
|
||||
|
||||
String getAddress() {
|
||||
return address
|
||||
}
|
||||
|
||||
void setAddress(String address) {
|
||||
this.address = address
|
||||
}
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package com.fangjia.script.temp.shenjian.poi.entity
|
||||
|
||||
class MyId {
|
||||
|
||||
/**
|
||||
* Ë÷Òýid
|
||||
*/
|
||||
String _id;
|
||||
|
||||
String get_id() {
|
||||
return _id
|
||||
}
|
||||
|
||||
void set_id(String _id) {
|
||||
this._id = _id
|
||||
}
|
||||
}
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
package com.fangjia.script.temp.shenjian.util
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils
|
||||
import com.fangjia.script.temp.shenjian.poi.entity.MyId
|
||||
import org.elasticsearch.action.DocWriteRequest
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.search.*
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.search.Scroll
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder
|
||||
import org.slf4j.Logger
|
||||
import org.slf4j.LoggerFactory
|
||||
|
||||
/**
|
||||
* @author shenjian
|
||||
* @since 2022/07/02
|
||||
*/
|
||||
class EsUtils {
|
||||
|
||||
private static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient()
|
||||
private static final Logger logger = LoggerFactory.getLogger(EsUtils.class)
|
||||
|
||||
/**
|
||||
* 滚动方式查询大批量es数据
|
||||
*
|
||||
* @param index 索引
|
||||
* @param boolQuery 查询语句
|
||||
* @param clazz 要返回的实体类型
|
||||
* @param <T>
|
||||
* @return
|
||||
*/
|
||||
public static <T extends MyId> List<T> scrollQuery(String index, QueryBuilder boolQuery, Class<T> clazz, def limitSize=300000) {
|
||||
List<T> result = new ArrayList<>();
|
||||
// 存活时间,当索引数据量特别大时,出现超时可能性大,此值适当调大
|
||||
Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30L));
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(boolQuery);
|
||||
// TEST 用!!!!!!!!!!!!!!!!!!!!!!
|
||||
//searchSourceBuilder.size(1);
|
||||
searchSourceBuilder.size(10000);
|
||||
SearchRequest searchRequest = new SearchRequest().indices(index).scroll(scroll).source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = null;
|
||||
try {
|
||||
searchResponse = ES_CLIENT.search(searchRequest, RequestOptions.DEFAULT);
|
||||
} catch (IOException e) {
|
||||
logger.error("查询es数据异常:{}", e);
|
||||
}
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
if (searchHits != null) {
|
||||
for (SearchHit searchHit : searchHits) {
|
||||
T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz);
|
||||
t.set_id(searchHit.getId())
|
||||
result.add(t);
|
||||
}
|
||||
}
|
||||
// 遍历搜索命中的数据,直到没有数据
|
||||
while (searchHits != null && searchHits.length > 0 && result.size() < limitSize) {
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
try {
|
||||
searchResponse = ES_CLIENT.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
} catch (IOException e) {
|
||||
logger.error("查询es数据异常:{}", e);
|
||||
}
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
if (searchHits != null && searchHits.length > 0) {
|
||||
for (SearchHit searchHit : searchHits) {
|
||||
T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz);
|
||||
t.set_id(searchHit.getId())
|
||||
result.add(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
// clean scroll
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
ClearScrollResponse clearScrollResponse = null;
|
||||
try {
|
||||
clearScrollResponse = ES_CLIENT.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
} catch (IOException e) {}
|
||||
boolean succeeded = clearScrollResponse.isSucceeded();
|
||||
if (succeeded) {
|
||||
return result;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 批量更新es某个字段值
|
||||
* @param index 索引名
|
||||
* @param field 字段名
|
||||
* @param val 字段新值
|
||||
* @param ids 要更新的文档id
|
||||
*/
|
||||
public static void bulkUpdateFieldByIds(String index, String field, Object val, List<String> ids) {
|
||||
if (ids.size() > 1000) {
|
||||
throw new RuntimeException("批量更新数量过多")
|
||||
}
|
||||
BulkRequest bulkRequest = new BulkRequest();
|
||||
for (String id : ids) {
|
||||
Map<String, Object> doc = new HashMap<>();
|
||||
doc.put(field, val)
|
||||
bulkRequest.add(new UpdateRequest(index, id).doc(doc));
|
||||
}
|
||||
ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT);
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger
|
||||
class GaoDeUtils {
|
||||
|
||||
// 新获取key
|
||||
def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418"]
|
||||
def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418", "10bcb421d2ee4e51ed9268d6dca58ad1"]
|
||||
def static AtomicInteger keyUsed = new AtomicInteger(0)
|
||||
|
||||
def static codeNameMap = new HashMap<String, String>(){{
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202204);
|
||||
def count2 = findTotalCount(source, 202205);
|
||||
def count1 = findTotalCount(source, 202205);
|
||||
def count2 = findTotalCount(source, 202206);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -132,7 +132,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
q.put("s_date", SHORT_DATE);
|
||||
def ret = []
|
||||
try{
|
||||
def dbcursor = dataProcessColl.find(q).sort(new BasicDBObject("m_date", -1).append("type",1))
|
||||
def dbcursor = dataProcessColl.find(q).sort(new BasicDBObject("type",1).append("m_date", -1))
|
||||
|
||||
if(dbcursor){
|
||||
for(def dbo : dbcursor){
|
||||
|
||||
@@ -1,6 +1,39 @@
|
||||
package com.fangjia.script.wenyan
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\国信达级联价格-上海.xlsx",{List item->
|
||||
// if(item["AB"]){
|
||||
// return
|
||||
// }
|
||||
// String region = null
|
||||
// String name = item["F"]
|
||||
// if(name.endsWith("小区")){
|
||||
// name = name.replaceAll("小区\$", "")
|
||||
// }
|
||||
// region = RegionData.formatRegion(item["B"], item["D"])
|
||||
// def results = []
|
||||
// DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(item["B"], region, name)).each {
|
||||
// if(!results.contains(it)){
|
||||
// results.add(it)
|
||||
// }
|
||||
// }
|
||||
// DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(item["B"], region).append("alias", name)).each {
|
||||
// if(!results.contains(it)){
|
||||
// results.add(it)
|
||||
// }
|
||||
// }
|
||||
// if(results.size() == 1){
|
||||
// item["Z"] = results[0]._id.toString()
|
||||
// item["AA"] = results[0].region
|
||||
// item["AB"] = results[0].name
|
||||
// println(item)
|
||||
// }
|
||||
//})
|
||||
|
||||
|
||||
|
||||
|
||||
import com.fangjia.core.ProvinceData
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
@@ -76,7 +109,48 @@ import org.apache.commons.lang.StringUtils
|
||||
// }, new BasicDBObject("township", 1))
|
||||
//}
|
||||
//println(count)
|
||||
//def GXD_CCB = [:]
|
||||
//def CCB_FJW = [:]
|
||||
//ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\4月级联待优化小区_3个城市.xlsx",sheetIndex = 1).each {List item->
|
||||
// if(item["V"] == "级联"){
|
||||
// if(item["Y"] == "国信达"){
|
||||
// GXD_CCB[item['J']] = item["E"]
|
||||
// }else if(item["Y"] == "房价网"){
|
||||
// CCB_FJW[item['E']] = item["J"]
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\4月级联待优化小区_3个城市.xlsx",{List item->
|
||||
// if(item["V"] == "级联" && item["Y"] == "国信达"){
|
||||
// def ccbId = GXD_CCB.get(item['J'], null)
|
||||
// def fjwId = CCB_FJW.get(ccbId, null)
|
||||
// if(fjwId != null){
|
||||
// item["AB"] = fjwId
|
||||
//// BasicDBObject seaweed = DbUtils.seaweed.findById(fjwId)
|
||||
//// List results = DbUtils.findAroundDistrict(seaweed)
|
||||
//// long total = 0
|
||||
//// long count = 0
|
||||
//// for(BasicDBObject s : results){
|
||||
//// def avg = DbUtils.findDistrictAveragePrice(seaweed.city, seaweed.region, seaweed.name, Date.parse('yyyyMMdd', '20220619'), seaweed.estate_type2)
|
||||
//// if(avg){
|
||||
//// total += avg.avg_price
|
||||
//// count += 1
|
||||
//// }
|
||||
//// }
|
||||
//// if(count > 0){
|
||||
//// item['Z'] = (int)(total/count)
|
||||
//// println(item)
|
||||
//// }
|
||||
// }
|
||||
// }
|
||||
//},sheetIndex = 1)
|
||||
|
||||
ElasticUtils.executeSqlQuery("select count(*), p_date from seawater where p_date > 20210501 group by p_date",{
|
||||
println(it)
|
||||
})
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\moban (3).xls",{List item->
|
||||
// item["B"] = RegionData.formatRegion(item["A"], item["B"])
|
||||
// println(item)
|
||||
//})
|
||||
int count = 0
|
||||
DbUtils.seaweed.find(DbUtils.buildSeaweedQuery("东莞")).each {def a->
|
||||
count += 1
|
||||
}
|
||||
println(count)
|
||||
@@ -0,0 +1,151 @@
|
||||
package com.fangjia.script.wenyan
|
||||
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.Document
|
||||
|
||||
//def data = ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\TC重新跑数据.xlsx")
|
||||
//def addMap = [:]
|
||||
//data.each {def list->
|
||||
// if(list["C"] == "有效"){
|
||||
// addMap[list["B"]] = list
|
||||
// }
|
||||
//}
|
||||
//ExcelUtils.modify("D:\\Users\\wenyan\\Downloads\\data_gj20220616.xlsx",{List item->
|
||||
// if(addMap.containsKey(item["B"])){
|
||||
// List list = addMap.get(item["B"])
|
||||
// item["C"] = list["D"]
|
||||
// item["D"] = list["E"]
|
||||
// item["E"] = list["F"]
|
||||
// item["S"] = "有效"
|
||||
// println(item)
|
||||
// }
|
||||
//})
|
||||
|
||||
def DISTRICT_ANALY_DATA = Mongo3Utils.getDbCollection("fangjia_stat", "district_analy_data")
|
||||
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\data_gj20220616.xlsx", { List item ->
|
||||
// if(item["S"] == null){
|
||||
// if(item["Q"] != null || item["R"] != null){
|
||||
// item["S"] = "有效"
|
||||
// }
|
||||
// }
|
||||
// if(item["S"] == null){
|
||||
// def source = null
|
||||
// JSONObject resp = OpenApiClient.addressMatch(item["A"], item["B"], "property")
|
||||
// source = "小区库"
|
||||
// if(resp.credit == 0){
|
||||
// resp = OpenApiClient.addressMatch(item["A"], item["B"], null)
|
||||
// source = "小区库"
|
||||
// }
|
||||
// if(resp.credit == 0){
|
||||
// resp = OpenApiClient.addressMatch(item["A"], item["B"], null, true)
|
||||
// source = "地图"
|
||||
// }
|
||||
//
|
||||
// if(resp.credit > 0){
|
||||
// println(resp)
|
||||
// item["C"] = resp.city
|
||||
// item["D"] = resp.district
|
||||
// item["E"] = resp.name
|
||||
// item["F"] = source
|
||||
// item["G"] = resp.lng
|
||||
// item["H"] = resp.lat
|
||||
// item["I"] = resp.propertyType
|
||||
// }
|
||||
// if(item["G"] != null && !["普通住宅", "商住", "别墅"].contains(item["I"])){
|
||||
// List seaweeds = DbUtils.findAroundDistrict(item["G"], item["H"], 1000 ,1, 2)
|
||||
// if(seaweeds.size() > 0){
|
||||
// BasicDBObject s = seaweeds[0]
|
||||
// item["C"] = s.city
|
||||
// item["D"] = s.region
|
||||
// item["E"] = s.name
|
||||
// item["F"] = "周边"
|
||||
// item["G"] = s.lng/10000000000d
|
||||
// item["H"] = s.lat/10000000000d
|
||||
// item["I"] = s.estate_type2
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
|
||||
// if(item["S"]|| item["T"]){
|
||||
// if(!item["i"]){
|
||||
// JSONObject result = GaoDeApiUtils.geo(item["A"], item["B"])
|
||||
// if(result != null && result.getInt("status") == 1){
|
||||
// JSONArray geocodes = result.getJSONArray("geocodes");
|
||||
// if(geocodes != null && geocodes.size() > 0){
|
||||
// JSONObject geocode = geocodes.getJSONObject(0);
|
||||
// String location = geocode.getString("location")
|
||||
// item["i"] = location.split(",")[0].toDouble()
|
||||
// item["j"] = location.split(",")[1].toDouble()
|
||||
// }
|
||||
// }
|
||||
// println(item)
|
||||
// }
|
||||
// }
|
||||
// if(item["D"] == null && item["F"] != null){
|
||||
// String kw = item["F"]
|
||||
// if(item["E"]){
|
||||
// kw = item["E"] + kw
|
||||
// }
|
||||
// JSONObject result = GaoDeApiUtils.geo(item["A"], kw)
|
||||
// if(result != null && result.getInt("status") == 1){
|
||||
// JSONArray geocodes = result.getJSONArray("geocodes");
|
||||
// if(geocodes != null && geocodes.size() > 0){
|
||||
// JSONObject geocode = geocodes.getJSONObject(0);
|
||||
// String location = geocode.getString("location")
|
||||
// item["i"] = location.split(",")[0].toDouble()
|
||||
// item["j"] = location.split(",")[1].toDouble()
|
||||
// }
|
||||
// }
|
||||
// println(item)
|
||||
// }
|
||||
// if(item["E"] != null && ["普通住宅", "商住", "别墅"].contains(item["I"])){
|
||||
// Document doc = DISTRICT_ANALY_DATA.find(new Document("city", item["C"]).append("region", item["D"]).append("district_name", item["E"])).first()
|
||||
// if(doc){
|
||||
// item["N"] =doc["rank"]
|
||||
// }
|
||||
// }
|
||||
// if (item["S"] != "有效" || item["F"] != null) {
|
||||
// return
|
||||
// }
|
||||
// JSONObject resp = OpenApiClient.addressMatch(item["A"], item["E"], "property", false, item["D"], null, "strict")
|
||||
// if(resp.credit > 0){
|
||||
// println(resp)
|
||||
// item["C"] = resp.city
|
||||
// item["D"] = resp.district
|
||||
// item["E"] = resp.name
|
||||
// item["F"] = "小区库"
|
||||
// item["G"] = resp.lng
|
||||
// item["H"] = resp.lat
|
||||
// item["I"] = resp.propertyType
|
||||
// }
|
||||
// println(resp)
|
||||
// if (item["F"] == null) {
|
||||
// String kw = item["E"]
|
||||
// if (item["D"]) {
|
||||
// kw = item["D"] + kw
|
||||
// }
|
||||
// JSONObject result = GaoDeApiUtils.geo(item["A"], kw)
|
||||
// if (result != null && result.getInt("status") == 1) {
|
||||
// JSONArray geocodes = result.getJSONArray("geocodes");
|
||||
// if (geocodes != null && geocodes.size() > 0) {
|
||||
// JSONObject geocode = geocodes.getJSONObject(0);
|
||||
// String location = geocode.getString("location")
|
||||
// item["G"] = location.split(",")[0].toDouble()
|
||||
// item["H"] = location.split(",")[1].toDouble()
|
||||
// item["F"] = "反查经纬度"
|
||||
// }
|
||||
// }
|
||||
// println(item)
|
||||
// }
|
||||
})
|
||||
+2
-2
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20220501", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20220601", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20220601", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20220701", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
|
||||
+1
-1
@@ -62,7 +62,7 @@ class ApiProductFieldStatBatch extends BaseBatch {
|
||||
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
|
||||
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000)
|
||||
def emails = [
|
||||
"yinjihuan@fangjia.com","panyingjiao@fangjia.com","liyan@fangjia.com", "jiangxiaochun@fangjia.com"
|
||||
"jiangxiaochun@fangjia.com"
|
||||
]
|
||||
static def allDis = []
|
||||
static DecimalFormat df = new DecimalFormat("0.00%")
|
||||
|
||||
+1
@@ -499,6 +499,7 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
addr.setAddressParserGoal(params.getAddressParserGoal());
|
||||
addr.setProvince(params.getProvince());
|
||||
addr.setDebugName(params.getDebugName());
|
||||
addr.setUseMap(params.isUseMap());
|
||||
return AddressProcessors.process(addr);
|
||||
}
|
||||
|
||||
|
||||
@@ -63,6 +63,7 @@ public class Address {
|
||||
private String suggestNamePinYin;
|
||||
private String polygonSeaweedId;
|
||||
private String geoRegion;
|
||||
private boolean useMap;
|
||||
private List<String> buildingNumbers;
|
||||
private PreciseLevel preciseLevel;
|
||||
private boolean pinYin;
|
||||
@@ -389,6 +390,14 @@ public class Address {
|
||||
isStandardAddress = standardAddress;
|
||||
}
|
||||
|
||||
public boolean isUseMap() {
|
||||
return useMap;
|
||||
}
|
||||
|
||||
public void setUseMap(boolean useMap) {
|
||||
this.useMap = useMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获得有效地址
|
||||
*
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user