This commit is contained in:
2022-07-13 11:45:50 +08:00
102 changed files with 7606 additions and 7453 deletions
@@ -1847,6 +1847,9 @@ public class DataUtil {
//
// System.out.println(FourUpSixInto("258.393", 2));
String href = encodePicUrl("62afd81d303c3f827833bb03", "o", "upload");
String link = "http://static.fangjia.com/stc/fs/"+href;
System.out.println(link);
}
}
@@ -105,6 +105,7 @@ public class FjtEvaluation implements SqlEntity{
private String certificateAddress; //产证地址
private Date downPdfDate;//下载报告时间
private Integer loan ;//是否续贷 1:是 0:否
@GeneratedValue(strategy = GenerationType.AUTO)
@Id
@@ -773,6 +774,15 @@ public class FjtEvaluation implements SqlEntity{
this.downPdfDate = downPdfDate;
}
@Column(name = "loan")
public Integer getLoan() {
return loan;
}
public void setLoan(Integer loan) {
this.loan = loan;
}
@Override
public String toString() {
return "FjtEvaluation{" +
@@ -11,7 +11,7 @@ public class ReportFileAttach implements SqlEntity {
private Integer userId;
private Integer evaluationId;
private String queryId;
private Integer fileClass;
private Integer fileClass;//1:询价单 2:产证 3:房屋照片 4:其它附件 5:产调报告或权籍
private String fileName;
private String fileId;
+3 -3
View File
@@ -33,7 +33,7 @@
<addClasspath>true</addClasspath>
<classpathPrefix>lib/</classpathPrefix>
<mainClass>com.fangjia.datastore.DataStoreServer</mainClass>
</manifest>
</manifest>
</archive>
</configuration>
</plugin>
@@ -94,8 +94,8 @@
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<version>2.6.5</version>
<artifactId>mongodb-driver-legacy</artifactId>
<version>3.9.1</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
@@ -1,22 +1,21 @@
package com.fangjia.datastore;
public class ConfigurationException extends Exception {
private static final long serialVersionUID = 6570405263317311822L;
public ConfigurationException() {
}
public ConfigurationException(String message) {
super(message);
}
public ConfigurationException(Throwable cause) {
super(cause);
}
public ConfigurationException(String message, Throwable cause) {
super(message, cause);
}
public class ConfigurationException extends Exception
{
private static final long serialVersionUID = 6570405263317311822L;
public ConfigurationException() {
}
public ConfigurationException(final String message) {
super(message);
}
public ConfigurationException(final Throwable cause) {
super(cause);
}
public ConfigurationException(final String message, final Throwable cause) {
super(message, cause);
}
}
@@ -1,80 +1,83 @@
package com.fangjia.datastore;
import com.fangjia.datastore.mongo.protocol.QueryMessage;
import java.util.concurrent.atomic.AtomicLong;
import com.fangjia.datastore.mongo.protocol.QueryMessage;
public class Cursor {
private static final AtomicLong ID_GENERATOR = new AtomicLong(1);
private long id;
private int limit;
private int skip;
private long totalCount;
private QueryMessage queryMessage;
public Cursor(int skip, int limit, QueryMessage queryMessage, long totalCount) {
id = ID_GENERATOR.getAndIncrement();
this.skip = skip;
this.limit = limit;
this.totalCount = totalCount;
this.queryMessage = queryMessage;
}
public long getId() {
return id;
}
public int getLimit() {
return limit;
}
public void setLimit(int limit) {
this.limit = limit;
}
public int nextSkip() {
skip += limit;
return skip;
}
public void setSkip(int skip) {
this.skip = skip;
}
public boolean hasMore() {
return this.totalCount > this.skip;
}
public long getTotalCount() {
return totalCount;
}
public void setTotalCount(long totalCount) {
this.totalCount = totalCount;
}
public QueryMessage getQueryMessage() {
return queryMessage;
}
public void setQueryMessage(QueryMessage queryMessage) {
this.queryMessage = queryMessage;
}
public int getSkip() {
return skip;
}
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("id:").append(id);
builder.append(", limit:").append(limit);
builder.append(", skip:").append(skip);
builder.append(", totalCount:").append(totalCount);
builder.append(", queryMessage:").append(queryMessage);
builder.append("}");
return builder.toString();
}
public class Cursor
{
private static final AtomicLong ID_GENERATOR;
private long id;
private int limit;
private int skip;
private long totalCount;
private QueryMessage queryMessage;
public Cursor(final int skip, final int limit, final QueryMessage queryMessage, final long totalCount) {
this.id = Cursor.ID_GENERATOR.getAndIncrement();
this.skip = skip;
this.limit = limit;
this.totalCount = totalCount;
this.queryMessage = queryMessage;
}
public long getId() {
return this.id;
}
public int getLimit() {
return this.limit;
}
public void setLimit(final int limit) {
this.limit = limit;
}
public int nextSkip() {
return this.skip += this.limit;
}
public void setSkip(final int skip) {
this.skip = skip;
}
public boolean hasMore() {
return this.totalCount > this.skip;
}
public long getTotalCount() {
return this.totalCount;
}
public void setTotalCount(final long totalCount) {
this.totalCount = totalCount;
}
public QueryMessage getQueryMessage() {
return this.queryMessage;
}
public void setQueryMessage(final QueryMessage queryMessage) {
this.queryMessage = queryMessage;
}
public int getSkip() {
return this.skip;
}
@Override
public String toString() {
final StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("id:").append(this.id);
builder.append(", limit:").append(this.limit);
builder.append(", skip:").append(this.skip);
builder.append(", totalCount:").append(this.totalCount);
builder.append(", queryMessage:").append(this.queryMessage);
builder.append("}");
return builder.toString();
}
static {
ID_GENERATOR = new AtomicLong(1L);
}
}
@@ -1,44 +1,35 @@
package com.fangjia.datastore;
import java.util.Collection;
import java.util.List;
import org.bson.BSONObject;
import com.fangjia.datastore.config.Reloadable;
import com.fangjia.datastore.config.StoreConfig;
import com.fangjia.datastore.solr.Index;
import java.util.Collection;
import com.fangjia.datastore.config.StoreConfig;
import com.fangjia.datastore.config.Reloadable;
public interface DataStore extends Reloadable {
StoreConfig getStoreConfig();
Mongo getMongo();
Object getAttribute(String key);
Object setAttribute(String key, Object attr);
Collection<Index> getIndexes() throws DataStoreException;
/**
* °´²éѯÌõ¼þÖØ½¨Ë÷Òý
*
* @param query
* @throws DataStoreException
*/
BSONObject reIndex(BSONObject query) throws DataStoreException;
void ensureIndex(Collection<Index> indexes) throws DataStoreException;
QueryResponse find(BSONObject query, BSONObject fields, BSONObject sort, BSONObject facet, int skip, int limit, boolean bulk,
boolean fromMaster) throws DataStoreException;
int count(BSONObject query, boolean fromMaster) throws DataStoreException;
void remove(BSONObject query) throws DataStoreException;
void update(BSONObject query, BSONObject data,String ip) throws DataStoreException;
void save(List<? extends BSONObject> docList) throws DataStoreException;
public interface DataStore extends Reloadable
{
StoreConfig getStoreConfig();
Mongo getMongo();
Object getAttribute(final String p0);
Object setAttribute(final String p0, final Object p1);
Collection<Index> getIndexes() throws DataStoreException;
BSONObject reIndex(final BSONObject p0) throws DataStoreException;
void ensureIndex(final Collection<Index> p0) throws DataStoreException;
QueryResponse find(final BSONObject p0, final BSONObject p1, final BSONObject p2, final BSONObject p3, final int p4, final int p5, final boolean p6, final boolean p7) throws DataStoreException;
int count(final BSONObject p0, final boolean p1) throws DataStoreException;
void remove(final BSONObject p0) throws DataStoreException;
int update(final BSONObject p0, final BSONObject p1) throws DataStoreException;
void save(final List<? extends BSONObject> p0) throws DataStoreException;
}
@@ -3,15 +3,15 @@ package com.fangjia.datastore;
import com.fangjia.datastore.config.DataStoreConfig;
import com.fangjia.datastore.config.Reloadable;
public interface DataStoreContext extends Reloadable {
DataStoreConfig getDataStoreConfig();
DataStore getDataStore(String storeName) throws DataStoreException;
Cursor getCursor(long cursorId);
void addCursor(Cursor cursor);
Cursor removeCursor(long cursorId);
public interface DataStoreContext extends Reloadable
{
DataStoreConfig getDataStoreConfig();
DataStore getDataStore(final String p0) throws DataStoreException;
Cursor getCursor(final long p0);
void addCursor(final Cursor p0);
Cursor removeCursor(final long p0);
}
@@ -1,56 +1,61 @@
package com.fangjia.datastore;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import com.fangjia.datastore.config.DataStoreConfig;
import com.fangjia.datastore.config.StoreConfig;
import java.util.Iterator;
import java.util.Set;
import java.util.HashMap;
import java.util.concurrent.TimeUnit;
import com.google.common.collect.MapMaker;
import java.util.Map;
import com.fangjia.datastore.config.DataStoreConfig;
public class DataStoreContextImpl implements DataStoreContext {
private final DataStoreConfig config;
private final Map<Long, Cursor> cursorMap;
private final Map<String, DataStore> stores;
public DataStoreContextImpl(DataStoreConfig config) throws ConfigurationException {
this.config = config;
cursorMap = new MapMaker().concurrencyLevel(50).expiration(config.getServerConfig().getCursorExpirationMinutes(),
TimeUnit.MINUTES).makeMap();
Set<String> storeNames = config.getStoreNames();
stores = new HashMap<String, DataStore>(storeNames.size());
for (String storeName : storeNames) {
StoreConfig storeConfig = this.config.getStoreConfig(storeName);
stores.put(storeName, new DataStoreImpl(storeConfig, new Mongo(storeConfig.getMongoConfig())));
}
}
public DataStoreConfig getDataStoreConfig() {
return config;
}
public DataStore getDataStore(String storeName) throws DataStoreException {
if (!stores.containsKey(storeName)) {
throw new DataStoreException("unknown store name: \"" + storeName + "\"");
}
return stores.get(storeName);
}
public void reload() throws ConfigurationException {
config.reload();
}
public Cursor getCursor(long cursorId) {
return this.cursorMap.get(cursorId);
}
public void addCursor(Cursor cursor) {
this.cursorMap.put(cursor.getId(), cursor);
}
public Cursor removeCursor(long cursorId) {
return this.cursorMap.remove(cursorId);
}
public class DataStoreContextImpl implements DataStoreContext
{
private final DataStoreConfig config;
private final Map<Long, Cursor> cursorMap;
private final Map<String, DataStore> stores;
public DataStoreContextImpl(final DataStoreConfig config) throws ConfigurationException {
this.config = config;
this.cursorMap = new MapMaker().concurrencyLevel(50).expiration((long)config.getServerConfig().getCursorExpirationMinutes(), TimeUnit.MINUTES).makeMap();
final Set<String> storeNames = config.getStoreNames();
this.stores = new HashMap<String, DataStore>(storeNames.size());
for (final String storeName : storeNames) {
final StoreConfig storeConfig = this.config.getStoreConfig(storeName);
this.stores.put(storeName, new DataStoreImpl(storeConfig, new Mongo(storeConfig.getMongoConfig())));
}
}
@Override
public DataStoreConfig getDataStoreConfig() {
return this.config;
}
@Override
public DataStore getDataStore(final String storeName) throws DataStoreException {
if (!this.stores.containsKey(storeName)) {
throw new DataStoreException("unknown store name: \"" + storeName + "\"");
}
return this.stores.get(storeName);
}
@Override
public void reload() throws ConfigurationException {
this.config.reload();
}
@Override
public Cursor getCursor(final long cursorId) {
return this.cursorMap.get(cursorId);
}
@Override
public void addCursor(final Cursor cursor) {
this.cursorMap.put(cursor.getId(), cursor);
}
@Override
public Cursor removeCursor(final long cursorId) {
return this.cursorMap.remove(cursorId);
}
}
@@ -1,23 +1,21 @@
package com.fangjia.datastore;
public class DataStoreException extends Exception {
private static final long serialVersionUID = -5492973985910379215L;
public DataStoreException() {
super();
}
public DataStoreException(String message, Throwable cause) {
super(message, cause);
}
public DataStoreException(String message) {
super(message);
}
public DataStoreException(Throwable cause) {
super(cause);
}
public class DataStoreException extends Exception
{
private static final long serialVersionUID = -5492973985910379215L;
public DataStoreException() {
}
public DataStoreException(final String message, final Throwable cause) {
super(message, cause);
}
public DataStoreException(final String message) {
super(message);
}
public DataStoreException(final Throwable cause) {
super(cause);
}
}
File diff suppressed because it is too large Load Diff
@@ -1,193 +1,178 @@
package com.fangjia.datastore;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.mina.core.buffer.IoBuffer;
import org.apache.mina.core.service.IoHandlerAdapter;
import org.slf4j.LoggerFactory;
import org.apache.mina.core.service.IoService;
import org.apache.mina.core.service.IoServiceListener;
import org.apache.mina.core.service.SimpleIoProcessorPool;
import org.apache.mina.core.session.IdleStatus;
import org.apache.mina.core.session.IoSession;
import org.apache.mina.filter.codec.ProtocolCodecException;
import org.apache.mina.filter.codec.ProtocolCodecFilter;
import org.apache.mina.filter.logging.LogLevel;
import org.apache.mina.filter.logging.LoggingFilter;
import org.apache.mina.transport.socket.SocketSessionConfig;
import org.apache.mina.transport.socket.nio.NioProcessor;
import org.apache.mina.transport.socket.nio.NioSession;
import org.apache.mina.transport.socket.nio.NioSocketAcceptor;
import java.util.Arrays;
import org.bson.BSONObject;
import org.bson.BasicBSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fangjia.datastore.config.DataStoreConfig;
import com.fangjia.datastore.handler.DeleteMessageHandler;
import com.fangjia.datastore.handler.GetMoreMessageHandler;
import com.fangjia.datastore.handler.InsertMessageHandler;
import com.fangjia.datastore.handler.KillCursorsMessageHandler;
import com.fangjia.datastore.handler.MessageHandler;
import com.fangjia.datastore.handler.QueryMessageHandler;
import com.fangjia.datastore.mongo.protocol.ReplyMessage;
import org.apache.mina.filter.codec.ProtocolCodecException;
import java.io.IOException;
import org.apache.mina.core.session.IoSession;
import org.apache.mina.transport.socket.SocketSessionConfig;
import java.net.SocketAddress;
import java.net.InetSocketAddress;
import org.apache.mina.core.session.IdleStatus;
import org.apache.mina.core.service.IoHandler;
import com.fangjia.datastore.handler.UpdateMessageHandler;
import com.fangjia.datastore.mongo.codec.BSONIoBufferAllocator;
import com.fangjia.datastore.mongo.protocol.UpdateMessage;
import com.fangjia.datastore.handler.DeleteMessageHandler;
import com.fangjia.datastore.mongo.protocol.DeleteMessage;
import com.fangjia.datastore.handler.KillCursorsMessageHandler;
import com.fangjia.datastore.mongo.protocol.KillCursorsMessage;
import com.fangjia.datastore.handler.GetMoreMessageHandler;
import com.fangjia.datastore.mongo.protocol.GetMoreMessage;
import com.fangjia.datastore.handler.QueryMessageHandler;
import com.fangjia.datastore.mongo.protocol.QueryMessage;
import com.fangjia.datastore.handler.InsertMessageHandler;
import com.fangjia.datastore.mongo.protocol.InsertMessage;
import java.util.concurrent.ExecutorService;
import org.apache.mina.filter.logging.LogLevel;
import org.apache.mina.filter.logging.LoggingFilter;
import org.apache.mina.core.filterchain.IoFilter;
import org.apache.mina.filter.codec.ProtocolDecoder;
import org.apache.mina.filter.codec.ProtocolEncoder;
import org.apache.mina.filter.codec.ProtocolCodecFilter;
import com.fangjia.datastore.mongo.codec.MongoProtocolDecoder;
import com.fangjia.datastore.mongo.codec.MongoProtocolEncoder;
import com.fangjia.datastore.mongo.protocol.DeleteMessage;
import com.fangjia.datastore.mongo.protocol.GetMoreMessage;
import com.fangjia.datastore.mongo.protocol.InsertMessage;
import com.fangjia.datastore.mongo.protocol.KillCursorsMessage;
import org.apache.mina.core.service.IoProcessor;
import java.util.concurrent.Executor;
import org.apache.mina.core.service.SimpleIoProcessorPool;
import org.apache.mina.transport.socket.nio.NioProcessor;
import java.util.concurrent.Executors;
import org.apache.mina.core.buffer.IoBufferAllocator;
import org.apache.mina.core.buffer.IoBuffer;
import com.fangjia.datastore.mongo.codec.BSONIoBufferAllocator;
import com.fangjia.datastore.config.DataStoreConfig;
import java.util.HashMap;
import com.fangjia.datastore.handler.MessageHandler;
import java.util.Map;
import com.fangjia.datastore.mongo.protocol.MongoMessage;
import com.fangjia.datastore.mongo.protocol.QueryMessage;
import com.fangjia.datastore.mongo.protocol.ReplyMessage;
import com.fangjia.datastore.mongo.protocol.UpdateMessage;
public class DataStoreServer extends IoHandlerAdapter implements IoServiceListener {
private final NioSocketAcceptor acceptor;
private final DataStoreContext context;
private static final Logger LOGGER = LoggerFactory.getLogger(DataStoreServer.class);
private static ThreadLocal<MongoMessage> CURRENT_MESSAGE = new ThreadLocal<MongoMessage>();
private final Map<Class<? extends MongoMessage>, MessageHandler<? extends MongoMessage>> handlerMap = new HashMap<Class<? extends MongoMessage>, MessageHandler<? extends MongoMessage>>();
public DataStoreServer(String[] args) throws Exception {
if (LOGGER.isInfoEnabled()) {
LOGGER.info("starting datastore...");
}
DataStoreConfig config = new DataStoreConfig(System.getProperty("datastore.home", "d:\\goojiaconfig\\datastore"));
config.setReadOnly(Boolean.getBoolean("datastore.readOnly"));
if (LOGGER.isInfoEnabled()) {
LOGGER.info(config.toString());
}
context = new DataStoreContextImpl(config);
IoBuffer.setAllocator(new BSONIoBufferAllocator());
final ExecutorService executorService = Executors.newCachedThreadPool();
acceptor = new NioSocketAcceptor(executorService, new SimpleIoProcessorPool<NioSession>(NioProcessor.class, config
.getServerConfig().getMaxConnections()));
acceptor.getFilterChain().addLast("codec",
new ProtocolCodecFilter(new MongoProtocolEncoder(), new MongoProtocolDecoder()));
acceptor.getFilterChain().addLast("performance",
new PerformanceLogFilter(config.getServerConfig().getPerformanceLogFilterMaxTime()));
if (!config.getServerConfig().isDisableLogging()) {
LoggingFilter loggingFilter = new LoggingFilter(DataStoreServer.class);
loggingFilter.setExceptionCaughtLogLevel(LogLevel.NONE);
acceptor.getFilterChain().addLast("logger", loggingFilter);
}
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
LOGGER.info("shutting down...");
acceptor.unbind();
executorService.shutdown();
LOGGER.info("shut down");
}
});
// register message handlers
register(InsertMessage.class, new InsertMessageHandler(context));
register(QueryMessage.class, new QueryMessageHandler(context));
register(GetMoreMessage.class, new GetMoreMessageHandler(context));
register(KillCursorsMessage.class, new KillCursorsMessageHandler(context));
register(DeleteMessage.class, new DeleteMessageHandler(context));
register(UpdateMessage.class, new UpdateMessageHandler(context));
acceptor.setHandler(this);
acceptor.addListener(this);
acceptor.setReuseAddress(true);
SocketSessionConfig sessionConfig = acceptor.getSessionConfig();
sessionConfig.setTcpNoDelay(true);
sessionConfig.setKeepAlive(true);
sessionConfig.setReuseAddress(true);
sessionConfig.setReadBufferSize(config.getServerConfig().getMinaReadBufferSize());
sessionConfig.setIdleTime(IdleStatus.BOTH_IDLE, 300);
acceptor.bind(new InetSocketAddress(config.getServerConfig().getPort()));
}
private void register(Class<? extends MongoMessage> messageClass, MessageHandler<? extends MongoMessage> handler) {
this.handlerMap.put(messageClass, handler);
}
/**
* @param args
* @throws Exception
* @throws
*/
public static void main(String[] args) throws Exception {
new DataStoreServer(args);
}
@Override
public void exceptionCaught(IoSession session, Throwable cause) throws Exception {
if (cause instanceof IOException && null != cause.getMessage() && cause.getMessage().contains("Connection reset by peer")) {
if (LOGGER.isInfoEnabled()) {
LOGGER.info("connection closed on session: {}", session);
}
} else if (cause instanceof ProtocolCodecException) {
// ProtocolDecoderException or ProtocolEncoderException
MongoMessage message = CURRENT_MESSAGE.get();
if (null != message) {
if (message instanceof QueryMessage) {
QueryMessage queryMessage = (QueryMessage) message;
ReplyMessage reply = new ReplyMessage();
reply.setNumberReturned(1);
reply.setResponseFlag(8);
reply.getMessageHeader().setResponseTo(queryMessage.getMessageHeader().getRequestID());
BSONObject msg = new BasicBSONObject("$err", cause.getMessage());
reply.setDocuments(Arrays.asList(msg));
session.write(reply);
}
}
} else if (LOGGER.isErrorEnabled()) {
LOGGER.error("exception caught on session:{" + session + "}", cause);
}
}
@Override
public void messageReceived(IoSession session, Object message) throws Exception {
if (message instanceof MongoMessage) {
MongoMessage mongoMessage = (MongoMessage) message;
CURRENT_MESSAGE.set(mongoMessage);
MessageHandler<? extends MongoMessage> handler = this.handlerMap.get(mongoMessage.getClass());
if (null != handler) {
handler.messageReceived(session, mongoMessage);
} else {
throw new RuntimeException("unknown message received:" + message);
}
} else {
throw new RuntimeException("unknown message received:" + message);
}
}
public void serviceActivated(IoService service) throws Exception {
if (LOGGER.isInfoEnabled()) {
LOGGER.info("datastore started");
LOGGER.info("listening at " + this.context.getDataStoreConfig().getServerConfig().getPort());
}
}
public void serviceIdle(IoService service, IdleStatus idleStatus) throws Exception {
// empty
}
public void serviceDeactivated(IoService service) throws Exception {
// empty
}
public void sessionDestroyed(IoSession session) throws Exception {
// empty
}
import org.slf4j.Logger;
import org.apache.mina.transport.socket.nio.NioSocketAcceptor;
import org.apache.mina.core.service.IoServiceListener;
import org.apache.mina.core.service.IoHandlerAdapter;
public class DataStoreServer extends IoHandlerAdapter implements IoServiceListener
{
private final NioSocketAcceptor acceptor;
private final DataStoreContext context;
private static final Logger LOGGER;
private static ThreadLocal<MongoMessage> CURRENT_MESSAGE;
private final Map<Class<? extends MongoMessage>, MessageHandler<? extends MongoMessage>> handlerMap;
public DataStoreServer(final String[] args) throws Exception {
this.handlerMap = new HashMap<Class<? extends MongoMessage>, MessageHandler<? extends MongoMessage>>();
if (DataStoreServer.LOGGER.isInfoEnabled()) {
DataStoreServer.LOGGER.info("starting datastore...");
}
final DataStoreConfig config = new DataStoreConfig(System.getProperty("datastore.home", "D:\\goojiaconfig\\seaweedsolr6"));
config.setReadOnly(Boolean.getBoolean("datastore.readOnly"));
if (DataStoreServer.LOGGER.isInfoEnabled()) {
DataStoreServer.LOGGER.info(config.toString());
}
this.context = new DataStoreContextImpl(config);
IoBuffer.setAllocator(new BSONIoBufferAllocator());
final ExecutorService executorService = Executors.newCachedThreadPool();
this.acceptor = new NioSocketAcceptor(executorService, new SimpleIoProcessorPool(NioProcessor.class, config.getServerConfig().getMaxConnections()));
this.acceptor.getFilterChain().addLast("codec", new ProtocolCodecFilter(new MongoProtocolEncoder(), new MongoProtocolDecoder()));
this.acceptor.getFilterChain().addLast("performance", new PerformanceLogFilter(config.getServerConfig().getPerformanceLogFilterMaxTime()));
if (!config.getServerConfig().isDisableLogging()) {
final LoggingFilter loggingFilter = new LoggingFilter(DataStoreServer.class);
loggingFilter.setExceptionCaughtLogLevel(LogLevel.NONE);
this.acceptor.getFilterChain().addLast("logger", loggingFilter);
}
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
DataStoreServer.LOGGER.info("shutting down...");
DataStoreServer.this.acceptor.unbind();
executorService.shutdown();
DataStoreServer.LOGGER.info("shut down");
}
});
this.register(InsertMessage.class, new InsertMessageHandler(this.context));
this.register(QueryMessage.class, new QueryMessageHandler(this.context));
this.register(GetMoreMessage.class, new GetMoreMessageHandler(this.context));
this.register(KillCursorsMessage.class, new KillCursorsMessageHandler(this.context));
this.register(DeleteMessage.class, new DeleteMessageHandler(this.context));
this.register(UpdateMessage.class, new UpdateMessageHandler(this.context));
this.acceptor.setHandler(this);
this.acceptor.addListener(this);
this.acceptor.setReuseAddress(true);
final SocketSessionConfig sessionConfig = this.acceptor.getSessionConfig();
sessionConfig.setTcpNoDelay(true);
sessionConfig.setKeepAlive(true);
sessionConfig.setReuseAddress(true);
sessionConfig.setReadBufferSize(config.getServerConfig().getMinaReadBufferSize());
sessionConfig.setIdleTime(IdleStatus.BOTH_IDLE, 300);
this.acceptor.bind(new InetSocketAddress(config.getServerConfig().getPort()));
}
private void register(final Class<? extends MongoMessage> messageClass, final MessageHandler<? extends MongoMessage> handler) {
this.handlerMap.put(messageClass, handler);
}
public static void main(final String[] args) throws Exception {
new DataStoreServer(args);
}
public void exceptionCaught(final IoSession session, final Throwable cause) throws Exception {
if (cause instanceof IOException && null != cause.getMessage() && cause.getMessage().contains("Connection reset by peer")) {
if (DataStoreServer.LOGGER.isInfoEnabled()) {
DataStoreServer.LOGGER.info("connection closed on session: {}", (Object)session);
}
}
else if (cause instanceof ProtocolCodecException) {
final MongoMessage message = DataStoreServer.CURRENT_MESSAGE.get();
if (null != message && message instanceof QueryMessage) {
final QueryMessage queryMessage = (QueryMessage)message;
final ReplyMessage reply = new ReplyMessage();
reply.setNumberReturned(1);
reply.setResponseFlag(8);
reply.getMessageHeader().setResponseTo(queryMessage.getMessageHeader().getRequestID());
final BSONObject msg = (BSONObject)new BasicBSONObject("$err", cause.getMessage());
reply.setDocuments(Arrays.<BSONObject>asList(msg));
session.write((Object)reply);
}
}
else if (DataStoreServer.LOGGER.isErrorEnabled()) {
DataStoreServer.LOGGER.error("exception caught on session:{" + session + "}", cause);
}
}
public void messageReceived(final IoSession session, final Object message) throws Exception {
if (!(message instanceof MongoMessage)) {
throw new RuntimeException("unknown message received:" + message);
}
final MongoMessage mongoMessage = (MongoMessage)message;
DataStoreServer.CURRENT_MESSAGE.set(mongoMessage);
final MessageHandler<? extends MongoMessage> handler = this.handlerMap.get(mongoMessage.getClass());
if (null != handler) {
handler.messageReceived(session, mongoMessage);
return;
}
throw new RuntimeException("unknown message received:" + message);
}
public void serviceActivated(final IoService service) throws Exception {
if (DataStoreServer.LOGGER.isInfoEnabled()) {
DataStoreServer.LOGGER.info("datastore started");
DataStoreServer.LOGGER.info("listening at " + this.context.getDataStoreConfig().getServerConfig().getPort());
}
}
public void serviceIdle(final IoService service, final IdleStatus idleStatus) throws Exception {
}
public void serviceDeactivated(final IoService service) throws Exception {
}
public void sessionDestroyed(final IoSession session) throws Exception {
}
static {
LOGGER = LoggerFactory.getLogger((Class)DataStoreServer.class);
DataStoreServer.CURRENT_MESSAGE = new ThreadLocal<MongoMessage>();
}
}
@@ -1,64 +1,22 @@
package com.fangjia.datastore;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.List;
import java.util.Set;
import org.bson.BSONObject;
import java.util.Set;
import org.bson.types.ObjectId;
import com.mongodb.DBObject;
import java.util.List;
public interface FileStore {
public final static String DEFAULT_CHARSET = "UTF-8";
/**
*
* @param fileList
* 仅包含大字段的文档列表
* @param dbObjList
* 已经保存到数据库的主对象列表
* @throws IllegalArgumentException
* 大字段数据不支持默认编码时抛出
*/
void save(List<DBObject> fileList, List<DBObject> dbObjList) throws DataStoreException;
/**
* 删除指定的大字段内容
*
* @param objIdList
* 删除的主对象的id列表
* @param removeFields
* 需要删除的大字段名集合
*/
public void remove(List<ObjectId> objIdList, Set<String> removeFields);
/**
* 删除所有大字段内容
*
* @param objIdList
*/
public void remove(List<ObjectId> objIdList);
/**
* @param objIdList
* 待更新的主对象id列表
* @param data
* 更新文档
* @throws UnsupportedEncodingException
* 大字段数据不支持默认编码时抛出
*/
public void update(List<ObjectId> objIdList, BSONObject data) throws DataStoreException;
/**
* @param docList
* 查询的结果文档
* @param fields
* 查询字段对象
* @return
* @throws IOException
* 读取大字段数据异常时、大字段数据不支持默认编码时抛出
*/
public List<DBObject> find(List<DBObject> docList, BSONObject fields) throws DataStoreException;
public interface FileStore
{
public static final String DEFAULT_CHARSET = "UTF-8";
void save(final List<DBObject> p0, final List<DBObject> p1) throws DataStoreException;
void remove(final List<ObjectId> p0, final Set<String> p1);
void remove(final List<ObjectId> p0);
void update(final List<ObjectId> p0, final BSONObject p1) throws DataStoreException;
List<DBObject> find(final List<DBObject> p0, final BSONObject p1) throws DataStoreException;
}
@@ -1,249 +1,229 @@
package com.fangjia.datastore;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.zip.GZIPOutputStream;
import org.apache.commons.io.IOUtils;
import com.mongodb.gridfs.GridFSDBFile;
import java.util.HashSet;
import com.mongodb.BasicDBObject;
import java.util.ArrayList;
import java.util.Set;
import java.io.OutputStream;
import java.util.zip.GZIPOutputStream;
import java.io.ByteArrayOutputStream;
import com.mongodb.gridfs.GridFSFile;
import java.util.Date;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
import com.fangjia.datastore.config.FieldTypeEnum;
import org.bson.BSONObject;
import org.bson.types.ObjectId;
import com.fangjia.datastore.config.Field;
import com.fangjia.datastore.config.FieldTypeEnum;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
import java.util.List;
import java.util.Iterator;
import java.util.HashMap;
import java.util.Collection;
import com.fangjia.datastore.config.Field;
import java.util.Map;
import com.mongodb.gridfs.GridFS;
import com.mongodb.gridfs.GridFSDBFile;
import com.mongodb.gridfs.GridFSFile;
public class FileStoreImpl implements FileStore {
private GridFS gridFS;
private Map<String, Field> bigFieldSet;
public FileStoreImpl(GridFS gridFS, Collection<Field> fields) {
this.gridFS = gridFS;
// 把大字段的字段名放入set
bigFieldSet = new HashMap<String, Field>();
for (Field field : fields) {
if (field.isBigField()) {
bigFieldSet.put(field.getName(), field);
}
}
}
public void save(List<DBObject> fileList, List<DBObject> dbObjList) {
if (null == fileList || null == dbObjList || fileList.size() != dbObjList.size())
throw new IllegalArgumentException("empty list or unequal length list is not allowed!");
Iterator<DBObject> dbObjIt = dbObjList.iterator();
for (Iterator<DBObject> fileIt = fileList.iterator(); fileIt.hasNext();) {
// 取得已保存主对象的id
String id = ((ObjectId) dbObjIt.next().get("_id")).toString();
BSONObject file = fileIt.next();
for (String fieldName : file.keySet()) {
byte[] bytes = null;
Field field = bigFieldSet.get(fieldName);
if (null == field) {
continue;
}
FieldTypeEnum fieldType = field.getType();
if (FieldTypeEnum.TEXT == fieldType) {
String value = (String) file.get(fieldName);
if (!StringUtils.isBlank(value)) {
try {
bytes = gzip(value);
} catch (IOException e) {
throw new IllegalArgumentException("unsupported encoding value!");
}
}
}
if (null != bytes && bytes.length > 0) {
GridFSFile mongofile = gridFS.createFile(bytes);
mongofile.put("uploadDate", new Date());
mongofile.put("filename", id);
mongofile.put("contentType", fieldName);
mongofile.save();
}
}
}
}
private byte[] gzip(String string) throws IOException {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
GZIPOutputStream gzipOutputStream = new GZIPOutputStream(outputStream);
gzipOutputStream.write(string.getBytes(DEFAULT_CHARSET));
return outputStream.toByteArray();
}
public void remove(List<ObjectId> objIdList) {
remove(objIdList, null);
}
public void remove(List<ObjectId> objIdList, Set<String> removeFields) {
// 删除表的所有大字段
if (null == removeFields)
removeFields = bigFieldSet.keySet();
// 若到删除的字段中没有大字段,直接返回
if (removeFields.size() == 0)
return;
List<String> idList = new ArrayList<String>(objIdList.size());
for (ObjectId objId : objIdList) {
idList.add(objId.toString());
}
// 删除大字段关联的GridFS
DBObject fileSeletor = new BasicDBObject();
fileSeletor.put("filename", new BasicDBObject("$in", idList));
fileSeletor.put("contentType", new BasicDBObject("$in", removeFields));
gridFS.remove(fileSeletor);
}
public void update(List<ObjectId> objIdList, BSONObject data) throws DataStoreException {
// 待删除的字段
BSONObject unset = (BSONObject) data.get("$unset");
if (null != unset) {
// 待删除的大字段set
Set<String> removeSet = new HashSet<String>();
for (String field : unset.keySet()) {
if (bigFieldSet.containsKey(field))
removeSet.add(field);
}
if (removeSet.size() > 0)
remove(objIdList, removeSet);
}
// 更新数据
BSONObject set = (BSONObject) data.get("$set");
if (null == set)
return;
Map<String, Object> fileMap = new HashMap<String, Object>();
for (String field : set.keySet()) {
if (bigFieldSet.containsKey(field)) {
fileMap.put(field, set.get(field));
}
}
// 更新数据中没有大字段
if (fileMap.size() == 0)
return;
// 删除大字段关联的GridFS
remove(objIdList, fileMap.keySet());
// 将新的内容存入GridFS
for (ObjectId objId : objIdList) {
String id = objId.toString();
for (String fieldName : fileMap.keySet()) {
byte[] bytes = null;
Field field = bigFieldSet.get(fieldName);
if (null == field) {
continue;
}
FieldTypeEnum fieldType = field.getType();
if (FieldTypeEnum.TEXT == fieldType) {
String value = (String) fileMap.get(fieldName);
if (!StringUtils.isBlank(value)) {
try {
bytes = gzip(value);
} catch (IOException e) {
throw new IllegalArgumentException("unsupported encoding value!");
}
}
}
if (null != bytes && bytes.length > 0) {
GridFSFile mongofile = gridFS.createFile(bytes);
mongofile.put("uploadDate", new Date());
mongofile.put("filename", id);
mongofile.put("contentType", fieldName);
mongofile.save();
}
}
}
}
public List<DBObject> find(List<DBObject> docList, BSONObject fields) throws DataStoreException {
// 判断查询的字段中有没有大字段
Set<String> queryBigField = new HashSet<String>();
if (null == fields) {
queryBigField = bigFieldSet.keySet();
} else {
for (String field : fields.keySet()) {
if (bigFieldSet.containsKey(field)) {
queryBigField.add(field);
}
}
}
// 没有大字段,则返回原结果文档
if (queryBigField.size() == 0)
return docList;
Map<String, DBObject> docMap = new HashMap<String, DBObject>(docList.size());
List<String> idList = new ArrayList<String>(docList.size());
for (DBObject dbObj : docList) {
ObjectId objId = (ObjectId) dbObj.get("_id");
if (null != objId) {
String id = objId.toString();
idList.add(id);
docMap.put(id, dbObj);
}
}
DBObject fileSeletor = new BasicDBObject();
fileSeletor.put("filename", new BasicDBObject("$in", idList));
fileSeletor.put("contentType", new BasicDBObject("$in", queryBigField));
List<GridFSDBFile> fileList = gridFS.find(fileSeletor);
for (GridFSDBFile file : fileList) {
// 对应主记录id
String id = file.getFilename();
// 对应主记录大字段名
String fieldName = file.getContentType();
DBObject obj = docMap.get(id);
if (obj.keySet().contains(fieldName)) {
try {
ByteArrayOutputStream data = new ByteArrayOutputStream();
GZIPOutputStream out = new GZIPOutputStream(data);
IOUtils.copy(file.getInputStream(), out);
Field field = bigFieldSet.get(fieldName);
if (null == field) {
continue;
}
FieldTypeEnum fieldType = field.getType();
if (FieldTypeEnum.TEXT == fieldType) {
obj.put(fieldName, new String(data.toByteArray(), DEFAULT_CHARSET));
}
} catch (IOException e) {
throw new DataStoreException("error extract text field[" + fieldName + "]:", e);
}
}
}
return docList;
}
public class FileStoreImpl implements FileStore
{
private GridFS gridFS;
private Map<String, Field> bigFieldSet;
public FileStoreImpl(final GridFS gridFS, final Collection<Field> fields) {
this.gridFS = gridFS;
this.bigFieldSet = new HashMap<String, Field>();
for (final Field field : fields) {
if (field.isBigField()) {
this.bigFieldSet.put(field.getName(), field);
}
}
}
@Override
public void save(final List<DBObject> fileList, final List<DBObject> dbObjList) {
if (null == fileList || null == dbObjList || fileList.size() != dbObjList.size()) {
throw new IllegalArgumentException("empty list or unequal length list is not allowed!");
}
final Iterator<DBObject> dbObjIt = dbObjList.iterator();
final Iterator<DBObject> fileIt = fileList.iterator();
while (fileIt.hasNext()) {
final String id = ((ObjectId)dbObjIt.next().get("_id")).toString();
final BSONObject file = (BSONObject)fileIt.next();
for (final String fieldName : file.keySet()) {
byte[] bytes = null;
final Field field = this.bigFieldSet.get(fieldName);
if (null == field) {
continue;
}
final FieldTypeEnum fieldType = field.getType();
if (FieldTypeEnum.TEXT == fieldType) {
final String value = (String)file.get(fieldName);
if (!StringUtils.isBlank(value)) {
try {
bytes = this.gzip(value);
}
catch (IOException e) {
throw new IllegalArgumentException("unsupported encoding value!");
}
}
}
if (null == bytes || bytes.length <= 0) {
continue;
}
final GridFSFile mongofile = (GridFSFile)this.gridFS.createFile(bytes);
mongofile.put("uploadDate", (Object)new Date());
mongofile.put("filename", (Object)id);
mongofile.put("contentType", (Object)fieldName);
mongofile.save();
}
}
}
private byte[] gzip(final String string) throws IOException {
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
final GZIPOutputStream gzipOutputStream = new GZIPOutputStream(outputStream);
gzipOutputStream.write(string.getBytes("UTF-8"));
return outputStream.toByteArray();
}
@Override
public void remove(final List<ObjectId> objIdList) {
this.remove(objIdList, null);
}
@Override
public void remove(final List<ObjectId> objIdList, Set<String> removeFields) {
if (null == removeFields) {
removeFields = this.bigFieldSet.keySet();
}
if (removeFields.size() == 0) {
return;
}
final List<String> idList = new ArrayList<String>(objIdList.size());
for (final ObjectId objId : objIdList) {
idList.add(objId.toString());
}
final DBObject fileSeletor = (DBObject)new BasicDBObject();
fileSeletor.put("filename", (Object)new BasicDBObject("$in", (Object)idList));
fileSeletor.put("contentType", (Object)new BasicDBObject("$in", (Object)removeFields));
this.gridFS.remove(fileSeletor);
}
@Override
public void update(final List<ObjectId> objIdList, final BSONObject data) throws DataStoreException {
final BSONObject unset = (BSONObject)data.get("$unset");
if (null != unset) {
final Set<String> removeSet = new HashSet<String>();
for (final String field : unset.keySet()) {
if (this.bigFieldSet.containsKey(field)) {
removeSet.add(field);
}
}
if (removeSet.size() > 0) {
this.remove(objIdList, removeSet);
}
}
final BSONObject set = (BSONObject)data.get("$set");
if (null == set) {
return;
}
final Map<String, Object> fileMap = new HashMap<String, Object>();
for (final String field2 : set.keySet()) {
if (this.bigFieldSet.containsKey(field2)) {
fileMap.put(field2, set.get(field2));
}
}
if (fileMap.size() == 0) {
return;
}
this.remove(objIdList, fileMap.keySet());
for (final ObjectId objId : objIdList) {
final String id = objId.toString();
for (final String fieldName : fileMap.keySet()) {
byte[] bytes = null;
final Field field3 = this.bigFieldSet.get(fieldName);
if (null == field3) {
continue;
}
final FieldTypeEnum fieldType = field3.getType();
if (FieldTypeEnum.TEXT == fieldType) {
final String value = (String)fileMap.get(fieldName);
if (!StringUtils.isBlank(value)) {
try {
bytes = this.gzip(value);
}
catch (IOException e) {
throw new IllegalArgumentException("unsupported encoding value!");
}
}
}
if (null == bytes || bytes.length <= 0) {
continue;
}
final GridFSFile mongofile = (GridFSFile)this.gridFS.createFile(bytes);
mongofile.put("uploadDate", (Object)new Date());
mongofile.put("filename", (Object)id);
mongofile.put("contentType", (Object)fieldName);
mongofile.save();
}
}
}
@Override
public List<DBObject> find(final List<DBObject> docList, final BSONObject fields) throws DataStoreException {
Set<String> queryBigField = new HashSet<String>();
if (null == fields) {
queryBigField = this.bigFieldSet.keySet();
}
else {
for (final String field : fields.keySet()) {
if (this.bigFieldSet.containsKey(field)) {
queryBigField.add(field);
}
}
}
if (queryBigField.size() == 0) {
return docList;
}
final Map<String, DBObject> docMap = new HashMap<String, DBObject>(docList.size());
final List<String> idList = new ArrayList<String>(docList.size());
for (final DBObject dbObj : docList) {
final ObjectId objId = (ObjectId)dbObj.get("_id");
if (null != objId) {
final String id = objId.toString();
idList.add(id);
docMap.put(id, dbObj);
}
}
final DBObject fileSeletor = (DBObject)new BasicDBObject();
fileSeletor.put("filename", (Object)new BasicDBObject("$in", (Object)idList));
fileSeletor.put("contentType", (Object)new BasicDBObject("$in", (Object)queryBigField));
final List<GridFSDBFile> fileList = (List<GridFSDBFile>)this.gridFS.find(fileSeletor);
for (final GridFSDBFile file : fileList) {
final String id2 = file.getFilename();
final String fieldName = file.getContentType();
final DBObject obj = docMap.get(id2);
if (obj.keySet().contains(fieldName)) {
try {
final ByteArrayOutputStream data = new ByteArrayOutputStream();
final GZIPOutputStream out = new GZIPOutputStream(data);
IOUtils.copy(file.getInputStream(), (OutputStream)out);
final Field field2 = this.bigFieldSet.get(fieldName);
if (null == field2) {
continue;
}
final FieldTypeEnum fieldType = field2.getType();
if (FieldTypeEnum.TEXT != fieldType) {
continue;
}
obj.put(fieldName, (Object)new String(data.toByteArray(), "UTF-8"));
}
catch (IOException e) {
throw new DataStoreException("error extract text field[" + fieldName + "]:", e);
}
}
}
return docList;
}
}
@@ -1,46 +1,46 @@
package com.fangjia.datastore;
import java.util.List;
import com.fangjia.datastore.config.MongoConfig;
import com.mongodb.DB;
import com.mongodb.MongoException;
import com.mongodb.WriteConcern;
import com.mongodb.MongoException;
import java.util.List;
import com.mongodb.DB;
import com.fangjia.datastore.config.MongoConfig;
public class Mongo {
private final MongoConfig mongoConfig;
public Mongo(MongoConfig mongoConfig) {
this.mongoConfig = mongoConfig;
}
private com.mongodb.Mongo getMongo() {
try {
return mongoConfig.getMongo();
} catch (ConfigurationException e) {
throw new RuntimeException(e);
}
}
public DB getDB(String dbname) {
return getMongo().getDB(dbname);
}
public List<String> getDatabaseNames() throws MongoException {
return getMongo().getDatabaseNames();
}
public void setWriteConcern(WriteConcern concern) {
getMongo().setWriteConcern(concern);
}
public WriteConcern getWriteConcern() {
return getMongo().getWriteConcern();
}
public String toString() {
return getMongo().toString();
}
public class Mongo
{
private final MongoConfig mongoConfig;
public Mongo(final MongoConfig mongoConfig) {
this.mongoConfig = mongoConfig;
}
private com.mongodb.Mongo getMongo() {
try {
return this.mongoConfig.getMongo();
}
catch (ConfigurationException e) {
throw new RuntimeException(e);
}
}
public DB getDB(final String dbname) {
return this.getMongo().getDB(dbname);
}
public List<String> getDatabaseNames() throws MongoException {
return (List<String>)this.getMongo().getDatabaseNames();
}
public void setWriteConcern(final WriteConcern concern) {
this.getMongo().setWriteConcern(concern);
}
public WriteConcern getWriteConcern() {
return this.getMongo().getWriteConcern();
}
@Override
public String toString() {
return this.getMongo().toString();
}
}
@@ -1,114 +1,119 @@
package com.fangjia.datastore;
import org.apache.mina.core.filterchain.IoFilterAdapter;
import org.apache.mina.core.session.IdleStatus;
import org.apache.mina.core.session.IoSession;
import org.apache.mina.core.write.WriteRequest;
import org.slf4j.Logger;
import com.fangjia.datastore.mongo.protocol.MongoMessage;
import com.fangjia.datastore.utils.LoggerFactory;
import org.apache.mina.core.session.IdleStatus;
import org.apache.mina.core.write.WriteRequest;
import org.apache.mina.core.filterchain.IoFilter;
import com.fangjia.datastore.mongo.protocol.MongoMessage;
import org.apache.mina.core.session.IoSession;
import org.slf4j.Logger;
import org.apache.mina.core.filterchain.IoFilterAdapter;
public class PerformanceLogFilter extends IoFilterAdapter {
private final long maxTime;
private static final Logger LOGGER = LoggerFactory.getPerformanceLogger(PerformanceLogFilter.class);
public PerformanceLogFilter(long maxTime) {
this.maxTime = maxTime;
}
private void logPerformace(long time, String methodName, IoSession session, Object message) {
Object[] params;
if (message instanceof MongoMessage) {
MongoMessage mongoMessage = (MongoMessage) message;
params = new Object[] { time, methodName, session, mongoMessage.toPrettyString() };
} else {
params = new Object[] { time, methodName, session, message };
}
if (time < maxTime * 3) {
LOGGER.warn("time={}, methodName={}, session={}, message={}", params);
} else {
LOGGER.error("time={}, methodName={}, session={}, message={}", params);
}
}
@Override
public void messageReceived(NextFilter nextFilter, IoSession session, Object message) throws Exception {
long time = System.currentTimeMillis();
try {
nextFilter.messageReceived(session, message);
} finally {
time = System.currentTimeMillis() - time;
if (time >= maxTime) {
logPerformace(time, "messageReceived", session, message);
}
}
}
@Override
public void messageSent(NextFilter nextFilter, IoSession session, WriteRequest writeRequest) throws Exception {
long time = System.currentTimeMillis();
try {
nextFilter.messageSent(session, writeRequest);
} finally {
time = System.currentTimeMillis() - time;
if (time >= maxTime) {
logPerformace(time, "messageSent", session, writeRequest);
}
}
}
@Override
public void sessionCreated(NextFilter nextFilter, IoSession session) throws Exception {
long time = System.currentTimeMillis();
try {
nextFilter.sessionCreated(session);
} finally {
time = System.currentTimeMillis() - time;
if (time >= maxTime) {
logPerformace(time, "sessionCreated", session, null);
}
}
}
@Override
public void sessionOpened(NextFilter nextFilter, IoSession session) throws Exception {
long time = System.currentTimeMillis();
try {
nextFilter.sessionOpened(session);
} finally {
time = System.currentTimeMillis() - time;
if (time >= maxTime) {
logPerformace(time, "sessionOpened", session, null);
}
}
}
@Override
public void sessionIdle(NextFilter nextFilter, IoSession session, IdleStatus status) throws Exception {
long time = System.currentTimeMillis();
try {
nextFilter.sessionIdle(session, status);
} finally {
time = System.currentTimeMillis() - time;
if (time >= maxTime) {
logPerformace(time, "sessionIdle", session, status);
}
}
}
@Override
public void sessionClosed(NextFilter nextFilter, IoSession session) throws Exception {
long time = System.currentTimeMillis();
try {
nextFilter.sessionClosed(session);
} finally {
time = System.currentTimeMillis() - time;
if (time >= maxTime) {
logPerformace(time, "sessionClosed", session, null);
}
}
}
public class PerformanceLogFilter extends IoFilterAdapter
{
private final long maxTime;
private static final Logger LOGGER;
public PerformanceLogFilter(final long maxTime) {
this.maxTime = maxTime;
}
private void logPerformace(final long time, final String methodName, final IoSession session, final Object message) {
Object[] params;
if (message instanceof MongoMessage) {
final MongoMessage mongoMessage = (MongoMessage)message;
params = new Object[] { time, methodName, session, mongoMessage.toPrettyString() };
}
else {
params = new Object[] { time, methodName, session, message };
}
if (time < this.maxTime * 3L) {
PerformanceLogFilter.LOGGER.warn("time={}, methodName={}, session={}, message={}", params);
}
else {
PerformanceLogFilter.LOGGER.error("time={}, methodName={}, session={}, message={}", params);
}
}
public void messageReceived(final NextFilter nextFilter, final IoSession session, final Object message) throws Exception {
long time = System.currentTimeMillis();
try {
nextFilter.messageReceived(session, message);
}
finally {
time = System.currentTimeMillis() - time;
if (time >= this.maxTime) {
this.logPerformace(time, "messageReceived", session, message);
}
}
}
public void messageSent(final NextFilter nextFilter, final IoSession session, final WriteRequest writeRequest) throws Exception {
long time = System.currentTimeMillis();
try {
nextFilter.messageSent(session, writeRequest);
}
finally {
time = System.currentTimeMillis() - time;
if (time >= this.maxTime) {
this.logPerformace(time, "messageSent", session, writeRequest);
}
}
}
public void sessionCreated(final NextFilter nextFilter, final IoSession session) throws Exception {
long time = System.currentTimeMillis();
try {
nextFilter.sessionCreated(session);
}
finally {
time = System.currentTimeMillis() - time;
if (time >= this.maxTime) {
this.logPerformace(time, "sessionCreated", session, null);
}
}
}
public void sessionOpened(final NextFilter nextFilter, final IoSession session) throws Exception {
long time = System.currentTimeMillis();
try {
nextFilter.sessionOpened(session);
}
finally {
time = System.currentTimeMillis() - time;
if (time >= this.maxTime) {
this.logPerformace(time, "sessionOpened", session, null);
}
}
}
public void sessionIdle(final NextFilter nextFilter, final IoSession session, final IdleStatus status) throws Exception {
long time = System.currentTimeMillis();
try {
nextFilter.sessionIdle(session, status);
}
finally {
time = System.currentTimeMillis() - time;
if (time >= this.maxTime) {
this.logPerformace(time, "sessionIdle", session, status);
}
}
}
public void sessionClosed(final NextFilter nextFilter, final IoSession session) throws Exception {
long time = System.currentTimeMillis();
try {
nextFilter.sessionClosed(session);
}
finally {
time = System.currentTimeMillis() - time;
if (time >= this.maxTime) {
this.logPerformace(time, "sessionClosed", session, null);
}
}
}
static {
LOGGER = LoggerFactory.getPerformanceLogger(PerformanceLogFilter.class);
}
}
@@ -1,72 +1,80 @@
package com.fangjia.datastore;
import java.util.Collections;
import org.apache.solr.client.solrj.response.FacetField;
import com.mongodb.DBObject;
import org.bson.types.ObjectId;
import java.util.List;
import org.apache.solr.client.solrj.response.FacetField;
import org.bson.types.ObjectId;
import com.mongodb.DBObject;
public class QueryResponse {
private List<ObjectId> idList = Collections.emptyList();
private List<DBObject> docList = Collections.emptyList();
private List<FacetField> facetFieldList = null;
private int numFound;
private int numReturned;
private int numToSkip;
public final static QueryResponse EMPTY_RESPONSE = new QueryResponse();
public List<ObjectId> getIdList() {
return idList;
}
public void setIdList(List<ObjectId> idList) {
this.idList = idList;
}
public boolean hasMore() {
return this.numFound > numReturned + numToSkip;
}
public int getNumFound() {
return numFound;
}
public void setNumFound(int numFound) {
this.numFound = numFound;
}
public int getNumReturned() {
return numReturned;
}
public void setNumReturned(int numReturned) {
this.numReturned = numReturned;
}
public int getNumToSkip() {
return numToSkip;
}
public void setNumToSkip(int numToSkip) {
this.numToSkip = numToSkip;
}
public List<FacetField> getFacetFieldList() {
return facetFieldList;
}
public void setFacetFieldList(List<FacetField> facetFieldList) {
this.facetFieldList = facetFieldList;
}
public List<DBObject> getDocList() {
return docList;
}
public void setDocList(List<DBObject> docList) {
this.docList = docList;
}
public class QueryResponse
{
private List<ObjectId> idList;
private List<DBObject> docList;
private List<FacetField> facetFieldList;
private int numFound;
private int numReturned;
private int numToSkip;
public static final QueryResponse EMPTY_RESPONSE;
public QueryResponse() {
this.idList = Collections.<ObjectId>emptyList();
this.docList = Collections.<DBObject>emptyList();
this.facetFieldList = null;
}
public List<ObjectId> getIdList() {
return this.idList;
}
public void setIdList(final List<ObjectId> idList) {
this.idList = idList;
}
public boolean hasMore() {
return this.numFound > this.numReturned + this.numToSkip;
}
public int getNumFound() {
return this.numFound;
}
public void setNumFound(final int numFound) {
this.numFound = numFound;
}
public int getNumReturned() {
return this.numReturned;
}
public void setNumReturned(final int numReturned) {
this.numReturned = numReturned;
}
public int getNumToSkip() {
return this.numToSkip;
}
public void setNumToSkip(final int numToSkip) {
this.numToSkip = numToSkip;
}
public List<FacetField> getFacetFieldList() {
return this.facetFieldList;
}
public void setFacetFieldList(final List<FacetField> facetFieldList) {
this.facetFieldList = facetFieldList;
}
public List<DBObject> getDocList() {
return this.docList;
}
public void setDocList(final List<DBObject> docList) {
this.docList = docList;
}
static {
EMPTY_RESPONSE = new QueryResponse();
}
}
@@ -1,22 +1,21 @@
package com.fangjia.datastore;
public class TriggerExecuteException extends Exception {
private static final long serialVersionUID = -1165411196743370433L;
public TriggerExecuteException() {
}
public TriggerExecuteException(String message) {
super(message);
}
public TriggerExecuteException(Throwable cause) {
super(cause);
}
public TriggerExecuteException(String message, Throwable cause) {
super(message, cause);
}
public class TriggerExecuteException extends Exception
{
private static final long serialVersionUID = -1165411196743370433L;
public TriggerExecuteException() {
}
public TriggerExecuteException(final String message) {
super(message);
}
public TriggerExecuteException(final Throwable cause) {
super(cause);
}
public TriggerExecuteException(final String message, final Throwable cause) {
super(message, cause);
}
}
@@ -1,165 +1,161 @@
package com.fangjia.datastore;
import java.io.IOException;
import org.slf4j.LoggerFactory;
import org.apache.http.HttpRequest;
import java.io.InputStream;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.List;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.params.AllClientPNames;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import com.mongodb.DBObject;
import org.bson.BasicBSONDecoder;
import org.apache.http.protocol.HttpContext;
import org.apache.http.HttpHost;
import org.apache.http.HttpEntity;
import org.apache.http.entity.ByteArrayEntity;
import org.sothis.core.util.GZIPUtils;
import org.bson.BasicBSONEncoder;
import com.mongodb.BasicDBObject;
import org.apache.http.client.methods.HttpPost;
import java.net.URL;
import java.net.URISyntaxException;
import java.io.IOException;
import org.bson.BSONObject;
import java.util.List;
import com.fangjia.datastore.config.Trigger;
import org.apache.http.params.HttpParams;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.params.HttpParams;
import org.apache.http.protocol.HttpContext;
import org.bson.BSONEncoder;
import org.bson.BSONObject;
import org.bson.BasicBSONDecoder;
import org.apache.http.conn.scheme.SchemeSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.client.HttpClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sothis.core.util.GZIPUtils;
import com.fangjia.datastore.config.Trigger;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
public class TriggerExecutor {
private final static int MAX_TOTAL = 200;
private final static int MAX_FOR_ROUTE = 200;
private final static String DEFAULT_CHARSET = "UTF-8";
private final static int DEFAULT_PORT = 80;
private final static int MAX_RETRY_COUNT = 10;
private final static Logger LOGGER = LoggerFactory.getLogger(TriggerExecutor.class);
private final static TriggerExecutor TRIGGER_EXECUTOR = new TriggerExecutor();
private final HttpClient httpClient;
private TriggerExecutor() {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("init trigger executor");
}
SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(new Scheme("http", DEFAULT_PORT, new PlainSocketFactory()));
HttpParams httpParams = new BasicHttpParams();
httpParams.setParameter(CoreProtocolPNames.HTTP_CONTENT_CHARSET, DEFAULT_CHARSET);
httpParams.setParameter(AllClientPNames.SO_TIMEOUT, 300000);
httpParams.setParameter(AllClientPNames.CONNECTION_TIMEOUT, 60000);
ThreadSafeClientConnManager connManager = new ThreadSafeClientConnManager(schemeRegistry);
connManager.setMaxTotal(MAX_TOTAL);
connManager.setDefaultMaxPerRoute(MAX_FOR_ROUTE);
httpClient = new DefaultHttpClient(connManager, httpParams);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("trigger executor initialized[{}]", toString());
}
}
public static TriggerExecutor getInstance() {
return TRIGGER_EXECUTOR;
}
public List<BSONObject> execute(Trigger trigger, List<BSONObject> docList) throws TriggerExecuteException {
try {
List<BSONObject> ret = doExecute(trigger, docList);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("execute trigger: " + trigger + ", doc size:" + docList.size() + ", return size:" + ret.size());
}
return ret;
} catch (IOException e) {
throw new TriggerExecuteException("error execute trigger: " + trigger, e);
} catch (URISyntaxException e) {
throw new TriggerExecuteException("error execute trigger: " + trigger, e);
}
}
private List<BSONObject> doExecute(Trigger trigger, List<BSONObject> docList) throws IOException, URISyntaxException,
TriggerExecuteException {
URL url = new URL(trigger.getInterface());
HttpPost httpPost = new HttpPost(url.toURI());
DBObject object = new BasicDBObject();
object.put("doc_list_size", docList.size());
object.put("doc_list", docList);
httpPost.setEntity(new ByteArrayEntity(GZIPUtils.makeAsByteArray(new BSONEncoder().encode(object))));
HttpHost host = new HttpHost(url.getHost(), url.getPort() == -1 ? DEFAULT_PORT : url.getPort());
HttpResponse httpResponse = executeWithRetry(host, httpPost, null, trigger);
InputStream inputStream = httpResponse.getEntity().getContent();
try {
if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
BSONObject returnObject = new BasicBSONDecoder().readObject(GZIPUtils.wrap(inputStream));
if (null != returnObject) {
List<BSONObject> ret = (List<BSONObject>) returnObject.get("doc_list");
if (ret == null) {
throw new TriggerExecuteException("trigger" + trigger + " return an null doc_list[" + ret + "]");
}
return ret;
} else {
throw new TriggerExecuteException("trigger" + trigger + " returns nothing at all !");
}
} else {
throw new TriggerExecuteException("error execute trigger:" + trigger + ", get http code:"
+ httpResponse.getStatusLine().getStatusCode());
}
} finally {
if (null != inputStream) {
inputStream.close();
}
}
}
private HttpResponse executeWithRetry(HttpHost host, HttpPost httpPost, HttpContext httpContext, Trigger trigger)
throws IOException, TriggerExecuteException {
int retryCounter = 0;
HttpResponse httpResponse = null;
while (true) {
try {
httpResponse = httpClient.execute(host, httpPost, (HttpContext) null);
if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
return httpResponse;
} else {
retryCounter++;
sleepAndRetry(new IllegalStateException("invalid http status code: "
+ httpResponse.getStatusLine().getStatusCode()), retryCounter, trigger);
}
} catch (Exception e) {
retryCounter++;
sleepAndRetry(e, retryCounter, trigger);
} finally {
if (null != httpResponse && httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
httpResponse.getEntity().getContent().close();
}
}
}
}
private void sleepAndRetry(Exception e, int retryCounter, Trigger trigger) throws TriggerExecuteException {
if (retryCounter > MAX_RETRY_COUNT) {
throw new TriggerExecuteException(
"error execute trigger[" + trigger + "] after " + MAX_RETRY_COUNT + " times retry!", e);
}
if (LOGGER.isErrorEnabled()) {
LOGGER.error("error execute trigger[" + trigger + "]:(" + e.getMessage() + "), retry for the " + retryCounter
+ "th time after " + (retryCounter * 30) + " seconds...");
}
try {
Thread.sleep(retryCounter * 30000);
} catch (InterruptedException e1) {
if (LOGGER.isWarnEnabled()) {
LOGGER.warn("", e);
}
}
}
public class TriggerExecutor
{
private static final int MAX_TOTAL = 200;
private static final int MAX_FOR_ROUTE = 200;
private static final String DEFAULT_CHARSET = "UTF-8";
private static final int DEFAULT_PORT = 80;
private static final int MAX_RETRY_COUNT = 10;
private static final Logger LOGGER;
private static final TriggerExecutor TRIGGER_EXECUTOR;
private final HttpClient httpClient;
private TriggerExecutor() {
if (TriggerExecutor.LOGGER.isDebugEnabled()) {
TriggerExecutor.LOGGER.debug("init trigger executor");
}
final SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(new Scheme("http", 80, (SchemeSocketFactory)new PlainSocketFactory()));
final HttpParams httpParams = (HttpParams)new BasicHttpParams();
httpParams.setParameter("http.protocol.content-charset", (Object)"UTF-8");
httpParams.setParameter("http.socket.timeout", (Object)300000);
httpParams.setParameter("http.connection.timeout", (Object)60000);
final ThreadSafeClientConnManager connManager = new ThreadSafeClientConnManager(schemeRegistry);
connManager.setMaxTotal(200);
connManager.setDefaultMaxPerRoute(200);
this.httpClient = (HttpClient)new DefaultHttpClient((ClientConnectionManager)connManager, httpParams);
if (TriggerExecutor.LOGGER.isDebugEnabled()) {
TriggerExecutor.LOGGER.debug("trigger executor initialized[{}]", (Object)this.toString());
}
}
public static TriggerExecutor getInstance() {
return TriggerExecutor.TRIGGER_EXECUTOR;
}
public List<BSONObject> execute(final Trigger trigger, final List<BSONObject> docList) throws TriggerExecuteException {
try {
final List<BSONObject> ret = this.doExecute(trigger, docList);
if (TriggerExecutor.LOGGER.isDebugEnabled()) {
TriggerExecutor.LOGGER.debug("execute trigger: " + trigger + ", doc size:" + docList.size() + ", return size:" + ret.size());
}
return ret;
}
catch (IOException e) {
throw new TriggerExecuteException("error execute trigger: " + trigger, e);
}
catch (URISyntaxException e2) {
throw new TriggerExecuteException("error execute trigger: " + trigger, e2);
}
}
private List<BSONObject> doExecute(final Trigger trigger, final List<BSONObject> docList) throws IOException, URISyntaxException, TriggerExecuteException {
final URL url = new URL(trigger.getInterface());
final HttpPost httpPost = new HttpPost(url.toURI());
final DBObject object = (DBObject)new BasicDBObject();
object.put("doc_list_size", (Object)docList.size());
object.put("doc_list", (Object)docList);
httpPost.setEntity((HttpEntity)new ByteArrayEntity(GZIPUtils.makeAsByteArray(new BasicBSONEncoder().encode((BSONObject)object))));
final HttpHost host = new HttpHost(url.getHost(), (url.getPort() == -1) ? 80 : url.getPort());
final HttpResponse httpResponse = this.executeWithRetry(host, httpPost, null, trigger);
final InputStream inputStream = httpResponse.getEntity().getContent();
try {
if (httpResponse.getStatusLine().getStatusCode() != 200) {
throw new TriggerExecuteException("error execute trigger:" + trigger + ", get http code:" + httpResponse.getStatusLine().getStatusCode());
}
final BSONObject returnObject = new BasicBSONDecoder().readObject(GZIPUtils.wrap(inputStream));
if (null == returnObject) {
throw new TriggerExecuteException("trigger" + trigger + " returns nothing at all !");
}
final List<BSONObject> ret = (List<BSONObject>)returnObject.get("doc_list");
if (ret == null) {
throw new TriggerExecuteException("trigger" + trigger + " return an null doc_list[" + ret + "]");
}
return ret;
}
finally {
if (null != inputStream) {
inputStream.close();
}
}
}
private HttpResponse executeWithRetry(final HttpHost host, final HttpPost httpPost, final HttpContext httpContext, final Trigger trigger) throws IOException, TriggerExecuteException {
int retryCounter = 0;
HttpResponse httpResponse = null;
while (true) {
try {
httpResponse = this.httpClient.execute(host, (HttpRequest)httpPost, (HttpContext)null);
if (httpResponse.getStatusLine().getStatusCode() == 200) {
return httpResponse;
}
++retryCounter;
this.sleepAndRetry(new IllegalStateException("invalid http status code: " + httpResponse.getStatusLine().getStatusCode()), retryCounter, trigger);
}
catch (Exception e) {
++retryCounter;
this.sleepAndRetry(e, retryCounter, trigger);
}
finally {
if (null != httpResponse && httpResponse.getStatusLine().getStatusCode() != 200) {
httpResponse.getEntity().getContent().close();
}
}
}
}
private void sleepAndRetry(final Exception e, final int retryCounter, final Trigger trigger) throws TriggerExecuteException {
if (retryCounter > 10) {
throw new TriggerExecuteException("error execute trigger[" + trigger + "] after " + 10 + " times retry!", e);
}
if (TriggerExecutor.LOGGER.isErrorEnabled()) {
TriggerExecutor.LOGGER.error("error execute trigger[" + trigger + "]:(" + e.getMessage() + "), retry for the " + retryCounter + "th time after " + retryCounter * 30 + " seconds...");
}
try {
Thread.sleep(retryCounter * 30000);
}
catch (InterruptedException e2) {
if (TriggerExecutor.LOGGER.isWarnEnabled()) {
TriggerExecutor.LOGGER.warn("", (Throwable)e);
}
}
}
static {
LOGGER = LoggerFactory.getLogger((Class)TriggerExecutor.class);
TRIGGER_EXECUTOR = new TriggerExecutor();
}
}
@@ -1,170 +1,172 @@
package com.fangjia.datastore.config;
import java.io.File;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.dom4j.Document;
import org.dom4j.DocumentException;
import java.io.File;
import java.util.Iterator;
import java.util.List;
import org.dom4j.Element;
import org.dom4j.Node;
import org.dom4j.Document;
import java.util.Collections;
import java.util.Set;
import com.fangjia.datastore.ConfigurationException;
import java.util.HashMap;
import java.util.Map;
import org.dom4j.io.SAXReader;
import com.fangjia.datastore.ConfigurationException;
public class DataStoreConfig implements Reloadable {
private final SAXReader reader = new SAXReader();
private final String serverConfigFile;
private final String storeConfigFile;
private final ServerConfig serverConfig;
private final Map<String, StoreConfig> storeConfigs;
private final Map<String, SolrConfig> solrConfigs;
private final Map<String, MongoConfig> mongoConfigs;
private boolean readOnly = false;
public DataStoreConfig(String datastoreHome) throws ConfigurationException {
this.serverConfigFile = datastoreHome + "/conf/server.xml";
this.storeConfigFile = datastoreHome + "/conf/store.xml";
this.serverConfig = new ServerConfig();
this.storeConfigs = new HashMap<String, StoreConfig>();
this.solrConfigs = new HashMap<String, SolrConfig>();
this.mongoConfigs = new HashMap<String, MongoConfig>();
load();
}
public ServerConfig getServerConfig() {
return serverConfig;
}
public SolrConfig getSolrConfigById(String id) {
return solrConfigs.get(id);
}
public MongoConfig getMongoConfigById(String id) {
return this.mongoConfigs.get(id);
}
public StoreConfig getStoreConfig(String storeName) {
return this.storeConfigs.get(storeName);
}
public Set<String> getStoreNames() {
return Collections.unmodifiableSet(this.storeConfigs.keySet());
}
public void reload() throws ConfigurationException {
load();
}
private synchronized void load() throws ConfigurationException {
Document document = read(this.serverConfigFile);
serverConfig.reload(document.selectSingleNode("//datastore//server"));
initSolrConfigs(document);
initMongoConfigs(document);
document = read(this.storeConfigFile);
initStoreConfigs(document, solrConfigs, mongoConfigs);
}
synchronized StoreConfig getOrCreateStoreConfig(String storeName) {
StoreConfig ret = this.storeConfigs.get(storeName);
if (null == ret) {
ret = new StoreConfig(this, storeName);
storeConfigs.put(storeName, ret);
}
return ret;
}
private synchronized SolrConfig getOrCreateSolrConfig(String solrId) {
SolrConfig ret = this.solrConfigs.get(solrId);
if (null == ret) {
ret = new SolrConfig(this, solrId);
solrConfigs.put(solrId, ret);
}
return ret;
}
private synchronized MongoConfig getOrCreateMongoConfig(String mongoId) throws ConfigurationException {
MongoConfig ret = this.mongoConfigs.get(mongoId);
if (null == ret) {
ret = new MongoConfig(mongoId);
mongoConfigs.put(mongoId, ret);
}
return ret;
}
private void initSolrConfigs(Document document) throws ConfigurationException {
List<Node> solrNodes = document.selectNodes("//datastore//solr");
if (null == solrNodes || solrNodes.isEmpty()) {
throw new ConfigurationException("no solr is configured in " + serverConfigFile + " !");
}
Map<String, SolrConfig> newSolrConfigs = new HashMap<String, SolrConfig>();
for (Node solrNode : solrNodes) {
String id = ((Element) solrNode).attributeValue("id");
SolrConfig config = getOrCreateSolrConfig(id);
config.reload(solrNode);
newSolrConfigs.put(id, config);
}
this.solrConfigs.clear();
this.solrConfigs.putAll(newSolrConfigs);
}
private void initMongoConfigs(Document document) throws ConfigurationException {
List<Node> mongoNodes = document.selectNodes("//datastore//mongo");
if (null == mongoNodes || mongoNodes.isEmpty()) {
throw new ConfigurationException("no mongo is configured in " + serverConfigFile + " !");
}
Map<String, MongoConfig> newMongoConfigs = new HashMap<String, MongoConfig>();
for (Node mongoNode : mongoNodes) {
String id = ((Element) mongoNode).attributeValue("id");
MongoConfig mongoConfig = getOrCreateMongoConfig(id);
mongoConfig.reload(mongoNode);
newMongoConfigs.put(id, mongoConfig);
}
this.mongoConfigs.clear();
this.mongoConfigs.putAll(newMongoConfigs);
}
private void initStoreConfigs(Document document, Map<String, SolrConfig> solrConfigMap,
Map<String, MongoConfig> mongoConfigMap) throws ConfigurationException {
List<Node> storeNodes = document.selectNodes("//datastore//store");
if (null == storeNodes || storeNodes.isEmpty()) {
throw new ConfigurationException("no store is configured in " + storeConfigFile + " !");
}
Map<String, StoreConfig> newStoreConfigs = new HashMap<String, StoreConfig>();
for (Node storeNode : storeNodes) {
String name = ((Element) storeNode).attributeValue("name");
final StoreConfig storeConfig = getOrCreateStoreConfig(name);
storeConfig.reload(storeNode);
newStoreConfigs.put(name, storeConfig);
}
this.storeConfigs.clear();
this.storeConfigs.putAll(newStoreConfigs);
}
private Document read(String configFile) throws ConfigurationException {
try {
return reader.read(new File(configFile));
} catch (DocumentException e) {
throw new ConfigurationException("error read datastore config file: " + configFile, e);
}
}
@Override
public String toString() {
StringBuilder ret = new StringBuilder();
ret.append("server config file:").append(this.serverConfigFile).append(", store config file: ").append(
this.storeConfigFile).append(", readOnly:").append(this.readOnly);
return ret.toString();
}
public boolean isReadOnly() {
return readOnly;
}
public void setReadOnly(boolean readOnly) {
this.readOnly = readOnly;
}
public class DataStoreConfig implements Reloadable
{
private final SAXReader reader;
private final String serverConfigFile;
private final String storeConfigFile;
private final ServerConfig serverConfig;
private final Map<String, StoreConfig> storeConfigs;
private final Map<String, SolrConfig> solrConfigs;
private final Map<String, MongoConfig> mongoConfigs;
private boolean readOnly;
public DataStoreConfig(final String datastoreHome) throws ConfigurationException {
this.reader = new SAXReader();
this.readOnly = false;
this.serverConfigFile = datastoreHome + "/conf/server.xml";
this.storeConfigFile = datastoreHome + "/conf/store.xml";
this.serverConfig = new ServerConfig();
this.storeConfigs = new HashMap<String, StoreConfig>();
this.solrConfigs = new HashMap<String, SolrConfig>();
this.mongoConfigs = new HashMap<String, MongoConfig>();
this.load();
}
public ServerConfig getServerConfig() {
return this.serverConfig;
}
public SolrConfig getSolrConfigById(final String id) {
return this.solrConfigs.get(id);
}
public MongoConfig getMongoConfigById(final String id) {
return this.mongoConfigs.get(id);
}
public StoreConfig getStoreConfig(final String storeName) {
return this.storeConfigs.get(storeName);
}
public Set<String> getStoreNames() {
return Collections.<String>unmodifiableSet((Set<? extends String>)this.storeConfigs.keySet());
}
@Override
public void reload() throws ConfigurationException {
this.load();
}
private synchronized void load() throws ConfigurationException {
Document document = this.read(this.serverConfigFile);
this.serverConfig.reload(document.selectSingleNode("//datastore//server"));
this.initSolrConfigs(document);
this.initMongoConfigs(document);
document = this.read(this.storeConfigFile);
this.initStoreConfigs(document, this.solrConfigs, this.mongoConfigs);
}
synchronized StoreConfig getOrCreateStoreConfig(final String storeName) {
StoreConfig ret = this.storeConfigs.get(storeName);
if (null == ret) {
ret = new StoreConfig(this, storeName);
this.storeConfigs.put(storeName, ret);
}
return ret;
}
private synchronized SolrConfig getOrCreateSolrConfig(final String solrId) {
SolrConfig ret = this.solrConfigs.get(solrId);
if (null == ret) {
ret = new SolrConfig(this, solrId);
this.solrConfigs.put(solrId, ret);
}
return ret;
}
private synchronized MongoConfig getOrCreateMongoConfig(final String mongoId) throws ConfigurationException {
MongoConfig ret = this.mongoConfigs.get(mongoId);
if (null == ret) {
ret = new MongoConfig(mongoId);
this.mongoConfigs.put(mongoId, ret);
}
return ret;
}
private void initSolrConfigs(final Document document) throws ConfigurationException {
final List<Node> solrNodes = (List<Node>)document.selectNodes("//datastore//solr");
if (null == solrNodes || solrNodes.isEmpty()) {
throw new ConfigurationException("no solr is configured in " + this.serverConfigFile + " !");
}
final Map<String, SolrConfig> newSolrConfigs = new HashMap<String, SolrConfig>();
for (final Node solrNode : solrNodes) {
final String id = ((Element)solrNode).attributeValue("id");
final SolrConfig config = this.getOrCreateSolrConfig(id);
config.reload(solrNode);
newSolrConfigs.put(id, config);
}
this.solrConfigs.clear();
this.solrConfigs.putAll(newSolrConfigs);
}
private void initMongoConfigs(final Document document) throws ConfigurationException {
final List<Node> mongoNodes = (List<Node>)document.selectNodes("//datastore//mongo");
if (null == mongoNodes || mongoNodes.isEmpty()) {
throw new ConfigurationException("no mongo is configured in " + this.serverConfigFile + " !");
}
final Map<String, MongoConfig> newMongoConfigs = new HashMap<String, MongoConfig>();
for (final Node mongoNode : mongoNodes) {
final String id = ((Element)mongoNode).attributeValue("id");
final MongoConfig mongoConfig = this.getOrCreateMongoConfig(id);
mongoConfig.reload(mongoNode);
newMongoConfigs.put(id, mongoConfig);
}
this.mongoConfigs.clear();
this.mongoConfigs.putAll(newMongoConfigs);
}
private void initStoreConfigs(final Document document, final Map<String, SolrConfig> solrConfigMap, final Map<String, MongoConfig> mongoConfigMap) throws ConfigurationException {
final List<Node> storeNodes = (List<Node>)document.selectNodes("//datastore//store");
if (null == storeNodes || storeNodes.isEmpty()) {
throw new ConfigurationException("no store is configured in " + this.storeConfigFile + " !");
}
final Map<String, StoreConfig> newStoreConfigs = new HashMap<String, StoreConfig>();
for (final Node storeNode : storeNodes) {
final String name = ((Element)storeNode).attributeValue("name");
final StoreConfig storeConfig = this.getOrCreateStoreConfig(name);
storeConfig.reload(storeNode);
newStoreConfigs.put(name, storeConfig);
}
this.storeConfigs.clear();
this.storeConfigs.putAll(newStoreConfigs);
}
private Document read(final String configFile) throws ConfigurationException {
try {
return this.reader.read(new File(configFile));
}
catch (DocumentException e) {
throw new ConfigurationException("error read datastore config file: " + configFile, (Throwable)e);
}
}
@Override
public String toString() {
final StringBuilder ret = new StringBuilder();
ret.append("server config file:").append(this.serverConfigFile).append(", store config file: ").append(this.storeConfigFile).append(", readOnly:").append(this.readOnly);
return ret.toString();
}
public boolean isReadOnly() {
return this.readOnly;
}
public void setReadOnly(final boolean readOnly) {
this.readOnly = readOnly;
}
}
@@ -1,127 +1,118 @@
package com.fangjia.datastore.config;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.bson.types.ObjectId;
import java.util.regex.Pattern;
import java.util.Date;
import java.util.Map;
import java.util.List;
import org.apache.commons.lang.StringUtils;
public class Field {
private String name;
private FieldTypeEnum type;
private boolean required;
Field() {
}
public String getName() {
return name;
}
public FieldTypeEnum getType() {
return type;
}
void setName(String name) {
if (StringUtils.isEmpty(name)) {
throw new IllegalArgumentException("invalid field name:" + name);
}
this.name = name;
}
void setType(FieldTypeEnum type) {
if (null == type) {
throw new IllegalArgumentException("invalid type of field[" + name + "]:" + type);
}
this.type = type;
}
public boolean isRequired() {
return required;
}
void setRequired(boolean required) {
this.required = required;
}
public boolean isBigField() {
return this.type == FieldTypeEnum.TEXT;
}
/**
* 验证一个值是否匹配其对应的类型
*
* @param value
* @return 匹配则返回true,不匹配返回false
*/
public boolean validate(Object value) {
if (this.required && value == null) {
return false;
}
if (value == null) {
return true;
}
Class<?> valueClass = value.getClass();
switch (this.type) {
case BOOLEAN:
return valueClass == boolean.class || valueClass == Boolean.class;
case BYTE:
return valueClass == byte.class || valueClass == Byte.class;
case INTEGER:
return valueClass == int.class || valueClass == Integer.class;
case LONG:
return valueClass == long.class || valueClass == Long.class;
case DOUBLE:
return valueClass == double.class || valueClass == Double.class;
case STRING:
case TEXT:
return valueClass == String.class;
case LIST:
return value instanceof List;
case MAP:
return value instanceof Map;
case DATETIME:
return value instanceof Date;
case BINARY:
return valueClass == byte[].class;
case REGEXP:
return valueClass == Pattern.class;
case OBJECTID:
return valueClass == ObjectId.class;
default:
throw new RuntimeException("un cased type:" + this);
}
}
@Override
public int hashCode() {
return new HashCodeBuilder().append(this.name).append(this.type).append(this.required).toHashCode();
}
@Override
public boolean equals(Object obj) {
if (obj == this) {
return true;
}
if (null == obj) {
return false;
}
if (!(obj instanceof Field)) {
return false;
}
Field other = (Field) obj;
return new EqualsBuilder().append(this.name, other.name).append(this.type, other.type)
.append(this.required, other.required).isEquals();
}
@Override
public String toString() {
return new StringBuilder().append("{field:").append(name).append(", type:").append(type).append(", required:")
.append(required).append("}").toString();
}
public class Field
{
private String name;
private FieldTypeEnum type;
private boolean required;
Field() {
}
public String getName() {
return this.name;
}
public FieldTypeEnum getType() {
return this.type;
}
void setName(final String name) {
if (StringUtils.isEmpty(name)) {
throw new IllegalArgumentException("invalid field name:" + name);
}
this.name = name;
}
void setType(final FieldTypeEnum type) {
if (null == type) {
throw new IllegalArgumentException("invalid type of field[" + this.name + "]:" + type);
}
this.type = type;
}
public boolean isRequired() {
return this.required;
}
void setRequired(final boolean required) {
this.required = required;
}
public boolean isBigField() {
return this.type == FieldTypeEnum.TEXT;
}
public boolean validate(final Object value) {
if (this.required && value == null) {
return false;
}
if (value == null) {
return true;
}
final Class<?> valueClass = value.getClass();
switch (this.type) {
case BOOLEAN:
return valueClass == Boolean.TYPE || valueClass == Boolean.class;
case BYTE:
return valueClass == Byte.TYPE || valueClass == Byte.class;
case INTEGER:
return valueClass == Integer.TYPE || valueClass == Integer.class;
case LONG:
return valueClass == Long.TYPE || valueClass == Long.class;
case DOUBLE:
return valueClass == Double.TYPE || valueClass == Double.class;
case STRING:
case TEXT:
return valueClass == String.class;
case LIST:
return value instanceof List;
case MAP:
return value instanceof Map;
case DATETIME:
return value instanceof Date;
case BINARY:
return valueClass == byte[].class;
case REGEXP:
return valueClass == Pattern.class;
case OBJECTID:
return valueClass == ObjectId.class;
default:
throw new RuntimeException("un cased type:" + this);
}
}
@Override
public int hashCode() {
return new HashCodeBuilder().append((Object)this.name).append((Object)this.type).append(this.required).toHashCode();
}
@Override
public boolean equals(final Object obj) {
if (obj == this) {
return true;
}
if (null == obj) {
return false;
}
if (!(obj instanceof Field)) {
return false;
}
final Field other = (Field)obj;
return new EqualsBuilder().append((Object)this.name, (Object)other.name).append((Object)this.type, (Object)other.type).append(this.required, other.required).isEquals();
}
@Override
public String toString() {
return "{field:" + this.name + ", type:" + this.type + ", required:" + this.required + "}";
}
}
@@ -1,22 +1,31 @@
package com.fangjia.datastore.config;
public enum FieldTypeEnum {
BOOLEAN, BYTE, INTEGER, LONG, DOUBLE, STRING, TEXT, LIST, MAP, DATETIME, BINARY, REGEXP, OBJECTID;
private FieldTypeEnum() {
}
public String getType() {
return this.name().toLowerCase();
}
public static FieldTypeEnum parse(String type) {
for (FieldTypeEnum typeEnum : FieldTypeEnum.values()) {
if (typeEnum.name().toLowerCase().equals(type)) {
return typeEnum;
}
}
return null;
}
public enum FieldTypeEnum
{
BOOLEAN,
BYTE,
INTEGER,
LONG,
DOUBLE,
STRING,
TEXT,
LIST,
MAP,
DATETIME,
BINARY,
REGEXP,
OBJECTID;
public String getType() {
return this.name().toLowerCase();
}
public static FieldTypeEnum parse(final String type) {
for (final FieldTypeEnum typeEnum : values()) {
if (typeEnum.name().toLowerCase().equals(type)) {
return typeEnum;
}
}
return null;
}
}
@@ -2,87 +2,98 @@ package com.fangjia.datastore.config;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.dom4j.Node;
import com.fangjia.datastore.ConfigurationException;
import com.fangjia.datastore.utils.XmlUtils;
import org.dom4j.Node;
import com.mongodb.MongoURI;
import org.apache.commons.lang.StringUtils;
import com.fangjia.datastore.ConfigurationException;
import com.mongodb.Mongo;
public final class MongoConfig {
private String host;
private int port;
private final String id;
private Mongo mongo;
MongoConfig(String id) throws ConfigurationException {
this.id = id;
}
private synchronized void init() throws ConfigurationException {
if (null == mongo) {
try {
mongo = new Mongo(host, port);
} catch (Exception e) {
throw new ConfigurationException("error init mongo[" + host + ":" + port, e);
}
}
}
public Mongo getMongo() throws ConfigurationException {
if (null == mongo) {
init();
}
return mongo;
}
synchronized void reload(Node mongoNode) throws ConfigurationException {
String host = XmlUtils.getNodeValue(mongoNode, "host", "datastore.mongo.db.fangjia.com");
int port = XmlUtils.getNodeValueAsInt(mongoNode, "port", 27017);
if (host.equals(this.host) && this.port == port) {
return;
}
this.host = host;
this.port = port;
if (null != mongo) {
try {
init();
} catch (Exception e) {
throw new ConfigurationException("error init mongo[" + host + ":" + port, e);
}
}
}
@Override
public int hashCode() {
return new HashCodeBuilder().append(this.host).append(this.port).append(this.id).toHashCode();
}
@Override
public boolean equals(Object obj) {
if (obj == this) {
return true;
}
if (null == obj) {
return false;
}
if (!(obj instanceof MongoConfig)) {
return false;
}
MongoConfig other = (MongoConfig) obj;
return new EqualsBuilder().append(this.id, other.id).append(this.host, other.host).append(this.port, other.port)
.isEquals();
}
@Override
public String toString() {
StringBuilder ret = new StringBuilder();
ret.append('{');
ret.append("id:").append(this.id);
ret.append(", host:").append(this.host);
ret.append(", port:").append(this.port);
ret.append('}');
return ret.toString();
}
public final class MongoConfig
{
private String host;
private String uri;
private int port;
private final String id;
private Mongo mongo;
MongoConfig(final String id) throws ConfigurationException {
this.id = id;
}
private synchronized void init() throws ConfigurationException {
if (null == this.mongo) {
try {
MongoURI mongoURI;
if (StringUtils.isNotBlank(this.uri)) {
mongoURI = new MongoURI(this.uri);
}
else {
mongoURI = new MongoURI("mongodb://" + this.host + ":" + this.port);
}
this.mongo = new Mongo(mongoURI);
}
catch (Exception e) {
throw new ConfigurationException("error init mongo[" + this.host + ":" + this.port, e);
}
}
}
public Mongo getMongo() throws ConfigurationException {
if (null == this.mongo) {
this.init();
}
return this.mongo;
}
synchronized void reload(final Node mongoNode) throws ConfigurationException {
final String host = XmlUtils.getNodeValue(mongoNode, "host", "datastore.mongo.db.fangjia.com");
final int port = XmlUtils.getNodeValueAsInt(mongoNode, "port", 27017);
final String uri = XmlUtils.getNodeValue(mongoNode, "uri", "");
if (host.equals(this.host) && this.port == port) {
return;
}
this.host = host;
this.port = port;
this.uri = uri;
if (null != this.mongo) {
try {
this.init();
}
catch (Exception e) {
throw new ConfigurationException("error init mongo[" + host + ":" + port, e);
}
}
}
@Override
public int hashCode() {
return new HashCodeBuilder().append((Object)this.host).append(this.port).append((Object)this.id).toHashCode();
}
@Override
public boolean equals(final Object obj) {
if (obj == this) {
return true;
}
if (null == obj) {
return false;
}
if (!(obj instanceof MongoConfig)) {
return false;
}
final MongoConfig other = (MongoConfig)obj;
return new EqualsBuilder().append((Object)this.id, (Object)other.id).append((Object)this.host, (Object)other.host).append(this.port, other.port).isEquals();
}
@Override
public String toString() {
final StringBuilder ret = new StringBuilder();
ret.append('{');
ret.append("id:").append(this.id);
ret.append(", host:").append(this.host);
ret.append(", port:").append(this.port);
ret.append('}');
return ret.toString();
}
}
@@ -2,6 +2,7 @@ package com.fangjia.datastore.config;
import com.fangjia.datastore.ConfigurationException;
public interface Reloadable {
void reload() throws ConfigurationException;
public interface Reloadable
{
void reload() throws ConfigurationException;
}
@@ -1,74 +1,72 @@
package com.fangjia.datastore.config;
import com.fangjia.datastore.utils.XmlUtils;
import com.fangjia.datastore.ConfigurationException;
import org.dom4j.Node;
import com.fangjia.datastore.ConfigurationException;
import com.fangjia.datastore.utils.XmlUtils;
public final class ServerConfig {
private int port;
private boolean disableLogging;
private long performanceLogFilterMaxTime;
private int minaReadBufferSize;
private int cursorExpirationMinutes;
private int defaultCursorLimit;
private int maxConnections;
private int triggerBatchSize;
private int reIndexBatchSize;
ServerConfig() {
super();
}
void reload(Node server) throws ConfigurationException {
if (null == server) {
throw new ConfigurationException("no server is configured!");
}
this.port = XmlUtils.getNodeValueAsInt(server, "port", 27017);
this.disableLogging = XmlUtils.getNodeValueAsBoolean(server, "disableLogging", false);
this.performanceLogFilterMaxTime = XmlUtils.getNodeValueAsLong(server, "performanceLogFilterMaxTime", 1000);
this.minaReadBufferSize = XmlUtils.getNodeValueAsInt(server, "minaReadBufferSize", 20480);
this.cursorExpirationMinutes = XmlUtils.getNodeValueAsInt(server, "cursorExpirationMinutes", 5);
this.defaultCursorLimit = XmlUtils.getNodeValueAsInt(server, "defaultCursorLimit", 20);
this.maxConnections = XmlUtils.getNodeValueAsInt(server, "maxConnections", 10);
this.triggerBatchSize = XmlUtils.getNodeValueAsInt(server, "triggerBatchSize", 200);
this.reIndexBatchSize = XmlUtils.getNodeValueAsInt(server, "reIndexBatchSize", 200);
}
public int getPort() {
return port;
}
public boolean isDisableLogging() {
return disableLogging;
}
public long getPerformanceLogFilterMaxTime() {
return performanceLogFilterMaxTime;
}
public int getMinaReadBufferSize() {
return minaReadBufferSize;
}
public int getCursorExpirationMinutes() {
return cursorExpirationMinutes;
}
public int getDefaultCursorLimit() {
return defaultCursorLimit;
}
public int getMaxConnections() {
return maxConnections;
}
public int getTriggerBatchSize() {
return triggerBatchSize;
}
public int getReIndexBatchSize() {
return reIndexBatchSize;
}
public final class ServerConfig
{
private int port;
private boolean disableLogging;
private long performanceLogFilterMaxTime;
private int minaReadBufferSize;
private int cursorExpirationMinutes;
private int defaultCursorLimit;
private int maxConnections;
private int triggerBatchSize;
private int reIndexBatchSize;
ServerConfig() {
}
void reload(final Node server) throws ConfigurationException {
if (null == server) {
throw new ConfigurationException("no server is configured!");
}
this.port = XmlUtils.getNodeValueAsInt(server, "port", 27017);
this.disableLogging = XmlUtils.getNodeValueAsBoolean(server, "disableLogging", false);
this.performanceLogFilterMaxTime = XmlUtils.getNodeValueAsLong(server, "performanceLogFilterMaxTime", 1000L);
this.minaReadBufferSize = XmlUtils.getNodeValueAsInt(server, "minaReadBufferSize", 20480);
this.cursorExpirationMinutes = XmlUtils.getNodeValueAsInt(server, "cursorExpirationMinutes", 5);
this.defaultCursorLimit = XmlUtils.getNodeValueAsInt(server, "defaultCursorLimit", 20);
this.maxConnections = XmlUtils.getNodeValueAsInt(server, "maxConnections", 10);
this.triggerBatchSize = XmlUtils.getNodeValueAsInt(server, "triggerBatchSize", 200);
this.reIndexBatchSize = XmlUtils.getNodeValueAsInt(server, "reIndexBatchSize", 200);
}
public int getPort() {
return this.port;
}
public boolean isDisableLogging() {
return this.disableLogging;
}
public long getPerformanceLogFilterMaxTime() {
return this.performanceLogFilterMaxTime;
}
public int getMinaReadBufferSize() {
return this.minaReadBufferSize;
}
public int getCursorExpirationMinutes() {
return this.cursorExpirationMinutes;
}
public int getDefaultCursorLimit() {
return this.defaultCursorLimit;
}
public int getMaxConnections() {
return this.maxConnections;
}
public int getTriggerBatchSize() {
return this.triggerBatchSize;
}
public int getReIndexBatchSize() {
return this.reIndexBatchSize;
}
}
@@ -1,132 +1,132 @@
package com.fangjia.datastore.config;
import com.fangjia.datastore.ConfigurationException;
import org.apache.commons.lang.StringUtils;
import com.fangjia.datastore.utils.XmlUtils;
import org.dom4j.Node;
import com.fangjia.datastore.ConfigurationException;
import com.fangjia.datastore.utils.XmlUtils;
public class SolrConfig {
private final String id;
private String host;
private String home;
private final DataStoreConfig dataStoreConfig;
private String shards;
private String slaves;
private SolrConfig[] shardConfigs;
private SolrConfig[] slaveConfigs;
private String filter;
SolrConfig(DataStoreConfig dataStoreConfig, String id) {
this.dataStoreConfig = dataStoreConfig;
this.id = id;
}
synchronized void reload(Node solrNode) throws ConfigurationException {
this.host = removeLastSlash(XmlUtils.getNodeValue(solrNode, "host", null));
this.shards = XmlUtils.getNodeValue(solrNode, "shards", null);
this.slaves = XmlUtils.getNodeValue(solrNode, "slaves", null);
this.home = XmlUtils.getNodeValue(solrNode, "home", null);
if (StringUtils.isNotEmpty(shards) && StringUtils.isEmpty(home)) {
throw new ConfigurationException("[" + id + "] invalid home attribute:" + home);
}
if (StringUtils.isEmpty(shards) && StringUtils.isEmpty(slaves) && StringUtils.isEmpty(host)) {
throw new ConfigurationException("[" + id + "] invalid host attribute:" + host);
}
this.filter = removeLastSlash(XmlUtils.getNodeValue(solrNode, "filter", null));
}
private SolrConfig[] getConfigs(String ids) {
SolrConfig[] ret;
if (StringUtils.isBlank(ids)) {
ret = new SolrConfig[0];
} else {
String[] idSplit = ids.split(",");
ret = new SolrConfig[idSplit.length];
for (int i = 0; i < idSplit.length; i++) {
ret[i] = dataStoreConfig.getSolrConfigById(idSplit[i]);
}
}
return ret;
}
public SolrConfig[] getShardsConfigs() {
if (null == shardConfigs) {
shardConfigs = this.getConfigs(shards);
}
return shardConfigs;
}
public SolrConfig[] getSlaveConfigs() {
if (null == slaveConfigs) {
slaveConfigs = this.getConfigs(slaves);
}
return slaveConfigs;
}
private String removeLastSlash(String s) {
if (StringUtils.isEmpty(s)) {
return s;
}
if (s.charAt(s.length() - 1) == '/') {
return s.substring(0, s.length() - 1);
}
return s;
}
public String getShardsParam(String coreName) {
SolrConfig[] shardConfigs = getShardsConfigs();
if (shardConfigs.length == 0) {
return "";
}
StringBuilder ret = new StringBuilder();
for (int i = 0; i < shardConfigs.length; i++) {
ret.append(shardConfigs[i].host).append('/').append(coreName);
if (i < shardConfigs.length - 1) {
ret.append(',');
}
}
return ret.toString();
}
public String getId() {
return id;
}
public String getHost() {
return host;
}
public DataStoreConfig getDataStoreConfig() {
return dataStoreConfig;
}
public String getShards() {
return shards;
}
public String getFilter() {
return filter;
}
public String getHome() {
return home;
}
@Override
public String toString() {
StringBuilder ret = new StringBuilder();
ret.append('{');
ret.append("id: ").append(this.id);
ret.append(", host: ").append(this.host);
ret.append(", shards: ").append(this.shards);
ret.append(", slaves: ").append(this.slaves);
ret.append('}');
return ret.toString();
}
public String getSlaves() {
return slaves;
}
public class SolrConfig
{
private final String id;
private String host;
private String home;
private final DataStoreConfig dataStoreConfig;
private String shards;
private String slaves;
private SolrConfig[] shardConfigs;
private SolrConfig[] slaveConfigs;
private String filter;
SolrConfig(final DataStoreConfig dataStoreConfig, final String id) {
this.dataStoreConfig = dataStoreConfig;
this.id = id;
}
synchronized void reload(final Node solrNode) throws ConfigurationException {
this.host = this.removeLastSlash(XmlUtils.getNodeValue(solrNode, "host", null));
this.shards = XmlUtils.getNodeValue(solrNode, "shards", null);
this.slaves = XmlUtils.getNodeValue(solrNode, "slaves", null);
this.home = XmlUtils.getNodeValue(solrNode, "home", null);
if (StringUtils.isNotEmpty(this.shards) && StringUtils.isEmpty(this.home)) {
throw new ConfigurationException("[" + this.id + "] invalid home attribute:" + this.home);
}
if (StringUtils.isEmpty(this.shards) && StringUtils.isEmpty(this.slaves) && StringUtils.isEmpty(this.host)) {
throw new ConfigurationException("[" + this.id + "] invalid host attribute:" + this.host);
}
this.filter = this.removeLastSlash(XmlUtils.getNodeValue(solrNode, "filter", null));
}
private SolrConfig[] getConfigs(final String ids) {
SolrConfig[] ret;
if (StringUtils.isBlank(ids)) {
ret = new SolrConfig[0];
}
else {
final String[] idSplit = ids.split(",");
ret = new SolrConfig[idSplit.length];
for (int i = 0; i < idSplit.length; ++i) {
ret[i] = this.dataStoreConfig.getSolrConfigById(idSplit[i]);
}
}
return ret;
}
public SolrConfig[] getShardsConfigs() {
if (null == this.shardConfigs) {
this.shardConfigs = this.getConfigs(this.shards);
}
return this.shardConfigs;
}
public SolrConfig[] getSlaveConfigs() {
if (null == this.slaveConfigs) {
this.slaveConfigs = this.getConfigs(this.slaves);
}
return this.slaveConfigs;
}
private String removeLastSlash(final String s) {
if (StringUtils.isEmpty(s)) {
return s;
}
if (s.charAt(s.length() - 1) == '/') {
return s.substring(0, s.length() - 1);
}
return s;
}
public String getShardsParam(final String coreName) {
final SolrConfig[] shardConfigs = this.getShardsConfigs();
if (shardConfigs.length == 0) {
return "";
}
final StringBuilder ret = new StringBuilder();
for (int i = 0; i < shardConfigs.length; ++i) {
ret.append(shardConfigs[i].host).append('/').append(coreName);
if (i < shardConfigs.length - 1) {
ret.append(',');
}
}
return ret.toString();
}
public String getId() {
return this.id;
}
public String getHost() {
return this.host;
}
public DataStoreConfig getDataStoreConfig() {
return this.dataStoreConfig;
}
public String getShards() {
return this.shards;
}
public String getFilter() {
return this.filter;
}
public String getHome() {
return this.home;
}
@Override
public String toString() {
final StringBuilder ret = new StringBuilder();
ret.append('{');
ret.append("id: ").append(this.id);
ret.append(", host: ").append(this.host);
ret.append(", shards: ").append(this.shards);
ret.append(", slaves: ").append(this.slaves);
ret.append('}');
return ret.toString();
}
public String getSlaves() {
return this.slaves;
}
}
@@ -1,229 +1,208 @@
package com.fangjia.datastore.config;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import java.util.Collection;
import org.bson.types.ObjectId;
import java.util.Iterator;
import java.util.List;
import com.fangjia.datastore.utils.XmlUtils;
import com.fangjia.datastore.ConfigurationException;
import org.apache.commons.lang.StringUtils;
import org.dom4j.Element;
import org.dom4j.Node;
import java.util.HashMap;
import java.util.Map;
import com.fangjia.datastore.ConfigurationException;
import com.fangjia.datastore.utils.XmlUtils;
public final class StoreConfig {
private final String name;
private int solrMaxTotalConnections;
private int solrMaxConnectionsPerHost;
private SolrConfig[] masterSolrConfigs;
private SolrConfig masterSolrConfig;
private SolrConfig slaveSolrConfig;
private MongoConfig mongoConfig;
private boolean textFieldToGridFS;
private final Trigger trigger;
private final Map<String, Field> fields;
private final DataStoreConfig dataStoreConfig;
StoreConfig(DataStoreConfig dataStoreConfig, String name) {
this.dataStoreConfig = dataStoreConfig;
fields = new HashMap<String, Field>();
trigger = new Trigger();
this.name = name;
}
synchronized void reload(Node storeNode) throws ConfigurationException {
// init solr
String solrMastersAttr = ((Element) storeNode).attributeValue("solrMasters");
if (StringUtils.isEmpty(solrMastersAttr)) {
throw new ConfigurationException("[" + name + "] invalid solrMasters attribute:" + solrMastersAttr);
}
String[] solrMasterIds = solrMastersAttr.split(",");
if (solrMasterIds.length > 2) {
throw new ConfigurationException("[" + name + "] too many solr server configured:" + solrMastersAttr);
}
masterSolrConfigs = new SolrConfig[solrMasterIds.length];
for (int i = 0; i < solrMasterIds.length; i++) {
masterSolrConfigs[i] = dataStoreConfig.getSolrConfigById(solrMasterIds[i]);
if (null == masterSolrConfigs[i]) {
throw new ConfigurationException("[" + name + "] unknown solr id: " + solrMasterIds[i]);
}
}
String solrSlaveAttr = System.getProperty("datastore." + name + ".solrSlave", ((Element) storeNode)
.attributeValue("solrSlave"));
if (StringUtils.isEmpty(solrSlaveAttr)) {
throw new ConfigurationException("[" + name + "] invalid solrSlave attribute:" + solrSlaveAttr);
}
slaveSolrConfig = dataStoreConfig.getSolrConfigById(solrSlaveAttr);
if (null == slaveSolrConfig) {
throw new ConfigurationException("[" + name + "] unknown solr id: " + solrSlaveAttr);
}
String solrMasterAttr = System.getProperty("datastore." + name + ".solrMaster", ((Element) storeNode)
.attributeValue("solrMaster"));
if (StringUtils.isEmpty(solrMasterAttr)) {
throw new ConfigurationException("[" + name + "] invalid solrMaster attribute:" + solrMasterAttr);
}
masterSolrConfig = dataStoreConfig.getSolrConfigById(solrMasterAttr);
if (null == masterSolrConfig) {
throw new ConfigurationException("[" + name + "] unknown solr id: " + solrMasterAttr);
}
if (slaveSolrConfig.getShardsConfigs().length != masterSolrConfig.getShardsConfigs().length) {
throw new ConfigurationException("[" + name + "] master and slave shard count must equals !");
}
// solr connections defaults
this.solrMaxConnectionsPerHost = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxConnectionsPerHost", 300);
if (slaveSolrConfig.getShardsConfigs().length > 0) {
this.solrMaxTotalConnections = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxTotalConnections", slaveSolrConfig
.getShardsConfigs().length
* solrMaxConnectionsPerHost);
} else {
this.solrMaxTotalConnections = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxTotalConnections",
solrMaxConnectionsPerHost);
}
// init mongo
String mongoId = ((Element) storeNode).attributeValue("mongo");
MongoConfig mongoConfig = dataStoreConfig.getMongoConfigById(mongoId);
if (null == mongoConfig) {
throw new ConfigurationException("[" + name + "] unknown mongo id: " + mongoId);
}
this.mongoConfig = mongoConfig;
// init other config
this.textFieldToGridFS = XmlUtils.getNodeValueAsBoolean(storeNode, "textFieldToGridFS", false);
// init fields
Map<String, Field> newFields = new HashMap<String, Field>();
List<Node> fieldNodes = storeNode.selectNodes("schema//field");
for (Node fieldNode : fieldNodes) {
Element element = (Element) fieldNode;
String fieldName = element.attributeValue("name");
Field field = getOrCreateField(fieldName);
field.setName(fieldName);
field.setRequired(Boolean.parseBoolean(element.attributeValue("required")));
field.setType(FieldTypeEnum.parse(element.attributeValue("type")));
newFields.put(fieldName, field);
}
this.fields.clear();
this.fields.putAll(newFields);
// init triggers
Element triggerNode = (Element) storeNode.selectSingleNode("trigger");
if(triggerNode != null) {
trigger.setInterface(triggerNode.attributeValue("interface"));
}
}
public Trigger getTrigger() {
return this.trigger;
}
public Field getField(String fieldName) {
return this.fields.get(fieldName);
}
public boolean containsField(String fieldName) {
return this.fields.containsKey(fieldName);
}
public boolean isFieldValueLegal(String fieldName, Object value) {
if ("_id".equals(fieldName)) {
return value instanceof ObjectId;
}
if (fields.containsKey(fieldName)) {
Field field = fields.get(fieldName);
return field.validate(value);
}
return false;
}
public boolean isBigField(String fieldName) {
return fields.get(fieldName).isBigField();
}
public Collection<Field> getFields() {
return Collections.unmodifiableCollection(this.fields.values());
}
synchronized Field getOrCreateField(String fieldName) {
if (StringUtils.isEmpty(fieldName)) {
throw new IllegalArgumentException("fieldName can not be null or empty!");
}
Field ret = this.fields.get(fieldName);
if (null == ret) {
ret = new Field();
ret.setName(fieldName);
this.fields.put(fieldName, ret);
}
return ret;
}
public String getName() {
return name;
}
public MongoConfig getMongoConfig() {
return mongoConfig;
}
public DataStoreConfig getDataStoreConfig() {
return dataStoreConfig;
}
public boolean isTextFieldToGridFS() {
return textFieldToGridFS;
}
public SolrConfig getSlaveSolrConfig() {
return slaveSolrConfig;
}
public SolrConfig[] getMasterSolrConfigs() {
return masterSolrConfigs;
}
public SolrConfig getMasterSolrConfig() {
return masterSolrConfig;
}
public int getSolrMaxTotalConnections() {
return solrMaxTotalConnections;
}
public int getSolrMaxConnectionsPerHost() {
return solrMaxConnectionsPerHost;
}
@Override
public String toString() {
StringBuilder ret = new StringBuilder();
ret.append("\n{");
ret.append("\n\tname: ").append(name);
ret.append(",\n\tmasterSolrConfig: ").append(masterSolrConfig);
ret.append(",\n\tslaveSolrConfig: ").append(this.slaveSolrConfig);
ret.append(",\n\tmongoConfig: ").append(mongoConfig);
ret.append(",\n\tsolrMaxTotalConnections: ").append(solrMaxTotalConnections);
ret.append(",\n\tsolrMaxConnectionsPerHost: ").append(this.solrMaxConnectionsPerHost);
ret.append(",\n\ttextFieldToGridFS: ").append(this.textFieldToGridFS);
ret.append(",\n\ttrigger: ").append(trigger);
ret.append(",\n\tmasterSolrConfigs: [\n");
for (int i = 0; i < this.masterSolrConfigs.length; i++) {
ret.append('\t').append('\t').append(this.masterSolrConfigs[i]);
if (i < this.masterSolrConfigs.length - 1) {
ret.append(',');
}
ret.append('\n');
}
ret.append("\t]\n}");
return ret.toString();
}
public final class StoreConfig
{
private final String name;
private int solrMaxTotalConnections;
private int solrMaxConnectionsPerHost;
private SolrConfig[] masterSolrConfigs;
private SolrConfig masterSolrConfig;
private SolrConfig slaveSolrConfig;
private MongoConfig mongoConfig;
private boolean textFieldToGridFS;
private final Trigger trigger;
private final Map<String, Field> fields;
private final DataStoreConfig dataStoreConfig;
StoreConfig(final DataStoreConfig dataStoreConfig, final String name) {
this.dataStoreConfig = dataStoreConfig;
this.fields = new HashMap<String, Field>();
this.trigger = new Trigger();
this.name = name;
}
synchronized void reload(final Node storeNode) throws ConfigurationException {
final String solrMastersAttr = ((Element)storeNode).attributeValue("solrMasters");
if (StringUtils.isEmpty(solrMastersAttr)) {
throw new ConfigurationException("[" + this.name + "] invalid solrMasters attribute:" + solrMastersAttr);
}
final String[] solrMasterIds = solrMastersAttr.split(",");
if (solrMasterIds.length > 2) {
throw new ConfigurationException("[" + this.name + "] too many solr server configured:" + solrMastersAttr);
}
this.masterSolrConfigs = new SolrConfig[solrMasterIds.length];
for (int i = 0; i < solrMasterIds.length; ++i) {
this.masterSolrConfigs[i] = this.dataStoreConfig.getSolrConfigById(solrMasterIds[i]);
if (null == this.masterSolrConfigs[i]) {
throw new ConfigurationException("[" + this.name + "] unknown solr id: " + solrMasterIds[i]);
}
}
final String solrSlaveAttr = System.getProperty("datastore." + this.name + ".solrSlave", ((Element)storeNode).attributeValue("solrSlave"));
if (StringUtils.isEmpty(solrSlaveAttr)) {
throw new ConfigurationException("[" + this.name + "] invalid solrSlave attribute:" + solrSlaveAttr);
}
this.slaveSolrConfig = this.dataStoreConfig.getSolrConfigById(solrSlaveAttr);
if (null == this.slaveSolrConfig) {
throw new ConfigurationException("[" + this.name + "] unknown solr id: " + solrSlaveAttr);
}
final String solrMasterAttr = System.getProperty("datastore." + this.name + ".solrMaster", ((Element)storeNode).attributeValue("solrMaster"));
if (StringUtils.isEmpty(solrMasterAttr)) {
throw new ConfigurationException("[" + this.name + "] invalid solrMaster attribute:" + solrMasterAttr);
}
this.masterSolrConfig = this.dataStoreConfig.getSolrConfigById(solrMasterAttr);
if (null == this.masterSolrConfig) {
throw new ConfigurationException("[" + this.name + "] unknown solr id: " + solrMasterAttr);
}
if (this.slaveSolrConfig.getShardsConfigs().length != this.masterSolrConfig.getShardsConfigs().length) {
throw new ConfigurationException("[" + this.name + "] master and slave shard count must equals !");
}
this.solrMaxConnectionsPerHost = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxConnectionsPerHost", 300);
if (this.slaveSolrConfig.getShardsConfigs().length > 0) {
this.solrMaxTotalConnections = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxTotalConnections", this.slaveSolrConfig.getShardsConfigs().length * this.solrMaxConnectionsPerHost);
}
else {
this.solrMaxTotalConnections = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxTotalConnections", this.solrMaxConnectionsPerHost);
}
final String mongoId = ((Element)storeNode).attributeValue("mongo");
final MongoConfig mongoConfig = this.dataStoreConfig.getMongoConfigById(mongoId);
if (null == mongoConfig) {
throw new ConfigurationException("[" + this.name + "] unknown mongo id: " + mongoId);
}
this.mongoConfig = mongoConfig;
this.textFieldToGridFS = XmlUtils.getNodeValueAsBoolean(storeNode, "textFieldToGridFS", false);
final Map<String, Field> newFields = new HashMap<String, Field>();
final List<Node> fieldNodes = (List<Node>)storeNode.selectNodes("schema//field");
for (final Node fieldNode : fieldNodes) {
final Element element = (Element)fieldNode;
final String fieldName = element.attributeValue("name");
final Field field = this.getOrCreateField(fieldName);
field.setName(fieldName);
field.setRequired(Boolean.parseBoolean(element.attributeValue("required")));
field.setType(FieldTypeEnum.parse(element.attributeValue("type")));
newFields.put(fieldName, field);
}
this.fields.clear();
this.fields.putAll(newFields);
final Element triggerNode = (Element)storeNode.selectSingleNode("trigger");
if (triggerNode != null) {
this.trigger.setInterface(triggerNode.attributeValue("interface"));
}
}
public Trigger getTrigger() {
return this.trigger;
}
public Field getField(final String fieldName) {
return this.fields.get(fieldName);
}
public boolean containsField(final String fieldName) {
return this.fields.containsKey(fieldName);
}
public boolean isFieldValueLegal(final String fieldName, final Object value) {
if ("_id".equals(fieldName)) {
return value instanceof ObjectId;
}
if (this.fields.containsKey(fieldName)) {
final Field field = this.fields.get(fieldName);
return field.validate(value);
}
return false;
}
public boolean isBigField(final String fieldName) {
return this.fields.get(fieldName).isBigField();
}
public Collection<Field> getFields() {
return Collections.<Field>unmodifiableCollection((Collection<? extends Field>)this.fields.values());
}
synchronized Field getOrCreateField(final String fieldName) {
if (StringUtils.isEmpty(fieldName)) {
throw new IllegalArgumentException("fieldName can not be null or empty!");
}
Field ret = this.fields.get(fieldName);
if (null == ret) {
ret = new Field();
ret.setName(fieldName);
this.fields.put(fieldName, ret);
}
return ret;
}
public String getName() {
return this.name;
}
public MongoConfig getMongoConfig() {
return this.mongoConfig;
}
public DataStoreConfig getDataStoreConfig() {
return this.dataStoreConfig;
}
public boolean isTextFieldToGridFS() {
return this.textFieldToGridFS;
}
public SolrConfig getSlaveSolrConfig() {
return this.slaveSolrConfig;
}
public SolrConfig[] getMasterSolrConfigs() {
return this.masterSolrConfigs;
}
public SolrConfig getMasterSolrConfig() {
return this.masterSolrConfig;
}
public int getSolrMaxTotalConnections() {
return this.solrMaxTotalConnections;
}
public int getSolrMaxConnectionsPerHost() {
return this.solrMaxConnectionsPerHost;
}
@Override
public String toString() {
final StringBuilder ret = new StringBuilder();
ret.append("\n{");
ret.append("\n\tname: ").append(this.name);
ret.append(",\n\tmasterSolrConfig: ").append(this.masterSolrConfig);
ret.append(",\n\tslaveSolrConfig: ").append(this.slaveSolrConfig);
ret.append(",\n\tmongoConfig: ").append(this.mongoConfig);
ret.append(",\n\tsolrMaxTotalConnections: ").append(this.solrMaxTotalConnections);
ret.append(",\n\tsolrMaxConnectionsPerHost: ").append(this.solrMaxConnectionsPerHost);
ret.append(",\n\ttextFieldToGridFS: ").append(this.textFieldToGridFS);
ret.append(",\n\ttrigger: ").append(this.trigger);
ret.append(",\n\tmasterSolrConfigs: [\n");
for (int i = 0; i < this.masterSolrConfigs.length; ++i) {
ret.append('\t').append('\t').append(this.masterSolrConfigs[i]);
if (i < this.masterSolrConfigs.length - 1) {
ret.append(',');
}
ret.append('\n');
}
ret.append("\t]\n}");
return ret.toString();
}
}
@@ -1,49 +1,49 @@
package com.fangjia.datastore.config;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.commons.lang.StringUtils;
public final class Trigger {
private String interface_;
Trigger() {
}
public String getInterface() {
return interface_;
}
void setInterface(String interface_) {
if (StringUtils.isEmpty(interface_) || !interface_.startsWith("http://")) {
throw new IllegalArgumentException("invalid trigger interface: " + interface_);
}
this.interface_ = interface_;
}
@Override
public int hashCode() {
return new HashCodeBuilder().append(this.interface_).toHashCode();
}
@Override
public boolean equals(Object obj) {
if (null == obj) {
return false;
}
if (this == obj) {
return true;
}
if (!(obj instanceof Trigger)) {
return false;
}
Trigger other = (Trigger) obj;
return new EqualsBuilder().append(this.interface_, other.interface_).isEquals();
}
@Override
public String toString() {
return new StringBuilder().append("{interface=").append(interface_).append("}").toString();
}
public final class Trigger
{
private String interface_;
Trigger() {
}
public String getInterface() {
return this.interface_;
}
void setInterface(final String interface_) {
if (StringUtils.isEmpty(interface_) || !interface_.startsWith("http://")) {
throw new IllegalArgumentException("invalid trigger interface: " + interface_);
}
this.interface_ = interface_;
}
@Override
public int hashCode() {
return new HashCodeBuilder().append((Object)this.interface_).toHashCode();
}
@Override
public boolean equals(final Object obj) {
if (null == obj) {
return false;
}
if (this == obj) {
return true;
}
if (!(obj instanceof Trigger)) {
return false;
}
final Trigger other = (Trigger)obj;
return new EqualsBuilder().append((Object)this.interface_, (Object)other.interface_).isEquals();
}
@Override
public String toString() {
return "{interface=" + this.interface_ + "}";
}
}
@@ -1,19 +1,20 @@
package com.fangjia.datastore.handler;
import com.fangjia.datastore.mongo.protocol.MongoMessage;
import com.fangjia.datastore.DataStore;
import com.fangjia.datastore.DataStoreContext;
import com.fangjia.datastore.mongo.protocol.DeleteMessage;
public class DeleteMessageHandler extends MessageHandler<DeleteMessage> {
public DeleteMessageHandler(DataStoreContext dataStoreContext) {
super(dataStoreContext);
}
public void messageReceived(DeleteMessage message,String ip) throws Exception {
assertNotReadOnly();
DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName());
dataStore.remove(message.getSelector());
}
public class DeleteMessageHandler extends MessageHandler<DeleteMessage>
{
public DeleteMessageHandler(final DataStoreContext dataStoreContext) {
super(dataStoreContext);
}
@Override
public void messageReceived(final DeleteMessage message) throws Exception {
this.assertNotReadOnly();
final DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName());
dataStore.remove(message.getSelector());
}
}
@@ -1,54 +1,59 @@
package com.fangjia.datastore.handler;
import java.util.List;
import org.bson.BSONObject;
import com.fangjia.datastore.mongo.protocol.MongoMessage;
import com.mongodb.DBObject;
import com.fangjia.datastore.QueryResponse;
import com.fangjia.datastore.mongo.protocol.QueryMessage;
import com.fangjia.datastore.Cursor;
import com.fangjia.datastore.DataStore;
import com.fangjia.datastore.DataStoreContext;
import com.fangjia.datastore.QueryResponse;
import com.fangjia.datastore.mongo.protocol.GetMoreMessage;
import com.fangjia.datastore.mongo.protocol.QueryMessage;
import com.fangjia.datastore.mongo.protocol.ReplyMessage;
import org.bson.BSONObject;
import java.util.List;
import com.fangjia.datastore.DataStoreContext;
import com.fangjia.datastore.mongo.protocol.GetMoreMessage;
public class GetMoreMessageHandler extends MessageHandler<GetMoreMessage> {
public GetMoreMessageHandler(DataStoreContext dataStoreContext) {
super(dataStoreContext);
}
public void messageReceived(GetMoreMessage message) throws Exception {
String fullCollectionName = message.getFullCollectionName();
if(fullCollectionName.contains("$cmd")){
ReplyMessage reply = prepareReplyMessage(message.getMessageHeader().getRequestID(), (List<? extends BSONObject>) null);
reply.setStartingFrom(0);
this.writeMessage(reply);
this.dataStoreContext.removeCursor(message.getCursorID());
return;
}
DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName());
Cursor cursor = this.dataStoreContext.getCursor(message.getCursorID());
if (null == cursor) {
ReplyMessage reply = prepareReplyMessage(message.getMessageHeader().getRequestID(), (List<? extends BSONObject>) null);
reply.setStartingFrom(0);
this.writeMessage(reply);
} else {
QueryMessage queryMessage = cursor.getQueryMessage();
QueryResponse response = dataStore.find(queryMessage.getRealQuery(), queryMessage.getReturnFieldSelector(),
queryMessage.getOrderByQuery(), queryMessage.getFacetQuery(), cursor.nextSkip(),
message.getNumberToReturn() - 1, queryMessage.isBulkQuery(), queryMessage.isFromMaster());
ReplyMessage reply = prepareReplyMessage(message.getMessageHeader().getRequestID(), response.getDocList());
reply.setStartingFrom(cursor.getSkip());
if (response.hasMore()) {
// cursor.setTotalCount(response.getNumFound());
// cursor.setLimit((int) response.getNumReturned());
// this.dataStoreContext.addCursor(cursor);
// reply.setCursorID(cursor.getId());
} else {
this.dataStoreContext.removeCursor(cursor.getId());
}
this.writeMessage(reply);
}
}
public class GetMoreMessageHandler extends MessageHandler<GetMoreMessage>
{
public GetMoreMessageHandler(final DataStoreContext dataStoreContext) {
super(dataStoreContext);
}
@Override
public void messageReceived(final GetMoreMessage message) throws Exception {
final String fullCollectionName = message.getFullCollectionName();
if (fullCollectionName.contains("$cmd")) {
final ReplyMessage reply = this.prepareReplyMessage(message.getMessageHeader().getRequestID(), (List<? extends BSONObject>)null);
reply.setStartingFrom(0);
this.writeMessage(reply);
this.dataStoreContext.removeCursor(message.getCursorID());
return;
}
final DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName());
final Cursor cursor = this.dataStoreContext.getCursor(message.getCursorID());
if (null == cursor) {
final ReplyMessage reply2 = this.prepareReplyMessage(message.getMessageHeader().getRequestID(), (List<? extends BSONObject>)null);
reply2.setStartingFrom(0);
this.writeMessage(reply2);
}
else {
final QueryMessage queryMessage = cursor.getQueryMessage();
final QueryResponse response = dataStore.find(queryMessage.getRealQuery(), queryMessage.getReturnFieldSelector(), queryMessage.getOrderByQuery(), queryMessage.getFacetQuery(), cursor.nextSkip(), this.useCursor() ? message.getNumberToReturn() : (message.getNumberToReturn() - 1), queryMessage.isBulkQuery(), queryMessage.isFromMaster());
final List<DBObject> doc = response.getDocList();
if (this.useCursor()) {
doc.remove(0);
}
final ReplyMessage reply3 = this.prepareReplyMessage(message.getMessageHeader().getRequestID(), (List<? extends BSONObject>)doc);
reply3.setStartingFrom(cursor.getSkip());
if (response.hasMore() && this.useCursor()) {
cursor.setTotalCount(response.getNumFound());
cursor.setLimit(response.getNumReturned());
this.dataStoreContext.addCursor(cursor);
reply3.setCursorID(cursor.getId());
}
else {
this.dataStoreContext.removeCursor(cursor.getId());
}
this.writeMessage(reply3);
}
}
}
@@ -1,115 +1,117 @@
package com.fangjia.datastore.handler;
import java.util.ArrayList;
import java.util.List;
import com.fangjia.datastore.mongo.protocol.MongoMessage;
import java.util.Iterator;
import org.apache.commons.lang.StringUtils;
import org.bson.BSONObject;
import java.util.ArrayList;
import com.fangjia.datastore.solr.Index;
import java.util.Collection;
import com.fangjia.datastore.DataStore;
import org.bson.BSONObject;
import java.util.List;
import com.fangjia.datastore.DataStoreContext;
import com.fangjia.datastore.mongo.protocol.InsertMessage;
import com.fangjia.datastore.solr.Index;
import com.fangjia.datastore.solr.Index.IndexTypeEnum;
public class InsertMessageHandler extends MessageHandler<InsertMessage> {
public InsertMessageHandler(DataStoreContext dataStoreContext) {
super(dataStoreContext);
}
public void messageReceived(InsertMessage message,String ip) throws Exception {
if (message.getFullCollectionName().endsWith("system.indexes")) {
ensureIndex(message);
} else {
insert(message);
}
}
/**
* db.foo.insert()
*
* @param message
* @throws Exception
*/
private void insert(InsertMessage message) throws Exception {
assertNotReadOnly();
DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName());
dataStore.save(message.getDocuments());
}
/**
* db.foo.ensureIndex()
*
* @param message
* @throws Exception
*/
private void ensureIndex(InsertMessage message) throws Exception {
assertNotReadOnly();
DataStore dataStore = this.dataStoreContext.getDataStore(String.valueOf(message.getDocuments().get(0).get("ns")));
List<Index> indexList = addToIndexList(message.getDocuments());
dataStore.ensureIndex(indexList);
}
private List<Index> addToIndexList(List<BSONObject> docList) {
if (null == docList || docList.isEmpty()) {
return null;
}
List<Index> indexList = new ArrayList<Index>();
for (BSONObject bson : docList) {
BSONObject keys = (BSONObject) bson.get("key");
if (null == keys) {
continue;
}
for (String key : keys.keySet()) {
Object value = keys.get(key);
if (value instanceof String) {
Index index = new Index(key, IndexTypeEnum.parse((String) value));
indexList.add(index);
} else if (value instanceof BSONObject) {
// TODO:
Index index = new Index();
index.setName(key);
for (String k : ((BSONObject) value).keySet()) {
Object v = ((BSONObject) value).get(k);
if (StringUtils.equalsIgnoreCase(k, "type")) {
if (v instanceof String) {
index.setType(IndexTypeEnum.parse((String) v));
}
} else if (StringUtils.equalsIgnoreCase(k, "stored")) {
if (v instanceof Boolean) {
index.setStored((Boolean) v);
}
} else if (StringUtils.equalsIgnoreCase(k, "indexed")) {
if (v instanceof Boolean) {
index.setIndexed((Boolean) v);
}
} else if (StringUtils.equalsIgnoreCase(k, "multiValued")) {
if (v instanceof Boolean) {
index.setMultiValued((Boolean) v);
}
} else if (StringUtils.equalsIgnoreCase(k, "dynamic")) {
if (v instanceof Boolean) {
index.setDynamic((Boolean) v);
}
} else if (StringUtils.equalsIgnoreCase(k, "comments")) {
if (v instanceof String) {
index.setComments((String) v);
}
}
}
if (index.getType() == null) {
throw new IllegalArgumentException("illegal schema field: type");
}
indexList.add(index);
} else {
if (logger.isErrorEnabled()) {
logger.error("invalid key value of ensureIndex:{}", value);
}
}
}
}
return indexList;
}
public class InsertMessageHandler extends MessageHandler<InsertMessage>
{
public InsertMessageHandler(final DataStoreContext dataStoreContext) {
super(dataStoreContext);
}
@Override
public void messageReceived(final InsertMessage message) throws Exception {
if (message.getFullCollectionName().endsWith("system.indexes")) {
this.ensureIndex(message);
}
else {
this.insert(message);
}
}
private void insert(final InsertMessage message) throws Exception {
this.assertNotReadOnly();
final DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName());
dataStore.save(message.getDocuments());
}
private void ensureIndex(final InsertMessage message) throws Exception {
this.assertNotReadOnly();
final DataStore dataStore = this.dataStoreContext.getDataStore(String.valueOf(message.getDocuments().get(0).get("ns")));
final List<Index> indexList = this.addToIndexList(message.getDocuments());
dataStore.ensureIndex(indexList);
}
private List<Index> addToIndexList(final List<BSONObject> docList) {
if (null == docList || docList.isEmpty()) {
return null;
}
final List<Index> indexList = new ArrayList<Index>();
for (final BSONObject bson : docList) {
final BSONObject keys = (BSONObject)bson.get("key");
if (null == keys) {
continue;
}
for (final String key : keys.keySet()) {
final Object value = keys.get(key);
if (value instanceof String) {
final Index index = new Index(key, Index.IndexTypeEnum.parse((String)value));
indexList.add(index);
}
else if (value instanceof BSONObject) {
final Index index = new Index();
index.setName(key);
for (final String k : ((BSONObject)value).keySet()) {
final Object v = ((BSONObject)value).get(k);
if (StringUtils.equalsIgnoreCase(k, "type")) {
if (!(v instanceof String)) {
continue;
}
index.setType(Index.IndexTypeEnum.parse((String)v));
}
else if (StringUtils.equalsIgnoreCase(k, "stored")) {
if (!(v instanceof Boolean)) {
continue;
}
index.setStored((Boolean) v);
}
else if (StringUtils.equalsIgnoreCase(k, "indexed")) {
if (!(v instanceof Boolean)) {
continue;
}
index.setIndexed((Boolean)v);
}
else if (StringUtils.equalsIgnoreCase(k, "multiValued")) {
if (!(v instanceof Boolean)) {
continue;
}
index.setMultiValued((Boolean)v);
}
else if (StringUtils.equalsIgnoreCase(k, "dynamic")) {
if (!(v instanceof Boolean)) {
continue;
}
index.setDynamic((Boolean)v);
}
else {
if (!StringUtils.equalsIgnoreCase(k, "comments") || !(v instanceof String)) {
continue;
}
index.setComments((String)v);
}
}
if (index.getType() == null) {
throw new IllegalArgumentException("illegal schema field: type");
}
indexList.add(index);
}
else {
if (!this.logger.isErrorEnabled()) {
continue;
}
this.logger.error("invalid key value of ensureIndex:{}", value);
}
}
}
return indexList;
}
}
@@ -1,23 +1,24 @@
package com.fangjia.datastore.handler;
import com.fangjia.datastore.mongo.protocol.MongoMessage;
import java.util.Iterator;
import java.util.List;
import com.fangjia.datastore.DataStoreContext;
import com.fangjia.datastore.mongo.protocol.KillCursorsMessage;
public class KillCursorsMessageHandler extends MessageHandler<KillCursorsMessage> {
public KillCursorsMessageHandler(DataStoreContext dataStoreContext) {
super(dataStoreContext);
}
public void messageReceived(KillCursorsMessage message,String ip) {
List<Long> cursorIds = message.getCursorIDs();
if (cursorIds != null) {
for (Long cursorId : cursorIds) {
this.dataStoreContext.removeCursor(cursorId);
}
}
}
public class KillCursorsMessageHandler extends MessageHandler<KillCursorsMessage>
{
public KillCursorsMessageHandler(final DataStoreContext dataStoreContext) {
super(dataStoreContext);
}
@Override
public void messageReceived(final KillCursorsMessage message) {
final List<Long> cursorIds = message.getCursorIDs();
if (cursorIds != null) {
for (final Long cursorId : cursorIds) {
this.dataStoreContext.removeCursor(cursorId);
}
}
}
}
@@ -1,184 +1,180 @@
package com.fangjia.datastore.handler;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.mina.core.session.IoSession;
import org.bson.BSONObject;
import org.bson.BasicBSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fangjia.datastore.DataStoreContext;
import com.fangjia.datastore.DataStoreException;
import com.fangjia.datastore.mongo.protocol.MongoMessage;
import java.util.Arrays;
import org.bson.BasicBSONObject;
import java.util.ArrayList;
import org.bson.BSONObject;
import java.util.List;
import com.fangjia.datastore.mongo.protocol.ReplyMessage;
import java.net.InetSocketAddress;
import org.apache.mina.core.session.IoSession;
import org.slf4j.LoggerFactory;
import org.slf4j.Logger;
import com.fangjia.datastore.DataStoreContext;
import java.util.Map;
import com.fangjia.datastore.mongo.protocol.MongoMessage;
public abstract class MessageHandler<T extends MongoMessage> {
private final static String LAST_ERROR_KEY = "com.goojia.datastore.LAST_ERROR_KEY";
private final static String PREV_ERROR_KEY = "com.goojia.datastore.PREV_ERROR_KEY";
private final static String SESSION_KEY = "com.goojia.datastore.SESSION_KEY";
private final static String ERROR_OCCURRED_KEY = "com.goojia.datastore.ERROR_OCCURRED_KEY";
private final static ThreadLocal<Map<String, Object>> THREAD_LOCAL = new ThreadLocal<Map<String, Object>>() {
@Override
protected Map<String, Object> initialValue() {
return new HashMap<String, Object>();
}
};
protected final DataStoreContext dataStoreContext;
protected final Logger logger = LoggerFactory.getLogger(this.getClass());
public MessageHandler(DataStoreContext dataStoreContext) {
this.dataStoreContext = dataStoreContext;
}
/**
* ´¦ÀíÌØ¶¨µÄÏûÏ¢
*
* @param message
*/
public abstract void messageReceived(T message,String ip) throws Exception;
@SuppressWarnings("unchecked")
public void messageReceived(IoSession session, MongoMessage message) throws Exception {
System.out.println(session.getRemoteAddress());
try {
setSession(session);
messageReceived((T) message,session.getRemoteAddress().toString());
if (!isErrorOccured()) {
this.clearLastError();
}
} catch (Throwable e) {
String hostAddress = "";
if(session.getRemoteAddress() instanceof InetSocketAddress) {
hostAddress = ((InetSocketAddress)session.getRemoteAddress()).getAddress().getHostAddress();
}
this.setLastError(buildExceptionMessage(e));
logger.error("error processing host:" + hostAddress + ",message: " + message, e);
} finally {
clearErrorOccured();
}
}
private void clearErrorOccured() {
THREAD_LOCAL.get().remove(ERROR_OCCURRED_KEY);
}
private void setErrorOccured() {
THREAD_LOCAL.get().put(ERROR_OCCURRED_KEY, true);
}
private boolean isErrorOccured() {
Boolean b = (Boolean) THREAD_LOCAL.get().get(ERROR_OCCURRED_KEY);
return null == b ? false : b;
}
protected IoSession getSession() {
return (IoSession) THREAD_LOCAL.get().get(SESSION_KEY);
}
private void setSession(IoSession session) {
THREAD_LOCAL.get().put(SESSION_KEY, session);
}
protected void setLastError(Object error) {
if (null != error) {
IoSession session = getSession();
Object lastError = session.getAttribute(LAST_ERROR_KEY);
if (null != lastError) {
session.setAttribute(PREV_ERROR_KEY, lastError);
}
setErrorOccured();
session.setAttribute(LAST_ERROR_KEY, String.valueOf(error));
}
}
private void clearLastError() {
getSession().removeAttribute(LAST_ERROR_KEY);
}
protected Object getLastError() {
return getSession().getAttribute(LAST_ERROR_KEY);
}
protected Object getPrevError() {
return getSession().getAttribute(PREV_ERROR_KEY);
}
protected void writeMessage(ReplyMessage message) {
getSession().write(message);
}
protected void writeAsReplyMessage(int responseTo, List<? extends BSONObject> docs) {
writeMessage(prepareReplyMessage(responseTo, docs));
}
@SuppressWarnings("unchecked")
protected ReplyMessage prepareReplyMessage(int responseTo, List<? extends BSONObject> docs) {
ReplyMessage reply = new ReplyMessage();
reply.setNumberReturned(null == docs ? 0 : docs.size());
reply.setResponseFlag(8);
reply.getMessageHeader().setResponseTo(responseTo);
if (null != docs && docs.size() > 0) {
reply.setDocuments((List<BSONObject>) docs);
}
return reply;
}
protected void writeAsReplyMessage(int responseTo, BSONObject... docs) {
writeMessage(prepareReplyMessage(responseTo, docs));
}
protected ReplyMessage prepareReplyMessage(int responseTo, BSONObject... docs) {
if (null != docs && docs.length > 0) {
List<BSONObject> docList = new ArrayList<BSONObject>(docs.length);
for (BSONObject doc : docs) {
docList.add(doc);
}
return prepareReplyMessage(responseTo, docList);
} else {
return prepareReplyMessage(responseTo, (List<? extends BSONObject>) null);
}
}
protected void writeAsErrorMessage(int responseTo, Object errorMessage) {
writeMessage(prepareErrorMessage(responseTo, errorMessage));
}
protected ReplyMessage prepareErrorMessage(int responseTo, Object errorMessage) {
ReplyMessage reply = new ReplyMessage();
reply.setNumberReturned(1);
reply.setResponseFlag(8);
reply.getMessageHeader().setResponseTo(responseTo);
BSONObject msg = new BasicBSONObject("$err", errorMessage == null ? null : errorMessage.toString());
msg.put("err", errorMessage == null ? null : errorMessage.toString());
msg.put("errmsg", errorMessage == null ? null : errorMessage.toString());
msg.put("ok", 0.0);
reply.setDocuments(Arrays.asList(msg));
return reply;
}
protected String buildExceptionMessage(Throwable e) {
StringBuilder ret = new StringBuilder();
ret.append(e.getClass().getName()).append(": ").append(e.getMessage());
Throwable cause = e;
while ((cause = cause.getCause()) != null) {
ret.append(", caused by ").append(cause.getClass().getName()).append(": ").append(cause.getMessage());
}
return ret.toString();
}
protected void assertNotReadOnly() throws DataStoreException {
if (this.dataStoreContext.getDataStoreConfig().isReadOnly()) {
throw new DataStoreException("read only server !");
}
}
public abstract class MessageHandler<T extends MongoMessage>
{
private static final String LAST_ERROR_KEY = "com.goojia.datastore.LAST_ERROR_KEY";
private static final String PREV_ERROR_KEY = "com.goojia.datastore.PREV_ERROR_KEY";
private static final String SESSION_KEY = "com.goojia.datastore.SESSION_KEY";
private static final String ERROR_OCCURRED_KEY = "com.goojia.datastore.ERROR_OCCURRED_KEY";
private static final ThreadLocal<Map<String, Object>> THREAD_LOCAL;
protected final DataStoreContext dataStoreContext;
protected final Logger logger = LoggerFactory.getLogger(this.getClass());
public MessageHandler(final DataStoreContext dataStoreContext) {
this.dataStoreContext = dataStoreContext;
}
public abstract void messageReceived(T message) throws Exception;
public void messageReceived(final IoSession session, final MongoMessage message) throws Exception {
try {
this.setSession(session);
this.messageReceived((T)message);
if (!this.isErrorOccured()) {
this.clearLastError();
}
}
catch (Throwable e) {
String hostAddress = "";
if (session.getRemoteAddress() instanceof InetSocketAddress) {
hostAddress = ((InetSocketAddress)session.getRemoteAddress()).getAddress().getHostAddress();
}
this.setLastError(this.buildExceptionMessage(e));
this.logger.error("error processing host:" + hostAddress + ",message: " + message, e);
}
finally {
this.clearErrorOccured();
}
}
private void clearErrorOccured() {
MessageHandler.THREAD_LOCAL.get().remove("com.goojia.datastore.ERROR_OCCURRED_KEY");
}
private void setErrorOccured() {
MessageHandler.THREAD_LOCAL.get().put("com.goojia.datastore.ERROR_OCCURRED_KEY", true);
}
private boolean isErrorOccured() {
final Boolean b = (Boolean) MessageHandler.THREAD_LOCAL.get().get("com.goojia.datastore.ERROR_OCCURRED_KEY");
return null != b && b;
}
protected IoSession getSession() {
return (IoSession)MessageHandler.THREAD_LOCAL.get().get("com.goojia.datastore.SESSION_KEY");
}
private void setSession(final IoSession session) {
MessageHandler.THREAD_LOCAL.get().put("com.goojia.datastore.SESSION_KEY", session);
}
protected void setLastError(final Object error) {
if (null != error) {
final IoSession session = this.getSession();
final Object lastError = session.getAttribute((Object)"com.goojia.datastore.LAST_ERROR_KEY");
if (null != lastError) {
session.setAttribute((Object)"com.goojia.datastore.PREV_ERROR_KEY", lastError);
}
this.setErrorOccured();
session.setAttribute((Object)"com.goojia.datastore.LAST_ERROR_KEY", (Object)String.valueOf(error));
}
}
private void clearLastError() {
this.getSession().removeAttribute((Object)"com.goojia.datastore.LAST_ERROR_KEY");
}
protected Object getLastError() {
return this.getSession().getAttribute((Object)"com.goojia.datastore.LAST_ERROR_KEY");
}
protected Object getPrevError() {
return this.getSession().getAttribute((Object)"com.goojia.datastore.PREV_ERROR_KEY");
}
protected void writeMessage(final ReplyMessage message) {
this.getSession().write((Object)message);
}
protected void writeAsReplyMessage(final int responseTo, final List<? extends BSONObject> docs) {
this.writeMessage(this.prepareReplyMessage(responseTo, docs));
}
protected ReplyMessage prepareReplyMessage(final int responseTo, final List<? extends BSONObject> docs) {
final ReplyMessage reply = new ReplyMessage();
reply.setNumberReturned((null == docs) ? 0 : docs.size());
reply.setResponseFlag(8);
reply.getMessageHeader().setResponseTo(responseTo);
if (null != docs && docs.size() > 0) {
reply.setDocuments((List<BSONObject>)docs);
}
return reply;
}
protected void writeAsReplyMessage(final int responseTo, final BSONObject... docs) {
this.writeMessage(this.prepareReplyMessage(responseTo, docs));
}
protected ReplyMessage prepareReplyMessage(final int responseTo, final BSONObject... docs) {
if (null != docs && docs.length > 0) {
final List<BSONObject> docList = new ArrayList<BSONObject>(docs.length);
for (final BSONObject doc : docs) {
docList.add(doc);
}
return this.prepareReplyMessage(responseTo, docList);
}
return this.prepareReplyMessage(responseTo, (List<? extends BSONObject>)null);
}
protected void writeAsErrorMessage(final int responseTo, final Object errorMessage) {
this.writeMessage(this.prepareErrorMessage(responseTo, errorMessage));
}
protected ReplyMessage prepareErrorMessage(final int responseTo, final Object errorMessage) {
final ReplyMessage reply = new ReplyMessage();
reply.setNumberReturned(1);
reply.setResponseFlag(8);
reply.getMessageHeader().setResponseTo(responseTo);
final BSONObject msg = (BSONObject)new BasicBSONObject("$err", (Object)((errorMessage == null) ? null : errorMessage.toString()));
msg.put("err", (Object)((errorMessage == null) ? null : errorMessage.toString()));
msg.put("errmsg", (Object)((errorMessage == null) ? null : errorMessage.toString()));
msg.put("ok", (Object)0.0);
reply.setDocuments(Arrays.<BSONObject>asList(msg));
return reply;
}
protected String buildExceptionMessage(final Throwable e) {
final StringBuilder ret = new StringBuilder();
ret.append(e.getClass().getName()).append(": ").append(e.getMessage());
Throwable cause = e;
while ((cause = cause.getCause()) != null) {
ret.append(", caused by ").append(cause.getClass().getName()).append(": ").append(cause.getMessage());
}
return ret.toString();
}
protected void assertNotReadOnly() throws DataStoreException {
if (this.dataStoreContext.getDataStoreConfig().isReadOnly()) {
throw new DataStoreException("read only server !");
}
}
protected boolean useCursor() {
return (Boolean) this.getSession().getAttribute((Object)"useCursor", false);
}
static {
THREAD_LOCAL = new ThreadLocal<Map<String, Object>>() {
@Override
protected Map<String, Object> initialValue() {
return new HashMap<String, Object>();
}
};
}
}
@@ -1,490 +1,501 @@
package com.fangjia.datastore.handler;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
import com.fangjia.datastore.mongo.protocol.MongoMessage;
import org.apache.commons.lang.StringUtils;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.mongodb.BasicDBList;
import org.apache.mina.core.session.IoSession;
import org.bson.BSONObject;
import org.bson.BasicBSONObject;
import com.fangjia.datastore.ConfigurationException;
import com.fangjia.datastore.DataStore;
import com.fangjia.datastore.DataStoreContext;
import com.fangjia.datastore.DataStoreException;
import com.fangjia.datastore.QueryResponse;
import com.fangjia.datastore.config.Field;
import com.fangjia.datastore.config.StoreConfig;
import com.fangjia.datastore.mongo.protocol.QueryMessage;
import com.fangjia.datastore.config.Field;
import com.fangjia.datastore.mongo.protocol.ReplyMessage;
import com.fangjia.datastore.QueryResponse;
import com.fangjia.datastore.Cursor;
import java.util.Collection;
import com.fangjia.datastore.solr.Index;
import com.fangjia.datastore.utils.MongoUtils;
import java.util.Iterator;
import java.util.Set;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
import java.util.ArrayList;
import com.fangjia.datastore.ConfigurationException;
import com.mongodb.BasicDBList;
import java.util.List;
import com.fangjia.datastore.DataStore;
import org.bson.types.BasicBSONList;
import org.bson.BasicBSONObject;
import org.bson.BSONObject;
import com.fangjia.datastore.DataStoreException;
import org.apache.mina.core.session.IoSession;
import java.net.InetSocketAddress;
import com.fangjia.datastore.utils.MongoUtils;
import com.fangjia.datastore.DataStoreContext;
import com.fangjia.datastore.mongo.protocol.QueryMessage;
public class QueryMessageHandler extends MessageHandler<QueryMessage> {
public QueryMessageHandler(DataStoreContext dataStoreContext) {
super(dataStoreContext);
}
public void messageReceived(QueryMessage message,String ip) throws Exception {
Object error = null;
try {
String[] nameSplit = MongoUtils.parseFullCollectionName(message.getFullCollectionName());
if (null == nameSplit) {
error = "invalid full collection name: " + message.getFullCollectionName();
} else {
doMessageReceived(message, nameSplit[0], nameSplit[1]);
}
} catch (Throwable e) {
IoSession session = getSession();
String hostAddress = "";
if(session != null && session.getRemoteAddress() instanceof InetSocketAddress) {
hostAddress = ((InetSocketAddress)session.getRemoteAddress()).getAddress().getHostAddress();
}
error = buildExceptionMessage(e);
this.writeAsErrorMessage(message.getMessageHeader().getRequestID(), error);
if (logger.isErrorEnabled()) {
logger.error("error process host:" + hostAddress + ",message: " + message, e);
}
} finally {
this.setLastError(error);
}
}
private void doMessageReceived(QueryMessage message, String dbName, String collectionName) throws DataStoreException {
if (collectionName.equals("$cmd")) {
onCommand(message, dbName);
return;
}
if (collectionName.equals("system.namespaces")) {
showCollections(message, dbName, collectionName);
} else if (collectionName.equals("system.indexes")) {
getIndexes(message, dbName, collectionName);
} else {
find(message, dbName, collectionName);
}
}
private void onCommand(QueryMessage message, String dbName) throws DataStoreException {
if (dbName.equals("admin")) {
onAdminCommand(message);
return;
}
BSONObject query = message.getQuery();
if (query.containsField("getlasterror")) {
getLastError(message, dbName);
} else if (query.containsField("getpreverror")) {
getPrevError(message, dbName);
} else if (query.containsField("count")) {
count(message, dbName);
} else if (query.containsField("getSchema")) {
getSchema(message, dbName);
} else if (query.containsField("reIndex")) {
reIndex(message, dbName);
} else if (query.containsField("reload")) {
reload(message, dbName);
} else if(query.containsField("dbStats")){
dbStats(message, dbName);
} else if(query.containsField("listCollections")){
listCollections(message, dbName);
} else if (query.containsField("isMaster") || query.containsField("ismaster")) {
isMaster(message);
} else if(query.containsField("buildinfo") || query.containsField("buildInfo")){
buildinfo(message);
} else if(query.containsField("serverStatus")){
serverStatus(message);
} else if(query.containsField("connectionStatus")){
connectionStatus(message);
} else if (query.containsField("find")) {
find(message, dbName, query.get("find").toString() );
} else {
delegateCommand(message, dbName);
}
}
private void onAdminCommand(QueryMessage message) throws DataStoreException {
BSONObject query = message.getQuery();
if (query.containsField("whatsmyuri")) {
whatsMyUri(message);
} else if (query.containsField("listDatabases")) {
listDatabases(message);
} else if (query.containsField("nonce")) {
nonce(message);
} else if (query.containsField("isMaster") || query.containsField("ismaster")) {
isMaster(message);
} else if(query.containsField("buildinfo") || query.containsField("buildInfo")){
buildinfo(message);
} else if (query.containsField("replSetGetStatus")) {
replSetGetStatus(message);
} else if(query.containsField("getlasterror")){
getLastError(message, "admin");
} else if(query.containsField("serverStatus")){
serverStatus(message);
} else if(query.containsField("connectionStatus")){
connectionStatus(message);
} else if (query.containsField("getLog")) {
getLog(message);
} else {
delegateAdminCommand(message);
}
}
private void replSetGetStatus(QueryMessage message) {
BSONObject doc = new BasicBSONObject();
doc.put("ok", 0);
doc.put("errmsg", "not running with --replSet");
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void delegateCommand(QueryMessage message, String dbName) throws DataStoreException {
throw new DataStoreException("unsupported command:" + message.getQuery());
}
private void getLog(QueryMessage message){
BasicDBList names = new BasicDBList();
names.add("global");
names.add("startupWarnings");
BSONObject doc = new BasicBSONObject();
doc.put("ok", 1);
doc.put("names", names);
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void dbStats(QueryMessage message, String dbName) throws DataStoreException {
BSONObject doc = new BasicBSONObject();
doc.put("db", dbName);
doc.put("collections", this.dataStoreContext.getDataStoreConfig().getStoreNames());
doc.put("ok", 1.0);
InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress();
doc.put("primary", localAddress.getAddress().getHostAddress());
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void delegateAdminCommand(QueryMessage message) throws DataStoreException {
throw new DataStoreException("unsupported command:" + message.getQuery());
}
private void reload(QueryMessage message, String dbName) throws DataStoreException {
assertNotReadOnly();
try {
this.dataStoreContext.reload();
this.dataStoreContext.getDataStore(dbName + "." + message.getQuery().get("reload")).reload();
BSONObject doc = new BasicBSONObject();
doc.put("ok", 1.0);
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
} catch (ConfigurationException e) {
throw new DataStoreException("error reload config: ", e);
}
}
/**
* show collections
*
* @param message
*/
private void showCollections(QueryMessage message, String dbName, String collectionName) throws DataStoreException {
Set<String> storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames();
List<DBObject> docList = new ArrayList<DBObject>(storeNames.size());
for (String sn : storeNames) {
if (sn.startsWith(dbName + ".")) {
docList.add(new BasicDBObject("name", sn));
}
}
writeAsReplyMessage(message.getMessageHeader().getRequestID(), docList);
}
/**
* db.foo.getIndexes()
*
* @param message
* @param dbName
* @param collectionName
* @throwstion
*/
private void getIndexes(QueryMessage message, String dbName, String collectionName) throws DataStoreException {
String ns = (String) message.getQuery().get("ns");
Collection<Index> indexes = this.dataStoreContext.getDataStore(ns).getIndexes();
List<BSONObject> docList = new ArrayList<BSONObject>(indexes.size());
for (Index i : indexes) {
docList.add(i.toBSONObject());
}
writeAsReplyMessage(message.getMessageHeader().getRequestID(), docList);
}
/**
* db.foo.find()
*
* @param message
* @param dbName
* @param collectionName
* @throws DataStoreException
*/
private void find(QueryMessage message, String dbName, String collectionName) throws DataStoreException {
String fullCollectionName;
if(message.getFullCollectionName().contains("$cmd")){
fullCollectionName = dbName + "." + collectionName;
}else{
fullCollectionName = message.getFullCollectionName();
}
DataStore dataStore = this.dataStoreContext.getDataStore(fullCollectionName);
QueryResponse response = dataStore.find(message.getRealQuery(), message.getReturnFieldSelector(), message
.getOrderByQuery(), message.getFacetQuery(), message.getNumberToSkip(), message.getNumberToReturn() - 1, message
.isBulkQuery(), message.isFromMaster());
ReplyMessage reply = prepareReplyMessage(message.getMessageHeader().getRequestID(), response.getDocList());
reply.setStartingFrom(message.getNumberToSkip());
// if (response.hasMore()) {
// Cursor cursor = new Cursor((int) message.getNumberToSkip(), (int)
// response.getNumReturned(), message,
// response.getNumFound());
// this.dataStoreContext.addCursor(cursor);
// reply.setCursorID(cursor.getId());
// }
this.writeMessage(reply);
}
/**
* db.getLastError()
*
* @param message
* @param dbName
*/
private void getLastError(QueryMessage message, String dbName) {
Object lastError = this.getLastError();
BSONObject doc = new BasicBSONObject();
doc.put("ok", 1.0);
doc.put("n", 1);
doc.put("$err", lastError == null ? null : lastError.toString());
doc.put("err", lastError == null ? null : lastError.toString());
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
/**
* db.getPrevError()
*
* @param message
* @param dbName
*/
private void getPrevError(QueryMessage message, String dbName) {
Object prevError = this.getPrevError();
BSONObject doc = new BasicBSONObject();
doc.put("ok", 1.0);
doc.put("n", 0);
doc.put("$err", prevError == null ? null : prevError.toString());
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
/**
* db.foo.count()
*
* @param message
* @param dbName
* @throws DataStoreException
*/
private void count(QueryMessage message, String dbName) throws DataStoreException {
BSONObject query = message.getQuery();
int count = this.dataStoreContext.getDataStore(dbName + "." + query.get("count")).count(message.getRealQuery(),
message.isFromMaster());
BSONObject doc = new BasicBSONObject();
doc.put("ok", 1.0);
doc.put("n", (double) count);
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
/**
* db.foo.rumCommnad('getSchema')
*
* @param message
* @param dbName
*/
private void getSchema(QueryMessage message, String dbName) throws DataStoreException {
BSONObject query = message.getQuery();
String collectionName = (String) query.get("getSchema");
StoreConfig storeConfig = this.dataStoreContext.getDataStore(dbName + "." + collectionName).getStoreConfig();
Collection<Field> fields = storeConfig.getFields();
List<DBObject> docList = new ArrayList<DBObject>(fields.size());
for (Field field : fields) {
BasicDBObject f = new BasicDBObject();
f.put("name", field.getName());
f.put("type", field.getType().getType());
docList.add(f);
}
BSONObject doc = new BasicBSONObject();
doc.put("ok", 1.0);
doc.put("schema", docList);
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
/**
* <p>
* 不带参数重建:db.foo.reIndex() 或 db.foo.runCommand('reIndex')
* </P>
*
* <p>
* 带参数重建:db.foo.runCommand('reIndex',{query:{cat:'sell',city:'上海'}})
* <p>
*
* @param message
* @param dbName
* @throws Exception
*/
private void reIndex(QueryMessage message, String dbName) throws DataStoreException {
assertNotReadOnly();
BSONObject query = message.getQuery();
String collectionName = (String) query.get("reIndex");
DataStore dataStore = this.dataStoreContext.getDataStore(dbName + "." + collectionName);
// 新增查询条件 add by yukai 2011/07/14
BSONObject doc = new BasicBSONObject();
Object realQuery = query.get("query");
if (null == realQuery)
doc.put("version", dataStore.reIndex(null));
else if (realQuery instanceof BSONObject)
dataStore.reIndex((BSONObject) realQuery);
else
throw new DataStoreException("invalid re-index query: " + realQuery);
doc.put("ok", 1.0);
doc.put("n", 0D);
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
/**
* db.runCommand('whatsmyuri')
*
* @param message
*/
private void whatsMyUri(QueryMessage message) {
BSONObject doc = new BasicBSONObject();
InetSocketAddress address = (InetSocketAddress) getSession().getRemoteAddress();
doc.put("you", address.getAddress().getHostAddress() + ":" + address.getPort());
doc.put("ok", 1.0);
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
/**
* show dbs
*
* @param message
*/
private void listDatabases(QueryMessage message) {
BSONObject doc = new BasicBSONObject();
doc.put("ok", 1.0);
Set<String> storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames();
Set<String> dbNames = new HashSet<String>();
for (String sn : storeNames) {
int i = sn.indexOf('.');
if (i > 0) {
String name = sn.substring(0, i);
dbNames.add(name);
}
}
List<DBObject> docs = new ArrayList<DBObject>(dbNames.size());
for (String dn : dbNames) {
docs.add(new BasicDBObject("name", dn));
}
doc.put("databases", docs);
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
/**
* show dbs
*
* @param message
*/
private void listCollections(QueryMessage message, String dbName) {
System.out.println(message.getMessageHeader().getRequestID());
BSONObject doc = new BasicBSONObject();
doc.put("ok", 1.0);
BSONObject cursor = new BasicDBObject();
doc.put("cursor", cursor);
cursor.put("id", (long)message.getMessageHeader().getRequestID());
cursor.put("ns", dbName + ".$cmd.listCollections");
BasicDBList firstBatch = new BasicDBList();
cursor.put("firstBatch", firstBatch);
Set<String> storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames();
for (String sn : storeNames) {
if (sn.startsWith(dbName + ".")) {
BSONObject collection = new BasicDBObject();
collection.put("name", sn.replace(dbName + ".", ""));
collection.put("options", new BasicDBObject());
firstBatch.add(collection);
}
}
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
/**
* db.runCommand('nonce')
*
* @param message
*/
private void nonce(QueryMessage message) {
BSONObject doc = new BasicBSONObject();
doc.put("ok", 1.0);
doc.put("errmsg", "no such cmd");
doc.put("bad cmd", new BasicBSONObject("nonce", true));
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void isMaster(QueryMessage message) {
BSONObject doc = new BasicBSONObject();
doc.put("ok", 1.0);
doc.put("ismaster", true);
doc.put("secondary", false);
InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress();
doc.put("primary", localAddress.getAddress().getHostAddress());
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void buildinfo(QueryMessage message) {
BasicDBList versionArray = new BasicDBList();
versionArray.add(3);
versionArray.add(2);
versionArray.add(9);
versionArray.add(0);
BSONObject doc = new BasicBSONObject();
doc.put("ok", 1.0);
doc.put("bits", 64.0);
doc.put("debug", false);
doc.put("version" ,"3.2.9");
doc.put("versionArray" ,versionArray);
doc.put("javascriptEngine", "V8");
InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress();
doc.put("primary", localAddress.getAddress().getHostAddress());
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void serverStatus(QueryMessage message){
BSONObject doc = new BasicBSONObject();
doc.put("ok", 1.0);
InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress();
doc.put("primary", localAddress.getAddress().getHostAddress());
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void connectionStatus(QueryMessage message){
BSONObject doc = new BasicBSONObject();
doc.put("ok", 1.0);
InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress();
doc.put("primary", localAddress.getAddress().getHostAddress());
writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
public class QueryMessageHandler extends MessageHandler<QueryMessage>
{
public QueryMessageHandler(final DataStoreContext dataStoreContext) {
super(dataStoreContext);
}
@Override
public void messageReceived(final QueryMessage message) throws Exception {
Object error = null;
try {
final String[] nameSplit = MongoUtils.parseFullCollectionName(message.getFullCollectionName());
if (null == nameSplit) {
error = "invalid full collection name: " + message.getFullCollectionName();
}
else {
this.doMessageReceived(message, nameSplit[0], nameSplit[1]);
}
}
catch (Throwable e) {
final IoSession session = this.getSession();
String hostAddress = "";
if (session != null && session.getRemoteAddress() instanceof InetSocketAddress) {
hostAddress = ((InetSocketAddress)session.getRemoteAddress()).getAddress().getHostAddress();
}
error = this.buildExceptionMessage(e);
this.writeAsErrorMessage(message.getMessageHeader().getRequestID(), error);
if (this.logger.isErrorEnabled()) {
this.logger.error("error process host:" + hostAddress + ",message: " + message, e);
}
}
finally {
this.setLastError(error);
}
}
private void doMessageReceived(final QueryMessage message, final String dbName, final String collectionName) throws DataStoreException {
if (collectionName.equals("$cmd")) {
this.onCommand(message, dbName);
return;
}
if (collectionName.equals("system.namespaces")) {
this.showCollections(message, dbName, collectionName);
}
else if (collectionName.equals("system.indexes")) {
this.getIndexes(message, dbName, collectionName);
}
else {
this.find(message, dbName, collectionName);
}
}
private void onCommand(final QueryMessage message, final String dbName) throws DataStoreException {
if (dbName.equals("admin")) {
this.onAdminCommand(message);
return;
}
final BSONObject query = message.getQuery();
if (query.containsField("getlasterror")) {
this.getLastError(message, dbName);
}
else if (query.containsField("getpreverror")) {
this.getPrevError(message, dbName);
}
else if (query.containsField("count")) {
this.count(message, dbName);
}
else if (query.containsField("getSchema")) {
this.getSchema(message, dbName);
}
else if (query.containsField("reIndex")) {
this.reIndex(message, dbName);
}
else if (query.containsField("reload")) {
this.reload(message, dbName);
}
else if (query.containsField("dbStats")) {
this.dbStats(message, dbName);
}
else if (query.containsField("listCollections")) {
this.listCollections(message, dbName);
}
else if (query.containsField("isMaster") || query.containsField("ismaster")) {
this.isMaster(message);
}
else if (query.containsField("buildinfo") || query.containsField("buildInfo")) {
this.buildinfo(message);
}
else if (query.containsField("serverStatus")) {
this.serverStatus(message);
}
else if (query.containsField("connectionStatus")) {
this.connectionStatus(message);
}
else if (query.containsField("update")) {
this.update(message, dbName);
}
else if (query.containsField("insert")) {
this.insert(message, dbName);
}
else if (query.containsField("ping")) {
this.ping(message);
}
else if (query.containsField("find")) {
this.find(message, dbName, query.get("find").toString());
}
else {
this.delegateCommand(message, dbName);
}
}
private void onAdminCommand(final QueryMessage message) throws DataStoreException {
final BSONObject query = message.getQuery();
if (query.containsField("whatsmyuri")) {
this.whatsMyUri(message);
}
else if (query.containsField("listDatabases")) {
this.listDatabases(message);
}
else if (query.containsField("nonce")) {
this.nonce(message);
}
else if (query.containsField("isMaster") || query.containsField("ismaster")) {
this.isMaster(message);
}
else if (query.containsField("buildinfo") || query.containsField("buildInfo")) {
this.buildinfo(message);
}
else if (query.containsField("replSetGetStatus")) {
this.replSetGetStatus(message);
}
else if (query.containsField("getlasterror")) {
this.getLastError(message, "admin");
}
else if (query.containsField("serverStatus")) {
this.serverStatus(message);
}
else if (query.containsField("connectionStatus")) {
this.connectionStatus(message);
}
else if (query.containsField("collstats")) {
this.collstats(message);
}
else if (query.containsField("getLog")) {
this.getLog(message);
}
else if (query.containsField("ping")) {
this.ping(message);
}
else {
this.delegateAdminCommand(message);
}
}
private void replSetGetStatus(final QueryMessage message) {
final BSONObject doc = (BSONObject)new BasicBSONObject();
doc.put("ok", (Object)0);
doc.put("errmsg", (Object)"not running with --replSet");
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void delegateCommand(final QueryMessage message, final String dbName) throws DataStoreException {
throw new DataStoreException("unsupported command:" + message.getQuery());
}
private void collstats(final QueryMessage message) {
final BSONObject doc = (BSONObject)new BasicBSONObject();
doc.put("ok", (Object)1);
doc.put("ns", (Object)message.getFullCollectionName());
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void ping(final QueryMessage message) throws DataStoreException {
final BSONObject doc = (BSONObject)new BasicBSONObject();
doc.put("ok", (Object)1);
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void update(final QueryMessage message, final String dbName) throws DataStoreException {
final DataStore dataStore = this.dataStoreContext.getDataStore(dbName + "." + message.getQuery().get("update"));
final BasicBSONList updates = (BasicBSONList)message.getQuery().get("updates");
final BSONObject selector = (BSONObject)((BasicBSONObject)updates.get(0)).get("q");
final BSONObject update = (BSONObject)((BasicBSONObject)updates.get(0)).get("u");
final int count = dataStore.update(selector, update);
final BSONObject doc = (BSONObject)new BasicBSONObject();
doc.put("nModified", (Object)count);
doc.put("n", (Object)count);
doc.put("ok", (Object)1);
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void insert(final QueryMessage message, final String dbName) throws DataStoreException {
final DataStore dataStore = this.dataStoreContext.getDataStore(dbName + "." + message.getQuery().get("insert"));
final List documents = (List)message.getQuery().get("documents");
dataStore.save(documents);
final BSONObject doc = (BSONObject)new BasicBSONObject();
doc.put("n", (Object)documents.size());
doc.put("ok", (Object)1);
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void getLog(final QueryMessage message) {
final BasicDBList names = new BasicDBList();
names.add((Object)"global");
names.add((Object)"startupWarnings");
final BSONObject doc = (BSONObject)new BasicBSONObject();
doc.put("ok", (Object)1);
doc.put("names", (Object)names);
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void dbStats(final QueryMessage message, final String dbName) throws DataStoreException {
final BSONObject doc = (BSONObject)new BasicBSONObject();
doc.put("db", (Object)dbName);
doc.put("collections", (Object)this.dataStoreContext.getDataStoreConfig().getStoreNames());
doc.put("ok", (Object)1.0);
final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress();
doc.put("primary", (Object)localAddress.getAddress().getHostAddress());
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void delegateAdminCommand(final QueryMessage message) throws DataStoreException {
throw new DataStoreException("unsupported command:" + message.getQuery());
}
private void reload(final QueryMessage message, final String dbName) throws DataStoreException {
this.assertNotReadOnly();
try {
this.dataStoreContext.reload();
this.dataStoreContext.getDataStore(dbName + "." + message.getQuery().get("reload")).reload();
final BSONObject doc = (BSONObject)new BasicBSONObject();
doc.put("ok", (Object)1.0);
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
catch (ConfigurationException e) {
throw new DataStoreException("error reload config: ", e);
}
}
private void showCollections(final QueryMessage message, final String dbName, final String collectionName) throws DataStoreException {
final Set<String> storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames();
final List<DBObject> docList = new ArrayList<DBObject>(storeNames.size());
for (final String sn : storeNames) {
if (sn.startsWith(dbName + ".")) {
docList.add((DBObject)new BasicDBObject("name", (Object)sn));
}
}
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), (List<? extends BSONObject>)docList);
}
private void getIndexes(final QueryMessage message, final String dbName, final String collectionName) throws DataStoreException {
final String ns = (String)message.getQuery().get("ns");
final Collection<Index> indexes = this.dataStoreContext.getDataStore(ns).getIndexes();
final List<BSONObject> docList = new ArrayList<BSONObject>(indexes.size());
for (final Index i : indexes) {
docList.add(i.toBSONObject());
}
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), docList);
}
private void find(final QueryMessage message, final String dbName, final String collectionName) throws DataStoreException {
String fullCollectionName;
if (message.getFullCollectionName().contains("$cmd")) {
fullCollectionName = dbName + "." + collectionName;
}
else {
fullCollectionName = message.getFullCollectionName();
}
final DataStore dataStore = this.dataStoreContext.getDataStore(fullCollectionName);
QueryResponse response;
if (this.useCursor()) {
response = dataStore.find(message.getRealQuery(), message.getReturnFieldSelector(), message.getOrderByQuery(), message.getFacetQuery(), message.getNumberToSkip(), message.getNumberToReturn(), message.isBulkQuery(), message.isFromMaster());
}
else {
response = dataStore.find(message.getRealQuery(), message.getReturnFieldSelector(), message.getOrderByQuery(), message.getFacetQuery(), message.getNumberToSkip(), message.getNumberToReturn() - 1, message.isBulkQuery(), message.isFromMaster());
}
final List<DBObject> doc = response.getDocList();
if (this.useCursor()) {
doc.remove(0);
}
final ReplyMessage reply = this.prepareReplyMessage(message.getMessageHeader().getRequestID(), (List<? extends BSONObject>)doc);
reply.setStartingFrom(message.getNumberToSkip());
if (response.hasMore() && this.useCursor()) {
final Cursor cursor = new Cursor(message.getNumberToSkip(), message.getNumberToReturn(), message, response.getNumFound());
this.dataStoreContext.addCursor(cursor);
reply.setCursorID(cursor.getId());
}
this.writeMessage(reply);
}
private void getLastError(final QueryMessage message, final String dbName) {
final Object lastError = this.getLastError();
final BSONObject doc = (BSONObject)new BasicBSONObject();
doc.put("ok", (Object)1.0);
doc.put("n", (Object)1);
doc.put("$err", (Object)((lastError == null) ? null : lastError.toString()));
doc.put("err", (Object)((lastError == null) ? null : lastError.toString()));
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void getPrevError(final QueryMessage message, final String dbName) {
final Object prevError = this.getPrevError();
final BSONObject doc = (BSONObject)new BasicBSONObject();
doc.put("ok", (Object)1.0);
doc.put("n", (Object)0);
doc.put("$err", (Object)((prevError == null) ? null : prevError.toString()));
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void count(final QueryMessage message, final String dbName) throws DataStoreException {
final BSONObject query = message.getQuery();
final int count = this.dataStoreContext.getDataStore(dbName + "." + query.get("count")).count(message.getRealQuery(), message.isFromMaster());
final BSONObject doc = (BSONObject)new BasicBSONObject();
doc.put("ok", (Object)1.0);
doc.put("n", (Object)(double)count);
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void getSchema(final QueryMessage message, final String dbName) throws DataStoreException {
final BSONObject query = message.getQuery();
final String collectionName = (String)query.get("getSchema");
final StoreConfig storeConfig = this.dataStoreContext.getDataStore(dbName + "." + collectionName).getStoreConfig();
final Collection<Field> fields = storeConfig.getFields();
final List<DBObject> docList = new ArrayList<DBObject>(fields.size());
for (final Field field : fields) {
final BasicDBObject f = new BasicDBObject();
f.put("name", field.getName());
f.put("type", field.getType().getType());
docList.add(f);
}
final BSONObject doc = new BasicBSONObject();
doc.put("ok", 1.0);
doc.put("schema", docList);
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void reIndex(final QueryMessage message, final String dbName) throws DataStoreException {
this.assertNotReadOnly();
final BSONObject query = message.getQuery();
final String collectionName = (String)query.get("reIndex");
final DataStore dataStore = this.dataStoreContext.getDataStore(dbName + "." + collectionName);
final BSONObject doc = (BSONObject)new BasicBSONObject();
final Object realQuery = query.get("query");
if (null == realQuery) {
doc.put("version", (Object)dataStore.reIndex(null));
}
else {
if (!(realQuery instanceof BSONObject)) {
throw new DataStoreException("invalid re-index query: " + realQuery);
}
dataStore.reIndex((BSONObject)realQuery);
}
doc.put("ok", (Object)1.0);
doc.put("n", (Object)0.0);
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void whatsMyUri(final QueryMessage message) {
final BSONObject doc = (BSONObject)new BasicBSONObject();
final InetSocketAddress address = (InetSocketAddress)this.getSession().getRemoteAddress();
doc.put("you", (Object)(address.getAddress().getHostAddress() + ":" + address.getPort()));
doc.put("ok", (Object)1.0);
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void listDatabases(final QueryMessage message) {
final BSONObject doc = (BSONObject)new BasicBSONObject();
doc.put("ok", (Object)1.0);
final Set<String> storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames();
final Set<String> dbNames = new HashSet<String>();
for (final String sn : storeNames) {
final int i = sn.indexOf(46);
if (i > 0) {
final String name = sn.substring(0, i);
dbNames.add(name);
}
}
final List<DBObject> docs = new ArrayList<DBObject>(dbNames.size());
for (final String dn : dbNames) {
docs.add((DBObject)new BasicDBObject("name", (Object)dn));
}
doc.put("databases", (Object)docs);
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void listCollections(final QueryMessage message, final String dbName) {
final BSONObject doc = (BSONObject)new BasicBSONObject();
doc.put("ok", (Object)1.0);
final BSONObject cursor = (BSONObject)new BasicDBObject();
doc.put("cursor", (Object)cursor);
cursor.put("id", (Object)(long)message.getMessageHeader().getRequestID());
cursor.put("ns", (Object)(dbName + ".$cmd.listCollections"));
final BasicDBList firstBatch = new BasicDBList();
cursor.put("firstBatch", (Object)firstBatch);
final Set<String> storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames();
for (final String sn : storeNames) {
if (sn.startsWith(dbName + ".")) {
final BSONObject collection = (BSONObject)new BasicDBObject();
collection.put("name", (Object)sn.replace(dbName + ".", ""));
collection.put("options", (Object)new BasicDBObject());
firstBatch.add((Object)collection);
}
}
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void nonce(final QueryMessage message) {
final BSONObject doc = (BSONObject)new BasicBSONObject();
doc.put("ok", (Object)1.0);
doc.put("errmsg", (Object)"no such cmd");
doc.put("bad cmd", (Object)new BasicBSONObject("nonce", (Object)true));
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void isMaster(final QueryMessage message) {
final BSONObject doc = (BSONObject)new BasicBSONObject();
doc.put("ok", (Object)1.0);
doc.put("ismaster", (Object)true);
doc.put("secondary", (Object)false);
final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress();
doc.put("primary", (Object)localAddress.getAddress().getHostAddress());
doc.put("maxWireVersion", (Object)2);
doc.put("minWireVersion", (Object)0);
final BasicBSONObject client = (BasicBSONObject)message.getQuery().get("client");
if (client != null) {
final String platform = client.getString("platform");
if (platform != null && (platform.contains("python") || platform.contains("Python"))) {
this.getSession().setAttribute((Object)"useCursor", (Object)true);
}
final BasicBSONObject applicationInfo = (BasicBSONObject)client.get("application");
if (applicationInfo != null && StringUtils.isNotBlank(applicationInfo.getString("name"))) {
this.getSession().setAttribute((Object)"useCursor", (Object)true);
}
}
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void buildinfo(final QueryMessage message) {
final BasicDBList versionArray = new BasicDBList();
versionArray.add((Object)2);
versionArray.add((Object)6);
versionArray.add((Object)8);
versionArray.add((Object)0);
final BSONObject doc = (BSONObject)new BasicBSONObject();
doc.put("ok", (Object)1.0);
doc.put("bits", (Object)64.0);
doc.put("debug", (Object)false);
doc.put("version", (Object)"2.6.8");
doc.put("versionArray", (Object)versionArray);
doc.put("javascriptEngine", (Object)"V8");
final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress();
doc.put("primary", (Object)localAddress.getAddress().getHostAddress());
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void serverStatus(final QueryMessage message) {
final BSONObject doc = (BSONObject)new BasicBSONObject();
doc.put("ok", (Object)1.0);
final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress();
doc.put("primary", (Object)localAddress.getAddress().getHostAddress());
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
private void connectionStatus(final QueryMessage message) {
final BSONObject doc = (BSONObject)new BasicBSONObject();
doc.put("ok", (Object)1.0);
final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress();
doc.put("primary", (Object)localAddress.getAddress().getHostAddress());
this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc);
}
}
@@ -1,27 +1,27 @@
package com.fangjia.datastore.handler;
import com.fangjia.datastore.mongo.protocol.MongoMessage;
import com.fangjia.datastore.DataStore;
import com.fangjia.datastore.DataStoreContext;
import com.fangjia.datastore.DataStoreException;
import com.fangjia.datastore.DataStoreContext;
import com.fangjia.datastore.mongo.protocol.UpdateMessage;
public class UpdateMessageHandler extends MessageHandler<UpdateMessage> {
public UpdateMessageHandler(DataStoreContext dataStoreContext) {
super(dataStoreContext);
}
public void messageReceived(UpdateMessage message,String ip) throws Exception {
assertNotReadOnly();
if (message.isUpsert()) {
throw new DataStoreException("upsert is not supported.");
}
if (!message.isMultiUpdate()) {
throw new DataStoreException("only multi update is supported.");
}
DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName());
dataStore.update(message.getSelector(), message.getUpdate(),ip);
}
public class UpdateMessageHandler extends MessageHandler<UpdateMessage>
{
public UpdateMessageHandler(final DataStoreContext dataStoreContext) {
super(dataStoreContext);
}
@Override
public void messageReceived(final UpdateMessage message) throws Exception {
this.assertNotReadOnly();
if (message.isUpsert()) {
throw new DataStoreException("upsert is not supported.");
}
if (!message.isMultiUpdate()) {
throw new DataStoreException("only multi update is supported.");
}
final DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName());
dataStore.update(message.getSelector(), message.getUpdate());
}
}
@@ -1,92 +1,75 @@
/**
* JMongo is a mongodb driver writtern in java.
* Copyright (C) 2010 Xiaohu Huang
*
* JMongo is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* JMongo is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
*/
package com.fangjia.datastore.mongo.codec;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ByteArrayOutputStream;
import org.apache.mina.core.buffer.IoBuffer;
import java.io.InputStream;
import org.apache.mina.core.buffer.IoBuffer;
public class BSONInputStream extends InputStream {
private final IoBuffer buffer;
public BSONInputStream(IoBuffer buffer) {
this.buffer = buffer;
}
@Override
public int read() {
return buffer.get();
}
public synchronized int read(byte b[], int off, int len) {
if (b == null) {
throw new NullPointerException();
} else if (off < 0 || len < 0 || len > b.length - off) {
throw new IndexOutOfBoundsException();
}
buffer.get(b, off, len);
return len;
}
public int readInteger() {
return this.buffer.getInt();
}
public long readLong() {
return this.buffer.getLong();
}
public double readDouble() {
return this.buffer.getDouble();
}
public String readCString() throws IOException {
ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
int b;
while (true) {
b = read();
if (b != 0) {
byteStream.write(b);
} else {
return new String(byteStream.toByteArray(), "UTF-8");
}
}
}
public String readString() throws IOException {
int len = readInteger();
if (len < 0) {
throw new IOException("can not read string from the stream");
}
if (len == 0) {
read();
return "";
}
byte[] buf = new byte[len];
read(buf);
return new String(buf, 0, len - 1, "UTF-8");
}
@Override
public int available() throws IOException {
return buffer.remaining();
}
public class BSONInputStream extends InputStream
{
private final IoBuffer buffer;
public BSONInputStream(final IoBuffer buffer) {
this.buffer = buffer;
}
@Override
public int read() {
return this.buffer.get();
}
@Override
public synchronized int read(final byte[] b, final int off, final int len) {
if (b == null) {
throw new NullPointerException();
}
if (off < 0 || len < 0 || len > b.length - off) {
throw new IndexOutOfBoundsException();
}
this.buffer.get(b, off, len);
return len;
}
public int readInteger() {
return this.buffer.getInt();
}
public long readLong() {
return this.buffer.getLong();
}
public double readDouble() {
return this.buffer.getDouble();
}
public String readCString() throws IOException {
final ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
while (true) {
final int b = this.read();
if (b == 0) {
break;
}
byteStream.write(b);
}
return new String(byteStream.toByteArray(), "UTF-8");
}
public String readString() throws IOException {
final int len = this.readInteger();
if (len < 0) {
throw new IOException("can not read string from the stream");
}
if (len == 0) {
this.read();
return "";
}
final byte[] buf = new byte[len];
this.read(buf);
return new String(buf, 0, len - 1, "UTF-8");
}
@Override
public int available() throws IOException {
return this.buffer.remaining();
}
}
@@ -1,95 +1,83 @@
package com.fangjia.datastore.mongo.codec;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import org.apache.mina.core.buffer.AbstractIoBuffer;
import java.nio.ByteOrder;
import java.nio.ByteBuffer;
import org.apache.mina.core.buffer.IoBuffer;
import org.apache.mina.core.buffer.IoBufferAllocator;
public class BSONIoBufferAllocator implements IoBufferAllocator {
public IoBuffer allocate(int capacity, boolean direct) {
return wrap(allocateNioBuffer(capacity, direct));
}
public ByteBuffer allocateNioBuffer(int capacity, boolean direct) {
ByteBuffer nioBuffer;
if (direct) {
nioBuffer = ByteBuffer.allocateDirect(capacity);
} else {
nioBuffer = ByteBuffer.allocate(capacity);
}
nioBuffer.order(ByteOrder.LITTLE_ENDIAN);
return nioBuffer;
}
public IoBuffer wrap(ByteBuffer nioBuffer) {
return new SimpleBuffer(nioBuffer);
}
public void dispose() {
// Do nothing
}
private class SimpleBuffer extends AbstractIoBuffer {
private ByteBuffer buf;
protected SimpleBuffer(ByteBuffer buf) {
super(BSONIoBufferAllocator.this, buf.capacity());
this.buf = buf;
this.buf.order(ByteOrder.LITTLE_ENDIAN);
}
protected SimpleBuffer(SimpleBuffer parent, ByteBuffer buf) {
super(parent);
this.buf = buf;
this.buf.order(ByteOrder.LITTLE_ENDIAN);
}
@Override
public ByteBuffer buf() {
return buf;
}
@Override
protected void buf(ByteBuffer buf) {
this.buf = buf;
}
@Override
protected IoBuffer duplicate0() {
return new SimpleBuffer(this, this.buf.duplicate());
}
@Override
protected IoBuffer slice0() {
return new SimpleBuffer(this, this.buf.slice());
}
@Override
protected IoBuffer asReadOnlyBuffer0() {
return new SimpleBuffer(this, this.buf.asReadOnlyBuffer());
}
@Override
public byte[] array() {
return buf.array();
}
@Override
public int arrayOffset() {
return buf.arrayOffset();
}
@Override
public boolean hasArray() {
return buf.hasArray();
}
@Override
public void free() {
// Do nothing
}
}
public class BSONIoBufferAllocator implements IoBufferAllocator
{
public IoBuffer allocate(final int capacity, final boolean direct) {
return this.wrap(this.allocateNioBuffer(capacity, direct));
}
public ByteBuffer allocateNioBuffer(final int capacity, final boolean direct) {
ByteBuffer nioBuffer;
if (direct) {
nioBuffer = ByteBuffer.allocateDirect(capacity);
}
else {
nioBuffer = ByteBuffer.allocate(capacity);
}
nioBuffer.order(ByteOrder.LITTLE_ENDIAN);
return nioBuffer;
}
public IoBuffer wrap(final ByteBuffer nioBuffer) {
return (IoBuffer)new SimpleBuffer(nioBuffer);
}
public void dispose() {
}
private class SimpleBuffer extends AbstractIoBuffer
{
private ByteBuffer buf;
protected SimpleBuffer(final ByteBuffer buf) {
super((IoBufferAllocator)BSONIoBufferAllocator.this, buf.capacity());
(this.buf = buf).order(ByteOrder.LITTLE_ENDIAN);
}
protected SimpleBuffer(final SimpleBuffer parent, final ByteBuffer buf) {
super((AbstractIoBuffer)parent);
(this.buf = buf).order(ByteOrder.LITTLE_ENDIAN);
}
public ByteBuffer buf() {
return this.buf;
}
protected void buf(final ByteBuffer buf) {
this.buf = buf;
}
protected IoBuffer duplicate0() {
return (IoBuffer)new SimpleBuffer(this, this.buf.duplicate());
}
protected IoBuffer slice0() {
return (IoBuffer)new SimpleBuffer(this, this.buf.slice());
}
protected IoBuffer asReadOnlyBuffer0() {
return (IoBuffer)new SimpleBuffer(this, this.buf.asReadOnlyBuffer());
}
public byte[] array() {
return this.buf.array();
}
public int arrayOffset() {
return this.buf.arrayOffset();
}
public boolean hasArray() {
return this.buf.hasArray();
}
public void free() {
}
}
}
@@ -1,102 +1,84 @@
/**
* JMongo is a mongodb driver writtern in java.
* Copyright (C) 2010 Xiaohu Huang
*
* JMongo is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* JMongo is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
*/
package com.fangjia.datastore.mongo.codec;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.io.IOException;
import java.io.ByteArrayOutputStream;
public class BSONOutputStream extends ByteArrayOutputStream {
private static final String ENCDOING = "UTF-8";
public BSONOutputStream() {
super();
}
public BSONOutputStream(int size) {
super(size);
}
public void writeCString(String s) throws IOException {
write(s.getBytes(ENCDOING));
write(0);
}
public void writeInteger(int i) throws IOException {
write(i);
write(i >> 8);
write(i >> 16);
write(i >> 24);
}
public void writeLong(long l) throws IOException {
write((byte) l);
write((byte) (l >> 8));
write((byte) (l >> 16));
write((byte) (l >> 24));
write((byte) (l >> 32));
write((byte) (l >> 40));
write((byte) (l >> 48));
write((byte) (l >> 56));
}
public void writeString(String s) throws IOException {
byte[] bs = s.getBytes(ENCDOING);
writeInteger(bs.length + 1);
write(bs);
write(0);
}
public void set(int index, int i) {
int newcount = Math.max(count, index + 4);
if (newcount > buf.length) {
buf = Arrays.copyOf(buf, Math.max(buf.length << 1, newcount));
}
buf[index] = (byte) i;
buf[index + 1] = (byte) (i >> 8);
buf[index + 2] = (byte) (i >> 16);
buf[index + 3] = (byte) (i >> 24);
count = newcount;
}
public void set(int index, byte[] b, int off, int len) {
if ((off < 0) || (off > b.length) || (len < 0) || ((off + len) > b.length) || ((off + len) < 0)) {
throw new IndexOutOfBoundsException();
} else if (len == 0) {
return;
}
int newcount = Math.max(count, index + len);
if (newcount > buf.length) {
buf = Arrays.copyOf(buf, Math.max(buf.length << 1, newcount));
}
System.arraycopy(b, off, buf, index, len);
count = newcount;
}
public void setInteger(int index, int i) {
byte[] b = new byte[] { (byte) i, (byte) (i >> 8), (byte) (i >> 16), (byte) (i >> 24) };
set(index, b, 0, b.length);
}
public void setLong(int index, long l) {
byte[] b = new byte[] { (byte) l, (byte) (l >> 8), (byte) (l >> 16), (byte) (l >> 24), (byte) (l >> 32),
(byte) (l >> 40), (byte) (l >> 48), (byte) (l >> 56) };
set(index, b, 0, b.length);
}
public class BSONOutputStream extends ByteArrayOutputStream
{
private static final String ENCDOING = "UTF-8";
public BSONOutputStream() {
}
public BSONOutputStream(final int size) {
super(size);
}
public void writeCString(final String s) throws IOException {
this.write(s.getBytes("UTF-8"));
this.write(0);
}
public void writeInteger(final int i) throws IOException {
this.write(i);
this.write(i >> 8);
this.write(i >> 16);
this.write(i >> 24);
}
public void writeLong(final long l) throws IOException {
this.write((byte)l);
this.write((byte)(l >> 8));
this.write((byte)(l >> 16));
this.write((byte)(l >> 24));
this.write((byte)(l >> 32));
this.write((byte)(l >> 40));
this.write((byte)(l >> 48));
this.write((byte)(l >> 56));
}
public void writeString(final String s) throws IOException {
final byte[] bs = s.getBytes("UTF-8");
this.writeInteger(bs.length + 1);
this.write(bs);
this.write(0);
}
public void set(final int index, final int i) {
final int newcount = Math.max(this.count, index + 4);
if (newcount > this.buf.length) {
this.buf = Arrays.copyOf(this.buf, Math.max(this.buf.length << 1, newcount));
}
this.buf[index] = (byte)i;
this.buf[index + 1] = (byte)(i >> 8);
this.buf[index + 2] = (byte)(i >> 16);
this.buf[index + 3] = (byte)(i >> 24);
this.count = newcount;
}
public void set(final int index, final byte[] b, final int off, final int len) {
if (off < 0 || off > b.length || len < 0 || off + len > b.length || off + len < 0) {
throw new IndexOutOfBoundsException();
}
if (len == 0) {
return;
}
final int newcount = Math.max(this.count, index + len);
if (newcount > this.buf.length) {
this.buf = Arrays.copyOf(this.buf, Math.max(this.buf.length << 1, newcount));
}
System.arraycopy(b, off, this.buf, index, len);
this.count = newcount;
}
public void setInteger(final int index, final int i) {
final byte[] b = { (byte)i, (byte)(i >> 8), (byte)(i >> 16), (byte)(i >> 24) };
this.set(index, b, 0, b.length);
}
public void setLong(final int index, final long l) {
final byte[] b = { (byte)l, (byte)(l >> 8), (byte)(l >> 16), (byte)(l >> 24), (byte)(l >> 32), (byte)(l >> 40), (byte)(l >> 48), (byte)(l >> 56) };
this.set(index, b, 0, b.length);
}
}
@@ -1,111 +1,108 @@
package com.fangjia.datastore.mongo.codec;
import org.slf4j.LoggerFactory;
import org.apache.mina.core.buffer.BufferDataException;
import com.fangjia.datastore.mongo.protocol.UpdateMessage;
import com.fangjia.datastore.mongo.protocol.ReplyMessage;
import com.fangjia.datastore.mongo.protocol.QueryMessage;
import com.fangjia.datastore.mongo.protocol.KillCursorsMessage;
import com.fangjia.datastore.mongo.protocol.InsertMessage;
import com.fangjia.datastore.mongo.protocol.GetMoreMessage;
import com.fangjia.datastore.mongo.protocol.DeleteMessage;
import com.fangjia.datastore.mongo.protocol.MongoMessage;
import org.apache.mina.filter.codec.ProtocolDecoderException;
import com.fangjia.datastore.mongo.protocol.OperationCode;
import org.apache.mina.filter.codec.ProtocolDecoderOutput;
import org.apache.mina.core.buffer.IoBuffer;
import org.apache.mina.core.session.IoSession;
import org.apache.mina.filter.codec.CumulativeProtocolDecoder;
import org.apache.mina.filter.codec.ProtocolDecoderException;
import org.apache.mina.filter.codec.ProtocolDecoderOutput;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fangjia.datastore.mongo.protocol.DeleteMessage;
import com.fangjia.datastore.mongo.protocol.GetMoreMessage;
import com.fangjia.datastore.mongo.protocol.InsertMessage;
import com.fangjia.datastore.mongo.protocol.KillCursorsMessage;
import com.fangjia.datastore.mongo.protocol.MongoMessage;
import com.fangjia.datastore.mongo.protocol.OperationCode;
import com.fangjia.datastore.mongo.protocol.QueryMessage;
import com.fangjia.datastore.mongo.protocol.ReplyMessage;
import com.fangjia.datastore.mongo.protocol.UpdateMessage;
public class MongoProtocolDecoder extends CumulativeProtocolDecoder {
private final static Logger LOGGER = LoggerFactory.getLogger(MongoProtocolDecoder.class);
@Override
protected boolean doDecode(IoSession session, IoBuffer in, ProtocolDecoderOutput out) throws Exception {
try {
if (!prefixedDataAvailable(in, 4, true)) {
return false;
}
int start = in.position();
int limit = in.limit();
int messageSize = in.getInt(start);
in.limit(start + messageSize);
try {
final IoBuffer buf = in.slice();
int opCodeValue = buf.getInt(buf.position() + 12);
OperationCode opCode = OperationCode.valueOf(opCodeValue);
MongoMessage message = newInstance(opCode);
BSONInputStream input = new BSONInputStream(buf);
message.read(input);
out.write(message);
return true;
} finally {
in.position(start + messageSize);
in.limit(limit);
}
} catch (Exception e) {
ProtocolDecoderException pde = new ProtocolDecoderException("error decode message:", e);
pde.setHexdump(in.getHexDump());
LOGGER.error("", pde);
throw pde;
}
}
private MongoMessage newInstance(OperationCode opCode) {
switch (opCode) {
case OP_DELETE:
return new DeleteMessage();
case OP_GET_MORE:
return new GetMoreMessage();
case OP_INSERT:
return new InsertMessage();
case OP_KILL_CURSORS:
return new KillCursorsMessage();
case OP_QUERY:
return new QueryMessage();
case OP_REPLY:
return new ReplyMessage();
case OP_UPDATE:
return new UpdateMessage();
case OP_MSG:
case RESERVED:
default:
return null;
}
}
public boolean prefixedDataAvailable(IoBuffer buffer, int prefixLength, boolean includePrefix) {
if (buffer.remaining() < prefixLength) {
return false;
}
int dataLength;
switch (prefixLength) {
case 1:
dataLength = buffer.getUnsigned(buffer.position());
break;
case 2:
dataLength = buffer.getUnsignedShort(buffer.position());
break;
case 4:
dataLength = buffer.getInt(buffer.position());
break;
default:
throw new IllegalArgumentException("prefixLength: " + prefixLength);
}
if (dataLength < 0) {
throw new BufferDataException("dataLength: " + dataLength);
}
if (includePrefix) {
return buffer.remaining() >= dataLength;
} else {
return buffer.remaining() - prefixLength >= dataLength;
}
}
import org.apache.mina.filter.codec.CumulativeProtocolDecoder;
public class MongoProtocolDecoder extends CumulativeProtocolDecoder
{
private static final Logger LOGGER;
protected boolean doDecode(final IoSession session, final IoBuffer in, final ProtocolDecoderOutput out) throws Exception {
try {
if (!this.prefixedDataAvailable(in, 4, true)) {
return false;
}
final int start = in.position();
final int limit = in.limit();
final int messageSize = in.getInt(start);
in.limit(start + messageSize);
try {
final IoBuffer buf = in.slice();
final int opCodeValue = buf.getInt(buf.position() + 12);
final OperationCode opCode = OperationCode.valueOf(opCodeValue);
final MongoMessage message = this.newInstance(opCode);
final BSONInputStream input = new BSONInputStream(buf);
message.read(input);
out.write((Object)message);
return true;
}
finally {
in.position(start + messageSize);
in.limit(limit);
}
}
catch (Exception e) {
final ProtocolDecoderException pde = new ProtocolDecoderException("error decode message:", (Throwable)e);
pde.setHexdump(in.getHexDump());
MongoProtocolDecoder.LOGGER.error("", (Throwable)pde);
throw pde;
}
}
private MongoMessage newInstance(final OperationCode opCode) {
switch (opCode) {
case OP_DELETE:
return new DeleteMessage();
case OP_GET_MORE:
return new GetMoreMessage();
case OP_INSERT:
return new InsertMessage();
case OP_KILL_CURSORS:
return new KillCursorsMessage();
case OP_QUERY:
return new QueryMessage();
case OP_REPLY:
return new ReplyMessage();
case OP_UPDATE:
return new UpdateMessage();
default:
return null;
}
}
public boolean prefixedDataAvailable(final IoBuffer buffer, final int prefixLength, final boolean includePrefix) {
if (buffer.remaining() < prefixLength) {
return false;
}
int dataLength = 0;
switch (prefixLength) {
case 1:
dataLength = buffer.getUnsigned(buffer.position());
break;
case 2:
dataLength = buffer.getUnsignedShort(buffer.position());
break;
case 4:
dataLength = buffer.getInt(buffer.position());
break;
default:
throw new IllegalArgumentException("prefixLength: " + prefixLength);
}
if (dataLength < 0) {
throw new BufferDataException("dataLength: " + dataLength);
}
if (includePrefix) {
return buffer.remaining() >= dataLength;
}
return buffer.remaining() - prefixLength >= dataLength;
}
static {
LOGGER = LoggerFactory.getLogger((Class)MongoProtocolDecoder.class);
}
}
@@ -1,36 +1,38 @@
package com.fangjia.datastore.mongo.codec;
import org.apache.mina.core.buffer.IoBuffer;
import org.apache.mina.core.session.IoSession;
import org.apache.mina.filter.codec.ProtocolEncoder;
import org.apache.mina.filter.codec.ProtocolEncoderException;
import org.apache.mina.filter.codec.ProtocolEncoderOutput;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.mina.filter.codec.ProtocolEncoderException;
import org.apache.mina.core.buffer.IoBuffer;
import com.fangjia.datastore.mongo.protocol.MongoMessage;
import org.apache.mina.filter.codec.ProtocolEncoderOutput;
import org.apache.mina.core.session.IoSession;
import org.slf4j.Logger;
import org.apache.mina.filter.codec.ProtocolEncoder;
public class MongoProtocolEncoder implements ProtocolEncoder {
private final static Logger LOGGER = LoggerFactory.getLogger(MongoProtocolEncoder.class);
public void encode(IoSession session, Object message, ProtocolEncoderOutput out) throws Exception {
try {
if (message instanceof MongoMessage) {
MongoMessage mongoMessage = (MongoMessage) message;
BSONOutputStream output = new BSONOutputStream(mongoMessage.getMessageHeader().getMessageLength());
mongoMessage.write(output);
out.write(IoBuffer.wrap(output.toByteArray()));
}
} catch (Exception e) {
ProtocolEncoderException pe = new ProtocolEncoderException("error encode message: " + message, e);
LOGGER.error("", pe);
throw pe;
}
}
public void dispose(IoSession session) throws Exception {
// empty
}
public class MongoProtocolEncoder implements ProtocolEncoder
{
private static final Logger LOGGER;
public void encode(final IoSession session, final Object message, final ProtocolEncoderOutput out) throws Exception {
try {
if (message instanceof MongoMessage) {
final MongoMessage mongoMessage = (MongoMessage)message;
final BSONOutputStream output = new BSONOutputStream(mongoMessage.getMessageHeader().getMessageLength());
mongoMessage.write(output);
out.write((Object)IoBuffer.wrap(output.toByteArray()));
}
}
catch (Exception e) {
final ProtocolEncoderException pe = new ProtocolEncoderException("error encode message: " + message, (Throwable)e);
MongoProtocolEncoder.LOGGER.error("", (Throwable)pe);
throw pe;
}
}
public void dispose(final IoSession session) throws Exception {
}
static {
LOGGER = LoggerFactory.getLogger((Class)MongoProtocolEncoder.class);
}
}
@@ -1,111 +1,98 @@
/**
* JMongo is a mongodb driver writtern in java.
* Copyright (C) 2010 Xiaohu Huang
*
* JMongo is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* JMongo is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
*/
package com.fangjia.datastore.mongo.protocol;
import java.io.IOException;
import org.bson.BSONDecoder;
import org.bson.BSONEncoder;
import org.bson.BSONObject;
import org.bson.BasicBSONDecoder;
import org.bson.BasicBSONObject;
import com.fangjia.datastore.mongo.codec.BSONInputStream;
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
import com.fangjia.datastore.utils.BSONUtils;
import org.bson.BSONDecoder;
import java.io.InputStream;
import org.bson.BasicBSONDecoder;
import com.fangjia.datastore.mongo.codec.BSONInputStream;
import java.io.IOException;
import org.bson.BSONEncoder;
import org.bson.BasicBSONEncoder;
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
import org.bson.BasicBSONObject;
import org.bson.BSONObject;
public class DeleteMessage implements MongoMessage {
private static final long serialVersionUID = -3350216587439425208L;
private MessageHeader messageHeader;
private String fullCollectionName;
private int flags;
private BSONObject selector;
public DeleteMessage() {
messageHeader = new MessageHeader(OperationCode.OP_DELETE);
selector = new BasicBSONObject();
}
public void write(BSONOutputStream out) throws IOException {
messageHeader.write(out);
out.writeInteger(0);
out.writeCString(this.fullCollectionName);
out.writeInteger(this.flags);
BSONEncoder encoder = new BSONEncoder();
out.write(encoder.encode(selector));
out.set(0, out.size());
}
public void read(BSONInputStream in) throws IOException {
messageHeader.read(in);
in.readInteger();
this.fullCollectionName = in.readCString();
this.flags = in.readInteger();
BSONDecoder decoder = new BasicBSONDecoder();
selector = decoder.readObject(in);
}
public MessageHeader getMessageHeader() {
return messageHeader;
}
public String getFullCollectionName() {
return fullCollectionName;
}
public void setFullCollectionName(String fullCollectionName) {
this.fullCollectionName = fullCollectionName;
}
public int getFlags() {
return flags;
}
public void setFlags(int flags) {
this.flags = flags;
}
public void setSingleRemove(boolean b) {
this.flags = (int) BSONUtils.bitSet(this.flags, 0, b);
}
public BSONObject getSelector() {
return selector;
}
public void setSelector(BSONObject selector) {
this.selector = selector;
}
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("messageHeader:").append(messageHeader);
builder.append(", fullCollectionName:").append(fullCollectionName);
builder.append(", flags:").append(flags);
builder.append(", selector:").append(selector);
builder.append("}");
return builder.toString();
}
public String toPrettyString() {
return toString();
}
public class DeleteMessage implements MongoMessage
{
private static final long serialVersionUID = -3350216587439425208L;
private MessageHeader messageHeader;
private String fullCollectionName;
private int flags;
private BSONObject selector;
public DeleteMessage() {
this.messageHeader = new MessageHeader(OperationCode.OP_DELETE);
this.selector = (BSONObject)new BasicBSONObject();
}
@Override
public void write(final BSONOutputStream out) throws IOException {
this.messageHeader.write(out);
out.writeInteger(0);
out.writeCString(this.fullCollectionName);
out.writeInteger(this.flags);
final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder();
out.write(encoder.encode(this.selector));
out.set(0, out.size());
}
@Override
public void read(final BSONInputStream in) throws IOException {
this.messageHeader.read(in);
in.readInteger();
this.fullCollectionName = in.readCString();
this.flags = in.readInteger();
final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder();
this.selector = decoder.readObject((InputStream)in);
}
@Override
public MessageHeader getMessageHeader() {
return this.messageHeader;
}
public String getFullCollectionName() {
return this.fullCollectionName;
}
public void setFullCollectionName(final String fullCollectionName) {
this.fullCollectionName = fullCollectionName;
}
public int getFlags() {
return this.flags;
}
public void setFlags(final int flags) {
this.flags = flags;
}
public void setSingleRemove(final boolean b) {
this.flags = (int)BSONUtils.bitSet(this.flags, 0, b);
}
public BSONObject getSelector() {
return this.selector;
}
public void setSelector(final BSONObject selector) {
this.selector = selector;
}
@Override
public String toString() {
final StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("messageHeader:").append(this.messageHeader);
builder.append(", fullCollectionName:").append(this.fullCollectionName);
builder.append(", flags:").append(this.flags);
builder.append(", selector:").append(this.selector);
builder.append("}");
return builder.toString();
}
@Override
public String toPrettyString() {
return this.toString();
}
}
@@ -1,97 +1,83 @@
/**
* JMongo is a mongodb driver writtern in java.
* Copyright (C) 2010 Xiaohu Huang
*
* JMongo is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* JMongo is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
*/
package com.fangjia.datastore.mongo.protocol;
import java.io.IOException;
import com.fangjia.datastore.mongo.codec.BSONInputStream;
import java.io.IOException;
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
public class GetMoreMessage implements MongoMessage {
private static final long serialVersionUID = -3350216587439425208L;
private MessageHeader messageHeader;
private String fullCollectionName;
private int numberToReturn;
private long cursorID;
public GetMoreMessage() {
messageHeader = new MessageHeader(OperationCode.OP_GET_MORE);
}
public void write(BSONOutputStream out) throws IOException {
messageHeader.write(out);
out.writeInteger(0);
out.writeCString(this.fullCollectionName);
out.writeInteger(this.numberToReturn);
out.writeLong(cursorID);
out.set(0, out.size());
}
public void read(BSONInputStream in) throws IOException {
messageHeader.read(in);
in.readInteger();
this.fullCollectionName = in.readCString();
this.numberToReturn = in.readInteger();
this.cursorID = in.readLong();
}
public MessageHeader getMessageHeader() {
return messageHeader;
}
public String getFullCollectionName() {
return fullCollectionName;
}
public void setFullCollectionName(String fullCollectionName) {
this.fullCollectionName = fullCollectionName;
}
public int getNumberToReturn() {
return numberToReturn;
}
public void setNumberToReturn(int numberToReturn) {
this.numberToReturn = numberToReturn;
}
public long getCursorID() {
return cursorID;
}
public void setCursorID(long cursorID) {
this.cursorID = cursorID;
}
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("messageHeader:").append(messageHeader);
builder.append(", fullCollectionName:").append(fullCollectionName);
builder.append(", numberToReturn:").append(numberToReturn);
builder.append(", cursorID:").append(cursorID);
builder.append("}");
return builder.toString();
}
public String toPrettyString() {
return toString();
}
public class GetMoreMessage implements MongoMessage
{
private static final long serialVersionUID = -3350216587439425208L;
private MessageHeader messageHeader;
private String fullCollectionName;
private int numberToReturn;
private long cursorID;
public GetMoreMessage() {
this.messageHeader = new MessageHeader(OperationCode.OP_GET_MORE);
}
@Override
public void write(final BSONOutputStream out) throws IOException {
this.messageHeader.write(out);
out.writeInteger(0);
out.writeCString(this.fullCollectionName);
out.writeInteger(this.numberToReturn);
out.writeLong(this.cursorID);
out.set(0, out.size());
}
@Override
public void read(final BSONInputStream in) throws IOException {
this.messageHeader.read(in);
in.readInteger();
this.fullCollectionName = in.readCString();
this.numberToReturn = in.readInteger();
this.cursorID = in.readLong();
}
@Override
public MessageHeader getMessageHeader() {
return this.messageHeader;
}
public String getFullCollectionName() {
return this.fullCollectionName;
}
public void setFullCollectionName(final String fullCollectionName) {
this.fullCollectionName = fullCollectionName;
}
public int getNumberToReturn() {
return this.numberToReturn;
}
public void setNumberToReturn(final int numberToReturn) {
this.numberToReturn = numberToReturn;
}
public long getCursorID() {
return this.cursorID;
}
public void setCursorID(final long cursorID) {
this.cursorID = cursorID;
}
@Override
public String toString() {
final StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("messageHeader:").append(this.messageHeader);
builder.append(", fullCollectionName:").append(this.fullCollectionName);
builder.append(", numberToReturn:").append(this.numberToReturn);
builder.append(", cursorID:").append(this.cursorID);
builder.append("}");
return builder.toString();
}
@Override
public String toPrettyString() {
return this.toString();
}
}
@@ -1,116 +1,105 @@
/**
* JMongo is a mongodb driver writtern in java.
* Copyright (C) 2010 Xiaohu Huang
*
* JMongo is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* JMongo is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
*/
package com.fangjia.datastore.mongo.protocol;
import java.io.IOException;
import org.bson.BSONDecoder;
import java.io.InputStream;
import org.bson.BasicBSONDecoder;
import java.util.ArrayList;
import com.fangjia.datastore.mongo.codec.BSONInputStream;
import java.io.IOException;
import java.util.Iterator;
import org.bson.BSONEncoder;
import org.bson.BasicBSONEncoder;
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
import org.bson.BSONObject;
import java.util.List;
import org.bson.BSONDecoder;
import org.bson.BSONEncoder;
import org.bson.BSONObject;
import org.bson.BasicBSONDecoder;
import com.fangjia.datastore.mongo.codec.BSONInputStream;
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
public class InsertMessage implements MongoMessage {
private static final long serialVersionUID = -3350216587439425208L;
private MessageHeader messageHeader;
private String fullCollectionName;
private List<BSONObject> documents;
public InsertMessage() {
messageHeader = new MessageHeader(OperationCode.OP_INSERT);
}
public void write(BSONOutputStream out) throws IOException {
messageHeader.write(out);
out.writeInteger(0);
out.writeCString(this.fullCollectionName);
if (null != documents && documents.size() > 0) {
BSONEncoder encoder = new BSONEncoder();
for (BSONObject doc : documents) {
out.write(encoder.encode(doc));
}
}
out.setInteger(0, out.size());
}
public void read(BSONInputStream in) throws IOException {
messageHeader.read(in);
in.readInteger();
this.fullCollectionName = in.readCString();
while (in.available() > 0) {
if (null == documents) {
documents = new ArrayList<BSONObject>();
}
BSONDecoder decoder = new BasicBSONDecoder();
documents.add(decoder.readObject(in));
}
}
public MessageHeader getMessageHeader() {
return messageHeader;
}
public String getFullCollectionName() {
return fullCollectionName;
}
public void setFullCollectionName(String fullCollectionName) {
this.fullCollectionName = fullCollectionName;
}
public List<BSONObject> getDocuments() {
return documents;
}
public void setDocuments(List<BSONObject> documents) {
this.documents = documents;
}
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("messageHeader:").append(messageHeader);
builder.append(", fullCollectionName:").append(fullCollectionName);
builder.append(", documents:").append(documents);
builder.append("}");
return builder.toString();
}
public String toPrettyString() {
StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("messageHeader:").append(messageHeader);
builder.append(", fullCollectionName:").append(fullCollectionName);
if (null != documents && documents.size() > 0) {
builder.append(", documents:").append(documents.get(0).get("_id"));
if (documents.size() > 1) {
builder.append("...").append(documents.size() - 1).append(" more");
}
} else {
builder.append(", documents:").append(documents);
}
builder.append("}");
return builder.toString();
}
public class InsertMessage implements MongoMessage
{
private static final long serialVersionUID = -3350216587439425208L;
private MessageHeader messageHeader;
private String fullCollectionName;
private List<BSONObject> documents;
public InsertMessage() {
this.messageHeader = new MessageHeader(OperationCode.OP_INSERT);
}
@Override
public void write(final BSONOutputStream out) throws IOException {
this.messageHeader.write(out);
out.writeInteger(0);
out.writeCString(this.fullCollectionName);
if (null != this.documents && this.documents.size() > 0) {
final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder();
for (final BSONObject doc : this.documents) {
out.write(encoder.encode(doc));
}
}
out.setInteger(0, out.size());
}
@Override
public void read(final BSONInputStream in) throws IOException {
this.messageHeader.read(in);
in.readInteger();
this.fullCollectionName = in.readCString();
while (in.available() > 0) {
if (null == this.documents) {
this.documents = new ArrayList<BSONObject>();
}
final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder();
this.documents.add(decoder.readObject((InputStream)in));
}
}
@Override
public MessageHeader getMessageHeader() {
return this.messageHeader;
}
public String getFullCollectionName() {
return this.fullCollectionName;
}
public void setFullCollectionName(final String fullCollectionName) {
this.fullCollectionName = fullCollectionName;
}
public List<BSONObject> getDocuments() {
return this.documents;
}
public void setDocuments(final List<BSONObject> documents) {
this.documents = documents;
}
@Override
public String toString() {
final StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("messageHeader:").append(this.messageHeader);
builder.append(", fullCollectionName:").append(this.fullCollectionName);
builder.append(", documents:").append(this.documents);
builder.append("}");
return builder.toString();
}
@Override
public String toPrettyString() {
final StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("messageHeader:").append(this.messageHeader);
builder.append(", fullCollectionName:").append(this.fullCollectionName);
if (null != this.documents && this.documents.size() > 0) {
builder.append(", documents:").append(this.documents.get(0).get("_id"));
if (this.documents.size() > 1) {
builder.append("...").append(this.documents.size() - 1).append(" more");
}
}
else {
builder.append(", documents:").append(this.documents);
}
builder.append("}");
return builder.toString();
}
}
@@ -1,96 +1,83 @@
/**
* JMongo is a mongodb driver writtern in java.
* Copyright (C) 2010 Xiaohu Huang
*
* JMongo is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* JMongo is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
*/
package com.fangjia.datastore.mongo.protocol;
import java.io.IOException;
import java.util.ArrayList;
import com.fangjia.datastore.mongo.codec.BSONInputStream;
import java.io.IOException;
import java.util.Iterator;
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
import java.util.List;
import com.fangjia.datastore.mongo.codec.BSONInputStream;
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
public class KillCursorsMessage implements MongoMessage {
private static final long serialVersionUID = -3350216587439425208L;
private MessageHeader messageHeader;
private int numberOfCursorIDs;
private List<Long> cursorIDs;
public KillCursorsMessage() {
messageHeader = new MessageHeader(OperationCode.OP_KILL_CURSORS);
}
public MessageHeader getMessageHeader() {
return messageHeader;
}
public void write(BSONOutputStream out) throws IOException {
messageHeader.write(out);
out.writeInteger(0);
out.writeInteger(numberOfCursorIDs);
if (null != cursorIDs) {
for (Long id : cursorIDs) {
out.writeLong(id);
}
}
out.set(0, out.size());
}
public void read(BSONInputStream in) throws IOException {
messageHeader.read(in);
in.readInteger();
numberOfCursorIDs = in.readInteger();
while (in.available() > 0) {
if (null == cursorIDs) {
cursorIDs = new ArrayList<Long>();
}
cursorIDs.add(in.readLong());
}
}
public int getNumberOfCursorIDs() {
return numberOfCursorIDs;
}
public void setNumberOfCursorIDs(int numberOfCursorIDs) {
this.numberOfCursorIDs = numberOfCursorIDs;
}
public List<Long> getCursorIDs() {
return cursorIDs;
}
public void setCursorIDs(List<Long> cursorIDs) {
this.cursorIDs = cursorIDs;
}
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("messageHeader:").append(messageHeader);
builder.append(", numberOfCursorIDs:").append(numberOfCursorIDs);
builder.append(", cursorIDs:").append(cursorIDs);
builder.append("}");
return builder.toString();
}
public String toPrettyString() {
return toString();
}
public class KillCursorsMessage implements MongoMessage
{
private static final long serialVersionUID = -3350216587439425208L;
private MessageHeader messageHeader;
private int numberOfCursorIDs;
private List<Long> cursorIDs;
public KillCursorsMessage() {
this.messageHeader = new MessageHeader(OperationCode.OP_KILL_CURSORS);
}
@Override
public MessageHeader getMessageHeader() {
return this.messageHeader;
}
@Override
public void write(final BSONOutputStream out) throws IOException {
this.messageHeader.write(out);
out.writeInteger(0);
out.writeInteger(this.numberOfCursorIDs);
if (null != this.cursorIDs) {
for (final Long id : this.cursorIDs) {
out.writeLong(id);
}
}
out.set(0, out.size());
}
@Override
public void read(final BSONInputStream in) throws IOException {
this.messageHeader.read(in);
in.readInteger();
this.numberOfCursorIDs = in.readInteger();
while (in.available() > 0) {
if (null == this.cursorIDs) {
this.cursorIDs = new ArrayList<Long>();
}
this.cursorIDs.add(in.readLong());
}
}
public int getNumberOfCursorIDs() {
return this.numberOfCursorIDs;
}
public void setNumberOfCursorIDs(final int numberOfCursorIDs) {
this.numberOfCursorIDs = numberOfCursorIDs;
}
public List<Long> getCursorIDs() {
return this.cursorIDs;
}
public void setCursorIDs(final List<Long> cursorIDs) {
this.cursorIDs = cursorIDs;
}
@Override
public String toString() {
final StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("messageHeader:").append(this.messageHeader);
builder.append(", numberOfCursorIDs:").append(this.numberOfCursorIDs);
builder.append(", cursorIDs:").append(this.cursorIDs);
builder.append("}");
return builder.toString();
}
@Override
public String toPrettyString() {
return this.toString();
}
}
@@ -1,106 +1,93 @@
/**
* JMongo is a mongodb driver writtern in java.
* Copyright (C) 2010 Xiaohu Huang
*
* JMongo is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* JMongo is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
*/
package com.fangjia.datastore.mongo.protocol;
import java.io.IOException;
import java.io.Serializable;
import java.util.concurrent.atomic.AtomicInteger;
import com.fangjia.datastore.mongo.codec.BSONInputStream;
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
import java.io.IOException;
import com.fangjia.datastore.mongo.codec.BSONInputStream;
import java.util.concurrent.atomic.AtomicInteger;
import java.io.Serializable;
public final class MessageHeader implements Writable, Readable, Serializable {
private static final long serialVersionUID = 8505270142383450097L;
private static final AtomicInteger ID_GENERATOR = new AtomicInteger(1);
private int messageLength;
private int requestID;
private int responseTo;
private OperationCode opCode;
public MessageHeader(OperationCode opCode) {
this.opCode = opCode;
this.requestID = ID_GENERATOR.getAndIncrement();
}
public MessageHeader() {
}
public void read(BSONInputStream in) throws IOException {
messageLength = in.readInteger();
requestID = in.readInteger();
responseTo = in.readInteger();
opCode = OperationCode.valueOf(in.readInteger());
}
public void write(BSONOutputStream out) throws IOException {
out.writeInteger(messageLength);
out.writeInteger(requestID);
out.writeInteger(responseTo);
out.writeInteger(opCode.getValue());
}
public int size() {
return 16;
}
public int getMessageLength() {
return messageLength;
}
public void setMessageLength(int messageLength) {
this.messageLength = messageLength;
}
public int getRequestID() {
return requestID;
}
public void setRequestID(int requestID) {
this.requestID = requestID;
}
public int getResponseTo() {
return responseTo;
}
public void setResponseTo(int responseTo) {
this.responseTo = responseTo;
}
public OperationCode getOpCode() {
return opCode;
}
public void setOpCode(OperationCode opCode) {
this.opCode = opCode;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("messageLength:").append(messageLength);
builder.append(", requestID:").append(requestID);
builder.append(", responseTo:").append(responseTo);
builder.append(", opCode:").append(opCode);
builder.append("}");
return builder.toString();
}
public final class MessageHeader implements Writable, Readable, Serializable
{
private static final long serialVersionUID = 8505270142383450097L;
private static final AtomicInteger ID_GENERATOR;
private int messageLength;
private int requestID;
private int responseTo;
private OperationCode opCode;
public MessageHeader(final OperationCode opCode) {
this.opCode = opCode;
this.requestID = MessageHeader.ID_GENERATOR.getAndIncrement();
}
public MessageHeader() {
}
@Override
public void read(final BSONInputStream in) throws IOException {
this.messageLength = in.readInteger();
this.requestID = in.readInteger();
this.responseTo = in.readInteger();
this.opCode = OperationCode.valueOf(in.readInteger());
}
@Override
public void write(final BSONOutputStream out) throws IOException {
out.writeInteger(this.messageLength);
out.writeInteger(this.requestID);
out.writeInteger(this.responseTo);
out.writeInteger(this.opCode.getValue());
}
public int size() {
return 16;
}
public int getMessageLength() {
return this.messageLength;
}
public void setMessageLength(final int messageLength) {
this.messageLength = messageLength;
}
public int getRequestID() {
return this.requestID;
}
public void setRequestID(final int requestID) {
this.requestID = requestID;
}
public int getResponseTo() {
return this.responseTo;
}
public void setResponseTo(final int responseTo) {
this.responseTo = responseTo;
}
public OperationCode getOpCode() {
return this.opCode;
}
public void setOpCode(final OperationCode opCode) {
this.opCode = opCode;
}
@Override
public String toString() {
final StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("messageLength:").append(this.messageLength);
builder.append(", requestID:").append(this.requestID);
builder.append(", responseTo:").append(this.responseTo);
builder.append(", opCode:").append(this.opCode);
builder.append("}");
return builder.toString();
}
static {
ID_GENERATOR = new AtomicInteger(1);
}
}
@@ -1,26 +1,10 @@
/**
* JMongo is a mongodb driver writtern in java.
* Copyright (C) 2010 Xiaohu Huang
*
* JMongo is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* JMongo is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
*/
package com.fangjia.datastore.mongo.protocol;
import java.io.Serializable;
public interface MongoMessage extends Readable, Writable, Serializable {
public MessageHeader getMessageHeader();
public String toPrettyString();
public interface MongoMessage extends Readable, Writable, Serializable
{
MessageHeader getMessageHeader();
String toPrettyString();
}
@@ -1,82 +1,39 @@
/**
* JMongo is a mongodb driver writtern in java.
* Copyright (C) 2010 Xiaohu Huang
*
* JMongo is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* JMongo is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
*/
package com.fangjia.datastore.mongo.protocol;
import java.util.HashMap;
import java.util.Map;
public enum OperationCode {
/**
* Reply to a client request. responseTo is set
*/
OP_REPLY(1),
/**
* generic msg command followed by a string
*/
OP_MSG(1000),
/**
* update document
*/
OP_UPDATE(2001),
/**
* insert new document
*/
OP_INSERT(2002),
/**
* formerly used for OP_GET_BY_OID
*/
RESERVED(2003),
/**
* query a collection
*/
OP_QUERY(2004),
/**
* Get more data from a query. See Cursors
*/
OP_GET_MORE(2005),
/**
* Delete documents
*/
OP_DELETE(2006),
/**
* Tell database client is done with a cursor
*/
OP_KILL_CURSORS(2007);
private final int value;
private final static Map<Integer, OperationCode> TYPE_MAP = new HashMap<Integer, OperationCode>();
static {
for (OperationCode e : OperationCode.values()) {
TYPE_MAP.put(e.value, e);
}
}
private OperationCode(int value) {
this.value = value;
}
public int getValue() {
return value;
}
public static OperationCode valueOf(int value) {
return TYPE_MAP.get(value);
}
public enum OperationCode
{
OP_REPLY(1),
OP_MSG(1000),
OP_UPDATE(2001),
OP_INSERT(2002),
RESERVED(2003),
OP_QUERY(2004),
OP_GET_MORE(2005),
OP_DELETE(2006),
OP_KILL_CURSORS(2007);
private final int value;
private static final Map<Integer, OperationCode> TYPE_MAP;
private OperationCode(final int value) {
this.value = value;
}
public int getValue() {
return this.value;
}
public static OperationCode valueOf(final int value) {
return OperationCode.TYPE_MAP.get(value);
}
static {
TYPE_MAP = new HashMap<Integer, OperationCode>();
for (final OperationCode e : values()) {
OperationCode.TYPE_MAP.put(e.value, e);
}
}
}
@@ -1,213 +1,197 @@
/**
* JMongo is a mongodb driver writtern in java.
* Copyright (C) 2010 Xiaohu Huang
*
* JMongo is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* JMongo is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
*/
package com.fangjia.datastore.mongo.protocol;
import java.io.IOException;
import org.bson.BSONDecoder;
import org.bson.BSONEncoder;
import org.bson.BSONObject;
import org.bson.BasicBSONDecoder;
import com.fangjia.datastore.mongo.codec.BSONInputStream;
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
import com.fangjia.datastore.utils.BSONUtils;
import org.bson.BSONDecoder;
import java.io.InputStream;
import org.bson.BasicBSONDecoder;
import com.fangjia.datastore.mongo.codec.BSONInputStream;
import java.io.IOException;
import org.bson.BSONEncoder;
import org.bson.BasicBSONEncoder;
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
import org.bson.BSONObject;
public class QueryMessage implements MongoMessage {
private static final long serialVersionUID = -3350216587439425208L;
private MessageHeader messageHeader;
private int options;
private int numberToSkip;
private int numberToReturn;
private String fullCollectionName;
private BSONObject query;
private BSONObject returnFieldSelector;
public QueryMessage() {
messageHeader = new MessageHeader(OperationCode.OP_QUERY);
}
public void write(BSONOutputStream out) throws IOException {
messageHeader.write(out);
out.writeInteger(options);
out.writeCString(this.fullCollectionName);
out.writeInteger(numberToSkip);
out.writeInteger(numberToReturn);
BSONEncoder encoder = new BSONEncoder();
out.write(encoder.encode(query));
if (null != returnFieldSelector) {
out.write(encoder.encode(returnFieldSelector));
}
out.set(0, out.size());
}
public void read(BSONInputStream in) throws IOException {
messageHeader.read(in);
options = in.readInteger();
this.fullCollectionName = in.readCString();
this.numberToSkip = in.readInteger();
this.numberToReturn = in.readInteger();
BSONDecoder decoder = new BasicBSONDecoder();
query = decoder.readObject(in);
if (in.available() > 0) {
returnFieldSelector = decoder.readObject(in);
}
}
public MessageHeader getMessageHeader() {
return messageHeader;
}
public String getFullCollectionName() {
return fullCollectionName;
}
public void setFullCollectionName(String fullCollectionName) {
this.fullCollectionName = fullCollectionName;
}
public int getOptions() {
return options;
}
public void setOptions(int options) {
this.options = options;
}
public void setNoCursorTimeout(boolean b) {
this.options = (int) BSONUtils.bitSet(options, 4, b);
}
public void setTailable(boolean b) {
this.options = (int) BSONUtils.bitSet(options, 1, b);
}
public void setSlaveOk(boolean b) {
this.options = (int) BSONUtils.bitSet(options, 2, b);
}
public void setAwaitData(boolean b) {
this.options = (int) BSONUtils.bitSet(options, 5, b);
}
public int getNumberToSkip() {
return numberToSkip;
}
public void setNumberToSkip(int numberToSkip) {
this.numberToSkip = numberToSkip;
}
public int getNumberToReturn() {
return numberToReturn;
}
public void setNumberToReturn(int numberToReturn) {
this.numberToReturn = numberToReturn;
}
public BSONObject getQuery() {
return query;
}
public void setQuery(BSONObject query) {
this.query = query;
}
public BSONObject getRealQuery() {
if (null == query) {
return null;
}
BSONObject q = (BSONObject) query.get("$query");
if (null == q) {
q = (BSONObject) query.get("query");
if (null == q) {
q = query;
}
}
return q;
}
public BSONObject getOrderByQuery() {
if (null == query) {
return null;
}
BSONObject q = (BSONObject) query.get("$orderby");
if (null == q) {
q = (BSONObject) query.get("orderby");
}
return q;
}
public BSONObject getFacetQuery() {
BSONObject q = getRealQuery();
if (null == q) {
return null;
}
return (BSONObject) q.get("$facet");
}
public boolean isBulkQuery() {
BSONObject q = getRealQuery();
if (null == q) {
return true;
}
Boolean bulk = (Boolean) q.get("$bulk");
return null == bulk ? true : bulk;
}
public boolean isFromMaster() {
BSONObject q = getRealQuery();
if (null == q) {
return false;
}
Boolean master = (Boolean) q.get("$master");
return null == master ? false : master;
}
public BSONObject getReturnFieldSelector() {
return returnFieldSelector;
}
public void setReturnFieldSelector(BSONObject returnFieldSelector) {
this.returnFieldSelector = returnFieldSelector;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("messageHeader:").append(messageHeader);
builder.append(", options:").append(options);
builder.append(", numberToSkip:").append(numberToSkip);
builder.append(", numberToReturn:").append(numberToReturn);
builder.append(", fullCollectionName:").append(fullCollectionName);
builder.append(", query:").append(query);
builder.append(", returnFieldSelector:").append(returnFieldSelector);
builder.append("}");
return builder.toString();
}
public String toPrettyString() {
return toString();
}
public class QueryMessage implements MongoMessage
{
private static final long serialVersionUID = -3350216587439425208L;
private MessageHeader messageHeader;
private int options;
private int numberToSkip;
private int numberToReturn;
private String fullCollectionName;
private BSONObject query;
private BSONObject returnFieldSelector;
public QueryMessage() {
this.messageHeader = new MessageHeader(OperationCode.OP_QUERY);
}
@Override
public void write(final BSONOutputStream out) throws IOException {
this.messageHeader.write(out);
out.writeInteger(this.options);
out.writeCString(this.fullCollectionName);
out.writeInteger(this.numberToSkip);
out.writeInteger(this.numberToReturn);
final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder();
out.write(encoder.encode(this.query));
if (null != this.returnFieldSelector) {
out.write(encoder.encode(this.returnFieldSelector));
}
out.set(0, out.size());
}
@Override
public void read(final BSONInputStream in) throws IOException {
this.messageHeader.read(in);
this.options = in.readInteger();
this.fullCollectionName = in.readCString();
this.numberToSkip = in.readInteger();
this.numberToReturn = in.readInteger();
final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder();
this.query = decoder.readObject((InputStream)in);
if (in.available() > 0) {
this.returnFieldSelector = decoder.readObject((InputStream)in);
}
}
@Override
public MessageHeader getMessageHeader() {
return this.messageHeader;
}
public String getFullCollectionName() {
return this.fullCollectionName;
}
public void setFullCollectionName(final String fullCollectionName) {
this.fullCollectionName = fullCollectionName;
}
public int getOptions() {
return this.options;
}
public void setOptions(final int options) {
this.options = options;
}
public void setNoCursorTimeout(final boolean b) {
this.options = (int)BSONUtils.bitSet(this.options, 4, b);
}
public void setTailable(final boolean b) {
this.options = (int)BSONUtils.bitSet(this.options, 1, b);
}
public void setSlaveOk(final boolean b) {
this.options = (int)BSONUtils.bitSet(this.options, 2, b);
}
public void setAwaitData(final boolean b) {
this.options = (int)BSONUtils.bitSet(this.options, 5, b);
}
public int getNumberToSkip() {
return this.numberToSkip;
}
public void setNumberToSkip(final int numberToSkip) {
this.numberToSkip = numberToSkip;
}
public int getNumberToReturn() {
return this.numberToReturn;
}
public void setNumberToReturn(final int numberToReturn) {
this.numberToReturn = numberToReturn;
}
public BSONObject getQuery() {
return this.query;
}
public void setQuery(final BSONObject query) {
this.query = query;
}
public BSONObject getRealQuery() {
if (null == this.query) {
return null;
}
BSONObject q = (BSONObject)this.query.get("$query");
if (null == q) {
q = (BSONObject)this.query.get("query");
if (null == q) {
q = this.query;
}
}
return q;
}
public BSONObject getOrderByQuery() {
if (null == this.query) {
return null;
}
BSONObject q = (BSONObject)this.query.get("$orderby");
if (null == q) {
q = (BSONObject)this.query.get("orderby");
}
return q;
}
public BSONObject getFacetQuery() {
final BSONObject q = this.getRealQuery();
if (null == q) {
return null;
}
return (BSONObject)q.get("$facet");
}
public boolean isBulkQuery() {
final BSONObject q = this.getRealQuery();
if (null == q) {
return true;
}
final Boolean bulk = (Boolean)q.get("$bulk");
return null == bulk || bulk;
}
public boolean isFromMaster() {
final BSONObject q = this.getRealQuery();
if (null == q) {
return false;
}
final Boolean master = (Boolean)q.get("$master");
return null != master && master;
}
public BSONObject getReturnFieldSelector() {
return this.returnFieldSelector;
}
public void setReturnFieldSelector(final BSONObject returnFieldSelector) {
this.returnFieldSelector = returnFieldSelector;
}
@Override
public String toString() {
final StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("messageHeader:").append(this.messageHeader);
builder.append(", options:").append(this.options);
builder.append(", numberToSkip:").append(this.numberToSkip);
builder.append(", numberToReturn:").append(this.numberToReturn);
builder.append(", fullCollectionName:").append(this.fullCollectionName);
builder.append(", query:").append(this.query);
builder.append(", returnFieldSelector:").append(this.returnFieldSelector);
builder.append("}");
return builder.toString();
}
@Override
public String toPrettyString() {
return this.toString();
}
}
@@ -1,26 +1,9 @@
/**
* JMongo is a mongodb driver writtern in java.
* Copyright (C) 2010 Xiaohu Huang
*
* JMongo is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* JMongo is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
*/
package com.fangjia.datastore.mongo.protocol;
import java.io.IOException;
import com.fangjia.datastore.mongo.codec.BSONInputStream;
public interface Readable {
public void read(BSONInputStream in) throws IOException;
public interface Readable
{
void read(final BSONInputStream p0) throws IOException;
}
@@ -1,138 +1,125 @@
/**
* JMongo is a mongodb driver writtern in java.
* Copyright (C) 2010 Xiaohu Huang
*
* JMongo is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* JMongo is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
*/
package com.fangjia.datastore.mongo.protocol;
import java.io.IOException;
import org.bson.BSONDecoder;
import java.io.InputStream;
import org.bson.BasicBSONDecoder;
import java.util.ArrayList;
import com.fangjia.datastore.mongo.codec.BSONInputStream;
import java.io.IOException;
import java.util.Iterator;
import org.bson.BSONEncoder;
import org.bson.BasicBSONEncoder;
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
import org.bson.BSONObject;
import java.util.List;
import org.bson.BSONDecoder;
import org.bson.BSONEncoder;
import org.bson.BSONObject;
import org.bson.BasicBSONDecoder;
import com.fangjia.datastore.mongo.codec.BSONInputStream;
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
public class ReplyMessage implements MongoMessage {
private static final long serialVersionUID = -3350216587439425208L;
private MessageHeader messageHeader;
private int responseFlag;
private long cursorID;
private int startingFrom;
private int numberReturned;
private List<BSONObject> documents;
public ReplyMessage() {
messageHeader = new MessageHeader(OperationCode.OP_REPLY);
}
public void write(BSONOutputStream out) throws IOException {
messageHeader.write(out);
out.writeInteger(responseFlag);
out.writeLong(cursorID);
out.writeInteger(startingFrom);
out.writeInteger(numberReturned);
if (null != this.documents) {
BSONEncoder encoder = new BSONEncoder();
for (BSONObject doc : this.documents) {
out.write(encoder.encode(doc));
}
}
out.set(0, out.size());
}
public void read(BSONInputStream in) throws IOException {
messageHeader.read(in);
responseFlag = in.readInteger();
cursorID = in.readLong();
startingFrom = in.readInteger();
numberReturned = in.readInteger();
documents = new ArrayList<BSONObject>(numberReturned);
if (numberReturned > 0) {
BSONDecoder decoder = new BasicBSONDecoder();
for (int i = 0; i < numberReturned; i++) {
documents.add(decoder.readObject(in));
}
}
}
public MessageHeader getMessageHeader() {
return messageHeader;
}
public int getResponseFlag() {
return responseFlag;
}
public void setResponseFlag(int responseFlag) {
this.responseFlag = responseFlag;
}
public long getCursorID() {
return cursorID;
}
public void setCursorID(long cursorID) {
this.cursorID = cursorID;
}
public int getStartingFrom() {
return startingFrom;
}
public void setStartingFrom(int startingFrom) {
this.startingFrom = startingFrom;
}
public int getNumberReturned() {
return numberReturned;
}
public void setNumberReturned(int numberReturned) {
this.numberReturned = numberReturned;
}
public List<BSONObject> getDocuments() {
return documents;
}
public void setDocuments(List<BSONObject> documents) {
this.documents = documents;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("messageHeader:").append(messageHeader);
builder.append(", responseFlag:").append(responseFlag);
builder.append(", cursorID:").append(cursorID);
builder.append(", startingFrom:").append(startingFrom);
builder.append(", numberReturned:").append(numberReturned);
builder.append(", documents:").append(documents);
builder.append("}");
return builder.toString();
}
public String toPrettyString() {
return toString();
}
public class ReplyMessage implements MongoMessage
{
private static final long serialVersionUID = -3350216587439425208L;
private MessageHeader messageHeader;
private int responseFlag;
private long cursorID;
private int startingFrom;
private int numberReturned;
private List<BSONObject> documents;
public ReplyMessage() {
this.messageHeader = new MessageHeader(OperationCode.OP_REPLY);
}
@Override
public void write(final BSONOutputStream out) throws IOException {
this.messageHeader.write(out);
out.writeInteger(this.responseFlag);
out.writeLong(this.cursorID);
out.writeInteger(this.startingFrom);
out.writeInteger(this.numberReturned);
if (null != this.documents) {
final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder();
for (final BSONObject doc : this.documents) {
out.write(encoder.encode(doc));
}
}
out.set(0, out.size());
}
@Override
public void read(final BSONInputStream in) throws IOException {
this.messageHeader.read(in);
this.responseFlag = in.readInteger();
this.cursorID = in.readLong();
this.startingFrom = in.readInteger();
this.numberReturned = in.readInteger();
this.documents = new ArrayList<BSONObject>(this.numberReturned);
if (this.numberReturned > 0) {
final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder();
for (int i = 0; i < this.numberReturned; ++i) {
this.documents.add(decoder.readObject((InputStream)in));
}
}
}
@Override
public MessageHeader getMessageHeader() {
return this.messageHeader;
}
public int getResponseFlag() {
return this.responseFlag;
}
public void setResponseFlag(final int responseFlag) {
this.responseFlag = responseFlag;
}
public long getCursorID() {
return this.cursorID;
}
public void setCursorID(final long cursorID) {
this.cursorID = cursorID;
}
public int getStartingFrom() {
return this.startingFrom;
}
public void setStartingFrom(final int startingFrom) {
this.startingFrom = startingFrom;
}
public int getNumberReturned() {
return this.numberReturned;
}
public void setNumberReturned(final int numberReturned) {
this.numberReturned = numberReturned;
}
public List<BSONObject> getDocuments() {
return this.documents;
}
public void setDocuments(final List<BSONObject> documents) {
this.documents = documents;
}
@Override
public String toString() {
final StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("messageHeader:").append(this.messageHeader);
builder.append(", responseFlag:").append(this.responseFlag);
builder.append(", cursorID:").append(this.cursorID);
builder.append(", startingFrom:").append(this.startingFrom);
builder.append(", numberReturned:").append(this.numberReturned);
builder.append(", documents:").append(this.documents);
builder.append("}");
return builder.toString();
}
@Override
public String toPrettyString() {
return this.toString();
}
}
@@ -1,137 +1,123 @@
/**
* JMongo is a mongodb driver writtern in java.
* Copyright (C) 2010 Xiaohu Huang
*
* JMongo is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* JMongo is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
*/
package com.fangjia.datastore.mongo.protocol;
import java.io.IOException;
import org.bson.BSONDecoder;
import org.bson.BSONEncoder;
import org.bson.BSONObject;
import org.bson.BasicBSONDecoder;
import org.bson.BasicBSONObject;
import com.fangjia.datastore.mongo.codec.BSONInputStream;
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
import com.fangjia.datastore.utils.BSONUtils;
import org.bson.BSONDecoder;
import java.io.InputStream;
import org.bson.BasicBSONDecoder;
import com.fangjia.datastore.mongo.codec.BSONInputStream;
import java.io.IOException;
import org.bson.BSONEncoder;
import org.bson.BasicBSONEncoder;
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
import org.bson.BasicBSONObject;
import org.bson.BSONObject;
public class UpdateMessage implements MongoMessage {
private static final long serialVersionUID = -5278293474222136247L;
private MessageHeader messageHeader;
private String fullCollectionName;
private int flags;
private BSONObject selector;
private BSONObject update;
public UpdateMessage() {
messageHeader = new MessageHeader(OperationCode.OP_UPDATE);
this.selector = new BasicBSONObject();
this.update = new BasicBSONObject();
}
public void write(BSONOutputStream out) throws IOException {
messageHeader.write(out);
out.writeInteger(0);
out.writeCString(this.fullCollectionName);
out.writeInteger(flags);
BSONEncoder encoder = new BSONEncoder();
out.write(encoder.encode(selector));
out.write(encoder.encode(update));
out.set(0, out.size());
}
public void read(BSONInputStream in) throws IOException {
messageHeader.read(in);
in.readInteger();
this.fullCollectionName = in.readCString();
this.flags = in.readInteger();
BSONDecoder decoder = new BasicBSONDecoder();
selector = decoder.readObject(in);
update = decoder.readObject(in);
}
public void setUpsert(boolean upsert) {
this.flags = (int) BSONUtils.bitSet(this.flags, 0, upsert);
}
public boolean isUpsert() {
return BSONUtils.isBitSet(flags, 0);
}
public void setMultiUpdate(boolean multiUpdate) {
this.flags = (int) BSONUtils.bitSet(this.flags, 1, multiUpdate);
}
public boolean isMultiUpdate() {
return BSONUtils.isBitSet(flags, 1);
}
public MessageHeader getMessageHeader() {
return messageHeader;
}
public String getFullCollectionName() {
return fullCollectionName;
}
public void setFullCollectionName(String fullCollectionName) {
this.fullCollectionName = fullCollectionName;
}
public int getFlags() {
return flags;
}
public void setFlags(int flags) {
this.flags = flags;
}
public BSONObject getSelector() {
return selector;
}
public void setSelector(BSONObject selector) {
this.selector = selector;
}
public BSONObject getUpdate() {
return update;
}
public void setUpdate(BSONObject update) {
this.update = update;
}
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("messageHeader:").append(messageHeader);
builder.append(", fullCollectionName:").append(fullCollectionName);
builder.append(", flags:").append(flags);
builder.append(", selector:").append(selector);
builder.append(", update:").append(update);
builder.append("}");
return builder.toString();
}
public String toPrettyString() {
return toString();
}
public class UpdateMessage implements MongoMessage
{
private static final long serialVersionUID = -5278293474222136247L;
private MessageHeader messageHeader;
private String fullCollectionName;
private int flags;
private BSONObject selector;
private BSONObject update;
public UpdateMessage() {
this.messageHeader = new MessageHeader(OperationCode.OP_UPDATE);
this.selector = (BSONObject)new BasicBSONObject();
this.update = (BSONObject)new BasicBSONObject();
}
@Override
public void write(final BSONOutputStream out) throws IOException {
this.messageHeader.write(out);
out.writeInteger(0);
out.writeCString(this.fullCollectionName);
out.writeInteger(this.flags);
final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder();
out.write(encoder.encode(this.selector));
out.write(encoder.encode(this.update));
out.set(0, out.size());
}
@Override
public void read(final BSONInputStream in) throws IOException {
this.messageHeader.read(in);
in.readInteger();
this.fullCollectionName = in.readCString();
this.flags = in.readInteger();
final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder();
this.selector = decoder.readObject((InputStream)in);
this.update = decoder.readObject((InputStream)in);
}
public void setUpsert(final boolean upsert) {
this.flags = (int)BSONUtils.bitSet(this.flags, 0, upsert);
}
public boolean isUpsert() {
return BSONUtils.isBitSet(this.flags, 0);
}
public void setMultiUpdate(final boolean multiUpdate) {
this.flags = (int)BSONUtils.bitSet(this.flags, 1, multiUpdate);
}
public boolean isMultiUpdate() {
return BSONUtils.isBitSet(this.flags, 1);
}
@Override
public MessageHeader getMessageHeader() {
return this.messageHeader;
}
public String getFullCollectionName() {
return this.fullCollectionName;
}
public void setFullCollectionName(final String fullCollectionName) {
this.fullCollectionName = fullCollectionName;
}
public int getFlags() {
return this.flags;
}
public void setFlags(final int flags) {
this.flags = flags;
}
public BSONObject getSelector() {
return this.selector;
}
public void setSelector(final BSONObject selector) {
this.selector = selector;
}
public BSONObject getUpdate() {
return this.update;
}
public void setUpdate(final BSONObject update) {
this.update = update;
}
@Override
public String toString() {
final StringBuilder builder = new StringBuilder();
builder.append("{");
builder.append("messageHeader:").append(this.messageHeader);
builder.append(", fullCollectionName:").append(this.fullCollectionName);
builder.append(", flags:").append(this.flags);
builder.append(", selector:").append(this.selector);
builder.append(", update:").append(this.update);
builder.append("}");
return builder.toString();
}
@Override
public String toPrettyString() {
return this.toString();
}
}
@@ -1,26 +1,9 @@
/**
* JMongo is a mongodb driver writtern in java.
* Copyright (C) 2010 Xiaohu Huang
*
* JMongo is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* JMongo is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
*/
package com.fangjia.datastore.mongo.protocol;
import java.io.IOException;
import com.fangjia.datastore.mongo.codec.BSONOutputStream;
public interface Writable {
public void write(BSONOutputStream out) throws IOException;
public interface Writable
{
void write(final BSONOutputStream p0) throws IOException;
}
@@ -1,11 +1,7 @@
package com.fangjia.datastore.solr;
import java.io.IOException;
import java.net.MalformedURLException;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.solr.client.solrj.SolrRequest;
@@ -15,66 +11,61 @@ import org.apache.solr.common.util.NamedList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class CommonsHttpSolrServer extends HttpSolrServer {
private static final long serialVersionUID = -3881911436089005421L;
import java.io.IOException;
import java.net.MalformedURLException;
private final static Logger LOGGER = LoggerFactory.getLogger(CommonsHttpSolrServer.class);
public class CommonsHttpSolrServer extends HttpSolrServer
{
private static final long serialVersionUID = -3881911436089005421L;
private static final Logger LOGGER;
private final HttpSolrClient solrServer;
private final String host;
private final String coreName;
public CommonsHttpSolrServer(final String host, final String coreName, final HttpClient httpClient) throws MalformedURLException {
this.host = host;
this.coreName = coreName;
(this.solrServer = new HttpSolrClient("http://" + host + '/' + coreName)).setAllowCompression(true);
}
@Override
public String getHost() {
return this.host;
}
@Override
public String getCoreName() {
return this.coreName;
}
@Override
public void reloadCore() throws SolrException {
try {
final HttpGet method = new HttpGet("http://" + this.host + "/admin/cores?action=RELOAD&core=" + this.coreName);
final HttpResponse response = this.solrServer.getHttpClient().execute(method);
if (response.getStatusLine().getStatusCode() != 200) {
throw new SolrException("[" + this.coreName + "] error reload core of solr[" + this.host + "]: " + IOUtils.toString(response.getEntity().getContent()));
}
if (LOGGER.isInfoEnabled()) {
LOGGER.info("[" + this.coreName + "] solr core of server[" + this.host + "] reloaded.");
}
if (LOGGER.isInfoEnabled()) {
LOGGER.info("[" + this.coreName + "] core reload success");
}
}
catch (IOException e) {
throw new SolrException("[" + this.coreName + "] error reload core", e);
}
}
private final HttpSolrClient solrServer;
private final String host;
private final String coreName;
public NamedList<Object> request(final SolrRequest request, final String arg1) throws SolrServerException, IOException {
return this.solrServer.request(request);
}
public CommonsHttpSolrServer(String host, String coreName, HttpClient httpClient) throws MalformedURLException {
this.host = host;
this.coreName = coreName;
this.solrServer = new HttpSolrClient("http://" + host + '/' + coreName);
this.solrServer.setAllowCompression(true);
}
@Override
public String getHost() {
return host;
}
@Override
public String getCoreName() {
return coreName;
}
@Override
public void reloadCore() throws SolrException {
try {
HttpGet method = new HttpGet("http://" + host + "/admin/cores?action=RELOAD&core=" + this.coreName);
HttpResponse response = solrServer.getHttpClient().execute(method);
if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
throw new SolrException("[" + this.coreName + "] error reload core of solr[" + host + "]: "
+ IOUtils.toString(response.getEntity().getContent()));
} else if (LOGGER.isInfoEnabled()) {
LOGGER.info("[" + this.coreName + "] solr core of server[" + host + "] reloaded.");
}
if (LOGGER.isInfoEnabled()) {
LOGGER.info("[" + this.coreName + "] core reload success");
}
} catch (IOException e) {
throw new SolrException("[" + this.coreName + "] error reload core", e);
}
}
/*@Override
public NamedList<Object> request(SolrRequest request) throws SolrServerException, IOException {
return solrServer.request(request);
}*/
@Override
public NamedList<Object> request(SolrRequest request, String arg1)
throws SolrServerException, IOException {
return solrServer.request(request);
}
@Override
public void shutdown() {
}
public void shutdown() {
}
static {
LOGGER = LoggerFactory.getLogger(CommonsHttpSolrServer.class);
}
}
@@ -1,138 +1,32 @@
package com.fangjia.datastore.solr;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import org.apache.commons.lang.time.FastDateFormat;
import org.bson.BSONObject;
import org.bson.types.ObjectId;
import com.fangjia.datastore.QueryResponse;
import org.bson.types.ObjectId;
import java.util.List;
import org.bson.BSONObject;
import java.util.Iterator;
import java.util.Collection;
import org.apache.commons.lang.time.FastDateFormat;
import com.fangjia.datastore.config.Reloadable;
/**
* 一个Core对应一个solr索引
*
* @author velna
*
*/
public interface Core extends Reloadable {
public final static int INDEX_BATCH_SIZE = 30000;
public final static FastDateFormat VERSION_DATE_FORMAT = FastDateFormat.getInstance("yyMMddHHmmssSSS");
/**
* 得到索引名称
*
* @return
*/
public String getName();
/**
* 得到所有索引字段
*
* @return
* @throws SolrException
*/
Collection<Index> getIndexes() throws SolrException;
/**
* 添加索引字段,必要的话重建索引
*
* @param indexes
* 添加的索引字段列表
* @param iterator
* 用于遍历数据
* @throws SolrException
*/
void ensureIndex(Collection<Index> indexes, Iterator<? extends BSONObject> iterator) throws SolrException;
/**
* 添加索引字段,必要的话重建索引
*
* @param index
* 添加的索引字段列表
* @param iterator
* 用于遍历数据
* @throws SolrException
*/
void ensureIndex(Index index, Iterator<? extends BSONObject> iterator) throws SolrException;
/**
* 重建索引
*
* @param objIdList
* 待重建的ObjIdList,全部重建时为null
* @param iterator
* 用于遍历数据
*/
void reIndex(List<ObjectId> objIdList, Iterator<? extends BSONObject> iterator) throws SolrException;
/**
* 保存索引文档
*
* @param doc
* @throws SolrException
*/
// void save(BSONObject doc) throws SolrException;
/**
* 保存多个索引文档
*
* @param docList
* @throws SolrException
*/
// void save(List<? extends BSONObject> docList) throws SolrException;
/**
* 保存多个索引文档
*
* @param iterator
* @throws SolrException
*/
void save(Iterator<? extends BSONObject> iterator) throws SolrException;
/**
* 根据ID删除索引文档
*
* @param id
* @throws SolrException
*/
// void deleteById(String id) throws SolrException;
/**
* 根据ID列表删除索引文档
*
* @param ids
* @throws SolrException
*/
// void deleteById(List<String> ids) throws SolrException;
/**
* 根据查询删除索引文档
*
* @param query
* @throws SolrException
*/
void deleteByQuery(BSONObject query) throws SolrException;
/**
* 查询索引
*
* @param query
* 查询条件
* @param sort
* 排序条件
* @param facet
* facet查询条件
* @param skip
* 从第skip条开始返回结果
* @param limit
* 返回limit条结果
* @param fromMaster
* 是否从主索引中读取数据
* @param bulk
* 是否使用bulk查询
* @return
* @throws SolrException
*/
QueryResponse find(BSONObject query, BSONObject sort, BSONObject facet, int skip, int limit, boolean fromMaster, boolean bulk)
throws SolrException;
public interface Core extends Reloadable
{
public static final int INDEX_BATCH_SIZE = 30000;
public static final FastDateFormat VERSION_DATE_FORMAT = FastDateFormat.getInstance("yyMMddHHmmssSSS");
String getName();
Collection<Index> getIndexes() throws SolrException;
void ensureIndex(final Collection<Index> p0, final Iterator<? extends BSONObject> p1) throws SolrException;
void ensureIndex(final Index p0, final Iterator<? extends BSONObject> p1) throws SolrException;
void reIndex(final List<ObjectId> p0, final Iterator<? extends BSONObject> p1) throws SolrException;
void save(final Iterator<? extends BSONObject> p0) throws SolrException;
void deleteByQuery(final BSONObject p0) throws SolrException;
QueryResponse find(final BSONObject p0, final BSONObject p1, final BSONObject p2, final int p3, final int p4, final boolean p5, final boolean p6) throws SolrException;
}
@@ -1,485 +1,480 @@
package com.fangjia.datastore.solr;
import com.fangjia.datastore.utils.LoggerFactory;
import com.fangjia.datastore.TriggerExecuteException;
import com.fangjia.datastore.config.Trigger;
import com.fangjia.datastore.TriggerExecutor;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.common.SolrDocument;
import java.util.ArrayList;
import com.fangjia.datastore.QueryResponse;
import org.apache.commons.lang.StringUtils;
import com.mongodb.BasicDBObject;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.commons.collections.CollectionUtils;
import org.bson.types.ObjectId;
import java.util.List;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Executor;
import java.util.concurrent.LinkedBlockingQueue;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrQuery.ORDER;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SpatialParams;
import org.bson.BSONObject;
import org.bson.types.ObjectId;
import org.slf4j.Logger;
import org.sothis.core.util.BlockingThreadPoolExecutor;
import com.fangjia.datastore.ConfigurationException;
import com.fangjia.datastore.QueryResponse;
import com.fangjia.datastore.TriggerExecuteException;
import com.fangjia.datastore.TriggerExecutor;
import com.fangjia.datastore.config.ServerConfig;
import com.fangjia.datastore.config.StoreConfig;
import com.fangjia.datastore.config.Trigger;
import java.util.concurrent.LinkedBlockingQueue;
import org.sothis.core.util.BlockingThreadPoolExecutor;
import java.util.concurrent.Executor;
import org.bson.BSONObject;
import java.util.concurrent.BlockingQueue;
import com.fangjia.datastore.utils.AsyncOperation;
import com.fangjia.datastore.utils.LoggerFactory;
import com.mongodb.BasicDBObject;
import com.fangjia.datastore.config.StoreConfig;
import com.fangjia.datastore.config.ServerConfig;
import org.slf4j.Logger;
public class CoreImpl implements Core {
private final static Logger LOGGER = LoggerFactory.getLogger(CoreImpl.class);
private final static Logger PERFORMANCE_LOGGER = LoggerFactory.getPerformanceLogger(CoreImpl.class);
private final static int TRIIGER_POOL_SIZE = Integer.getInteger("datastore.triggerPoolSize", 10);
public class CoreImpl implements Core
{
private static final Logger LOGGER;
private static final Logger PERFORMANCE_LOGGER;
private static final int TRIIGER_POOL_SIZE;
private final String name;
private final SolrServer solrServer;
private final ServerConfig serverConfig;
private final StoreConfig storeConfig;
private final AsyncOperation asynchOperation;
private final BlockingQueue<BSONObject> solrAddQueue;
private final BlockingQueue<BSONObject> triggerQueue;
private final Executor triggerExecutorService;
public CoreImpl(final StoreConfig storeConfig) throws ConfigurationException {
this.name = storeConfig.getName();
this.storeConfig = storeConfig;
this.solrServer = new SolrServerImpl(this.storeConfig);
this.serverConfig = storeConfig.getDataStoreConfig().getServerConfig();
if (CoreImpl.LOGGER.isInfoEnabled()) {
CoreImpl.LOGGER.info("init solr core: " + this.name + ", storeConfig: " + storeConfig);
}
this.asynchOperation = AsyncOperation.getInstance();
this.triggerExecutorService = new BlockingThreadPoolExecutor(this.name + "-Trigger", 1, CoreImpl.TRIIGER_POOL_SIZE, CoreImpl.TRIIGER_POOL_SIZE, 60000L);
this.solrAddQueue = new LinkedBlockingQueue<BSONObject>(60000);
final Thread solrIndexAddThread = new Thread(new SolrAddRunnable(), this.name + "-SolrAdd");
solrIndexAddThread.setDaemon(true);
solrIndexAddThread.start();
this.triggerQueue = new LinkedBlockingQueue<BSONObject>(this.serverConfig.getTriggerBatchSize() * 4);
final Thread triggerExecuteThread = new Thread(new TriggerRunnable(), this.name + "-Trigger");
triggerExecuteThread.setDaemon(true);
triggerExecuteThread.start();
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
while (!CoreImpl.this.solrAddQueue.isEmpty()) {
CoreImpl.LOGGER.info(CoreImpl.this.name + ": wait for solr queue, current size: " + CoreImpl.this.solrAddQueue.size());
try {
Thread.sleep(1000L);
}
catch (InterruptedException e) {
CoreImpl.LOGGER.error("", (Throwable)e);
}
}
CoreImpl.LOGGER.info(CoreImpl.this.name + ": shutdown.");
}
});
if (CoreImpl.LOGGER.isInfoEnabled()) {
CoreImpl.LOGGER.info("solr core " + this.name + " initialized");
}
}
private void saveToSolr(final Iterator<? extends BSONObject> iterator, final String operationName) throws SolrException {
long c = 0L;
final long startTime = System.currentTimeMillis();
while (iterator.hasNext()) {
try {
this.triggerQueue.put(iterator.next());
}
catch (InterruptedException e) {
throw new SolrException("error put object: ", e);
}
++c;
if (null != operationName && c % this.serverConfig.getTriggerBatchSize() == 0L && CoreImpl.LOGGER.isInfoEnabled()) {
CoreImpl.LOGGER.info("{}: {}, {}/s", new Object[] { operationName, c, (int)(c / ((System.currentTimeMillis() - startTime) / 1000.0)) });
}
}
}
@Override
public void reload() throws ConfigurationException {
this.solrServer.reload();
try {
this.solrServer.reloadCores();
}
catch (SolrException e) {
throw new ConfigurationException(e);
}
}
@Override
public Collection<Index> getIndexes() throws SolrException {
return this.solrServer.getSolrSchema().getIndexes();
}
@Override
public void ensureIndex(final Collection<Index> indexes, final Iterator<? extends BSONObject> iterator) throws SolrException {
try {
final String operationName = "ensureIndex_" + System.currentTimeMillis();
if (CoreImpl.LOGGER.isInfoEnabled()) {
CoreImpl.LOGGER.info("{}: ensure index[{}] {} ...", new Object[] { this.getName(), operationName, indexes });
}
if (this.solrServer.getSolrSchema().saveIndex(indexes) > 0) {
this.solrServer.reloadCores();
}
this.saveToSolr(iterator, operationName);
if (CoreImpl.LOGGER.isInfoEnabled()) {
CoreImpl.LOGGER.info("{}: ensure index[{}] {} finished", new Object[] { this.getName(), operationName, indexes });
}
}
catch (SolrException e) {
CoreImpl.LOGGER.error("ensureIndex: ", (Throwable)e);
}
}
@Override
public void ensureIndex(final Index index, final Iterator<? extends BSONObject> iterator) throws SolrException {
this.ensureIndex(Arrays.<Index>asList(index), iterator);
}
@Override
public void reIndex(final List<ObjectId> objIdList, final Iterator<? extends BSONObject> iterator) throws SolrException {
final boolean logInfo = CollectionUtils.isEmpty((Collection)objIdList);
final long startTime = System.currentTimeMillis();
final String operationName = logInfo ? (this.getName() + "_re-index_" + startTime) : null;
if (logInfo && CoreImpl.LOGGER.isInfoEnabled()) {
CoreImpl.LOGGER.info(operationName + " started.");
}
this.saveToSolr(iterator, operationName);
if (logInfo && CoreImpl.LOGGER.isInfoEnabled()) {
CoreImpl.LOGGER.info("{} finished, total time: {}s.", operationName, (System.currentTimeMillis() - startTime) / 1000L);
}
}
@Override
public void save(final Iterator<? extends BSONObject> iterator) throws SolrException {
this.asynchOperation.run(new Runnable() {
@Override
public void run() {
try {
CoreImpl.this.saveToSolr(iterator, "save_" + System.currentTimeMillis());
}
catch (SolrException e) {
CoreImpl.LOGGER.error(CoreImpl.this.name + ": error save objects into solr:", (Throwable)e);
}
}
});
}
@Override
public void deleteByQuery(final BSONObject query) throws SolrException {
final String solrQuery = this.toSolrQuery(query).get("q");
if (CoreImpl.LOGGER.isDebugEnabled()) {
CoreImpl.LOGGER.debug("deleteByQuery: {}", (Object)solrQuery);
}
this.solrServer.deleteByQuery(solrQuery);
}
private SolrQuery toSolrQuery(final BSONObject query) throws SolrException {
return this.toSolrQuery(query, false);
}
private SolrQuery toSolrQuery(final BSONObject orgQuery, final boolean useFilterQuery) throws SolrException {
System.err.println(orgQuery.toString());
final BSONObject query = (BSONObject)new BasicDBObject();
query.putAll(orgQuery);
final SolrQueryBuilder queryBuilder = new SolrQueryBuilder();
final SolrQuery solrQuery = new SolrQuery();
if (useFilterQuery && query != null && query.containsField("visible") && query.containsField("city") && query.containsField("cat") && query.get("cat") instanceof String) {
final StringBuilder fq = new StringBuilder();
fq.append("+cat:").append(query.get("cat"));
fq.append(" +city:").append(query.get("city"));
fq.append(" +visible:").append(((Number)query.get("visible")).intValue());
solrQuery.addFilterQuery(fq.toString());
query.removeField("cat");
query.removeField("city");
query.removeField("visible");
}
for (final String key : query.keySet()) {
if (key.charAt(0) == '$') {
continue;
}
final Object value = query.get(key);
final Index index = this.assertFieldExistence(key);
if (value instanceof BSONObject) {
final BSONObject subQuery = (BSONObject)value;
Object g = subQuery.get("$gt");
boolean ge = false;
if (null == g) {
g = subQuery.get("$gte");
ge = (g != null);
}
Object l = subQuery.get("$lt");
boolean le = false;
if (null == l) {
l = subQuery.get("$lte");
le = (l != null);
}
if (ge && null == l) {
le = true;
}
if (le && null == g) {
ge = true;
}
if (null != g || null != l) {
queryBuilder.range(key, index.normalizeValue(g), index.normalizeValue(l), ge, le);
}
final Object in = subQuery.get("$in");
if (null != in) {
queryBuilder.in(key, index.normalizeValue((List<Object>)in));
}
final Object ne = subQuery.get("$ne");
if (null == ne) {
continue;
}
queryBuilder.ne(key, index.normalizeValue(ne));
}
else {
queryBuilder.eq(key, index.normalizeValue(value));
}
}
String q = queryBuilder.toQuery().toString();
if (query.get("$sr") != null) {
final Object value2 = query.get("$sr");
final BSONObject bsonObject = (BSONObject)value2;
final String val = bsonObject.get("sr.query").toString();
if (val != null && !"".equals(val)) {
q = q + " +(" + val + ")";
}
}
if (query.get("$sd") != null) {
final Object value2 = query.get("$sd");
final BSONObject bsonObject = (BSONObject)value2;
final String distance = (String)bsonObject.get("sd.distance");
final String field = (String)bsonObject.get("sd.sfield");
final String point = (String)bsonObject.get("sd.point");
if (!StringUtils.isNotBlank(point) || !StringUtils.isNotBlank(distance) || !StringUtils.isNotBlank(field)) {
throw new IllegalArgumentException("paramName $sd value illegal : " + value2);
}
solrQuery.setParam("sfield", field);
solrQuery.setParam("d", distance);
solrQuery.setParam("pt", point);
solrQuery.addSort("geodist()", SolrQuery.ORDER.asc);
solrQuery.addFilterQuery("{!geofilt}");
}
if (query.get("$fangjiawang_query_type") != null) {
final Object value2 = query.get("$fangjiawang_query_type");
solrQuery.add("q.op", value2.toString());
}
if (StringUtils.isEmpty(q)) {
solrQuery.setQuery("*:*");
}
else {
solrQuery.setQuery(q);
}
return solrQuery;
}
private Index assertFieldExistence(final String key) throws SolrException {
final Index index = this.solrServer.getSolrSchema().getIndex(key);
if (null == index) {
throw new SolrException("index '" + key + "' is not exist!");
}
return index;
}
@Override
public QueryResponse find(final BSONObject query, final BSONObject sort, final BSONObject facet, int skip, int limit, final boolean fromMaster, final boolean bulk) throws SolrException {
final long start = System.currentTimeMillis();
if (limit < 0) {
limit = this.serverConfig.getDefaultCursorLimit();
}
if (skip < 0) {
skip = 0;
}
final SolrQuery solrQuery = this.toSolrQuery(query, true);
solrQuery.setFields("_id");
solrQuery.setStart(skip);
solrQuery.setRows(limit);
if (null != sort) {
for (final String key : sort.keySet()) {
final int value = ((Number)sort.get(key)).intValue();
if (!"score".equals(key)) {
this.assertFieldExistence(key);
}
solrQuery.addSort(key, (value > 0) ? SolrQuery.ORDER.asc : SolrQuery.ORDER.desc);
}
}
if (null != facet) {
final List<String> fields = (List<String>)facet.get("fields");
if (null != fields) {
solrQuery.setFacet(true);
Integer facetMinCount = null;
if (null != facet.get("minCount")) {
facetMinCount = ((Number)facet.get("minCount")).intValue();
}
solrQuery.setFacetMinCount((null == facetMinCount) ? 1 : ((int)facetMinCount));
Integer facetLimit = null;
if (null != facet.get("limit")) {
facetLimit = ((Number)facet.get("limit")).intValue();
}
solrQuery.setFacetLimit((null == facetLimit) ? 100 : facetLimit);
for (final String field : fields) {
this.assertFieldExistence(field);
solrQuery.addFacetField(field);
}
}
}
final org.apache.solr.client.solrj.response.QueryResponse queryResponse = this.solrServer.query(solrQuery, fromMaster, bulk);
final long time = System.currentTimeMillis() - start;
if (time > 100L) {
String host = solrQuery.get("__host");
solrQuery.remove("__host");
final StringBuilder sb = new StringBuilder();
sb.append("query solr: ").append(solrQuery);
if(StringUtils.isNotBlank(host)){
sb.append(", host: ").append(host);
}
sb.append(", time: ").append(time);
sb.append(", qtime: ").append(queryResponse.getQTime());
sb.append(", numFound: ").append(queryResponse.getResults().getNumFound());
sb.append(", numReturned: ").append(queryResponse.getResults().size());
sb.append(", master: ").append(fromMaster);
CoreImpl.PERFORMANCE_LOGGER.info(sb.toString());
if (time < 1000L) {
CoreImpl.PERFORMANCE_LOGGER.info(sb.toString());
}
else if (time < 3000L) {
CoreImpl.PERFORMANCE_LOGGER.warn(sb.toString());
}
else {
CoreImpl.PERFORMANCE_LOGGER.error(sb.toString());
}
}
final QueryResponse response = this.toSolrResponse(queryResponse);
return response;
}
private QueryResponse toSolrResponse(final org.apache.solr.client.solrj.response.QueryResponse response) {
if (response == null) {
return QueryResponse.EMPTY_RESPONSE;
}
final SolrDocumentList solrDocumentList = response.getResults();
final int numReturned = solrDocumentList.size();
final List<ObjectId> idList = new ArrayList<ObjectId>(numReturned);
for (int i = 0; i < numReturned; ++i) {
final String value = ((SolrDocument)solrDocumentList.get(i)).getFirstValue("_id").toString();
idList.add(new ObjectId(value));
}
final QueryResponse solrResponse = new QueryResponse();
solrResponse.setNumFound((int)solrDocumentList.getNumFound());
solrResponse.setNumReturned(numReturned);
solrResponse.setNumToSkip((int)solrDocumentList.getStart());
solrResponse.setIdList(idList);
solrResponse.setFacetFieldList(response.getFacetFields());
return solrResponse;
}
@Override
public String getName() {
return this.name;
}
private List<BSONObject> executeTrigger(final List<BSONObject> list) throws TriggerExecuteException {
final Trigger trigger = this.storeConfig.getTrigger();
if (null == trigger) {
if (CoreImpl.LOGGER.isWarnEnabled()) {
CoreImpl.LOGGER.warn("null trigger!");
}
return list;
}
if (null == trigger.getInterface()) {
return list;
}
return TriggerExecutor.getInstance().execute(trigger, list);
}
static {
LOGGER = LoggerFactory.getLogger(CoreImpl.class);
PERFORMANCE_LOGGER = LoggerFactory.getPerformanceLogger(CoreImpl.class);
TRIIGER_POOL_SIZE = Integer.getInteger("datastore.triggerPoolSize", 10);
}
private final String name;
private final SolrServer solrServer;
private final ServerConfig serverConfig;
private final StoreConfig storeConfig;
private final AsyncOperation asynchOperation;
private final BlockingQueue<BSONObject> solrAddQueue;
private final BlockingQueue<BSONObject> triggerQueue;
private final Executor triggerExecutorService;
private class TriggerRunnable implements Runnable {
public CoreImpl(StoreConfig storeConfig) throws ConfigurationException {
this.name = storeConfig.getName();
this.storeConfig = storeConfig;
this.solrServer = new SolrServerImpl(this.storeConfig);
this.serverConfig = storeConfig.getDataStoreConfig().getServerConfig();
if (LOGGER.isInfoEnabled()) {
LOGGER.info("init solr core: " + name + ", storeConfig: " + storeConfig);
}
public void run() {
int sleepTime = 0;
while (true) {
try {
final List<BSONObject> dbObjects = new ArrayList<BSONObject>(serverConfig.getTriggerBatchSize());
triggerQueue.drainTo(dbObjects, serverConfig.getTriggerBatchSize());
if (!dbObjects.isEmpty()) {
triggerExecutorService.execute(new Runnable() {
this.asynchOperation = AsyncOperation.getInstance();
public void run() {
try {
long time = System.currentTimeMillis();
List<BSONObject> list = executeTrigger(dbObjects);
time = System.currentTimeMillis() - time;
if (LOGGER.isInfoEnabled() && time > 0) {
LOGGER.info("execute trigger: {}/{} objects, {}ms, {}/s", new Object[] { list.size(),
dbObjects.size(), time, dbObjects.size() * 1000 / time });
}
for (BSONObject obj : list) {
solrAddQueue.put(obj);
}
} catch (Exception e) {
LOGGER.error("error execute trigger: ", e);
}
}
// trigger executor
this.triggerExecutorService = new BlockingThreadPoolExecutor(name + "-Trigger", 1, TRIIGER_POOL_SIZE, TRIIGER_POOL_SIZE,
60000);
});
sleepTime = serverConfig.getTriggerBatchSize() - dbObjects.size();
if (sleepTime > 100) {
Thread.sleep(sleepTime * 3);
}
} else {
Thread.sleep(serverConfig.getTriggerBatchSize());
}
} catch (Exception e) {
LOGGER.error("error execute trigger: ", e);
}
}
}
// solr add thread
this.solrAddQueue = new LinkedBlockingQueue<BSONObject>(INDEX_BATCH_SIZE * 2);
Thread solrIndexAddThread = new Thread(new SolrAddRunnable(), name + "-SolrAdd");
solrIndexAddThread.setDaemon(true);
solrIndexAddThread.start();
// trigger thread
this.triggerQueue = new LinkedBlockingQueue<BSONObject>(this.serverConfig.getTriggerBatchSize() * 4);
Thread triggerExecuteThread = new Thread(new TriggerRunnable(), name + "-Trigger");
triggerExecuteThread.setDaemon(true);
triggerExecuteThread.start();
// shutdown hook
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
while (!solrAddQueue.isEmpty()) {
LOGGER.info(CoreImpl.this.name + ": wait for solr queue, current size: " + solrAddQueue.size());
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
LOGGER.error("", e);
}
}
LOGGER.info(CoreImpl.this.name + ": shutdown.");
}
});
if (LOGGER.isInfoEnabled()) {
LOGGER.info("solr core " + name + " initialized");
}
}
private class TriggerRunnable implements Runnable {
public void run() {
int sleepTime = 0;
while (true) {
try {
final List<BSONObject> dbObjects = new ArrayList<BSONObject>(serverConfig.getTriggerBatchSize());
triggerQueue.drainTo(dbObjects, serverConfig.getTriggerBatchSize());
if (!dbObjects.isEmpty()) {
triggerExecutorService.execute(new Runnable() {
public void run() {
try {
long time = System.currentTimeMillis();
List<BSONObject> list = executeTrigger(dbObjects);
time = System.currentTimeMillis() - time;
if (LOGGER.isInfoEnabled() && time > 0) {
LOGGER.info("execute trigger: {}/{} objects, {}ms, {}/s", new Object[] { list.size(),
dbObjects.size(), time, dbObjects.size() * 1000 / time });
}
for (BSONObject obj : list) {
solrAddQueue.put(obj);
}
} catch (Exception e) {
LOGGER.error("error execute trigger: ", e);
}
}
});
sleepTime = serverConfig.getTriggerBatchSize() - dbObjects.size();
if (sleepTime > 100) {
Thread.sleep(sleepTime * 3);
}
} else {
Thread.sleep(serverConfig.getTriggerBatchSize());
}
} catch (Exception e) {
LOGGER.error("error execute trigger: ", e);
}
}
}
}
private class SolrAddRunnable implements Runnable {
public void run() {
final List<BSONObject> dbObjects = new ArrayList<BSONObject>(INDEX_BATCH_SIZE);
final int maxSleepTime = 10000;
int sleepTime = 0;
while (true) {
try {
solrAddQueue.drainTo(dbObjects, INDEX_BATCH_SIZE);
if (!dbObjects.isEmpty()) {
solrServer.add(dbObjects);
sleepTime = maxSleepTime - dbObjects.size();
if (sleepTime > 100) {
Thread.sleep(sleepTime);
}
} else {
Thread.sleep(maxSleepTime);
}
} catch (Exception e) {
LOGGER.error("error save to solr: ", e);
} finally {
dbObjects.clear();
}
}
}
}
private void saveToSolr(final Iterator<? extends BSONObject> iterator, String operationName) throws SolrException {
long c = 0;
long startTime = System.currentTimeMillis();
while (iterator.hasNext()) {
try {
triggerQueue.put(iterator.next());
} catch (InterruptedException e) {
throw new SolrException("error put object: ", e);
}
c++;
if (null != operationName && c % serverConfig.getTriggerBatchSize() == 0 && LOGGER.isInfoEnabled()) {
LOGGER.info("{}: {}, {}/s", new Object[] { operationName, c,
(int) (c / ((System.currentTimeMillis() - startTime) / 1000.0)) });
}
}
}
public void reload() throws ConfigurationException {
this.solrServer.reload();
try {
this.solrServer.reloadCores();
} catch (SolrException e) {
throw new ConfigurationException(e);
}
}
public Collection<Index> getIndexes() throws SolrException {
return this.solrServer.getSolrSchema().getIndexes();
}
public void ensureIndex(final Collection<Index> indexes, final Iterator<? extends BSONObject> iterator) throws SolrException {
try {
String operationName = "ensureIndex_" + System.currentTimeMillis();
if (LOGGER.isInfoEnabled()) {
LOGGER.info("{}: ensure index[{}] {} ...", new Object[] { CoreImpl.this.getName(), operationName, indexes });
}
if (solrServer.getSolrSchema().saveIndex(indexes) > 0) {
this.solrServer.reloadCores();
}
saveToSolr(iterator, operationName);
if (LOGGER.isInfoEnabled()) {
LOGGER.info("{}: ensure index[{}] {} finished", new Object[] { CoreImpl.this.getName(), operationName, indexes });
}
} catch (SolrException e) {
LOGGER.error("ensureIndex: ", e);
}
}
public void ensureIndex(Index index, Iterator<? extends BSONObject> iterator) throws SolrException {
ensureIndex(Arrays.asList(index), iterator);
}
public void reIndex(final List<ObjectId> objIdList, final Iterator<? extends BSONObject> iterator) throws SolrException {
boolean logInfo = CollectionUtils.isEmpty(objIdList);
long startTime = System.currentTimeMillis();
String operationName = logInfo ? CoreImpl.this.getName() + "_re-index_" + startTime : null;
if (logInfo && LOGGER.isInfoEnabled()) {
LOGGER.info(operationName + " started.");
}
saveToSolr(iterator, operationName);
if (logInfo && LOGGER.isInfoEnabled()) {
LOGGER.info("{} finished, total time: {}s.", operationName, (System.currentTimeMillis() - startTime) / 1000);
}
}
public void save(final Iterator<? extends BSONObject> iterator) throws SolrException {
asynchOperation.run(new Runnable() {
public void run() {
try {
saveToSolr(iterator, "save_" + System.currentTimeMillis());
} catch (SolrException e) {
LOGGER.error(name + ": error save objects into solr:", e);
}
}
});
}
public void deleteByQuery(final BSONObject query) throws SolrException {
String solrQuery = toSolrQuery(query).get(CommonParams.Q);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("deleteByQuery: {}", solrQuery);
}
this.solrServer.deleteByQuery(solrQuery);
}
private SolrQuery toSolrQuery(BSONObject query) throws SolrException {
return toSolrQuery(query, false);
}
private SolrQuery toSolrQuery(final BSONObject orgQuery, boolean useFilterQuery) throws SolrException {
System.err.println(orgQuery.toString());
BSONObject query = new BasicDBObject();
query.putAll(orgQuery);
SolrQueryBuilder queryBuilder = new SolrQueryBuilder();
SolrQuery solrQuery = new SolrQuery();
if (useFilterQuery && query != null && query.containsField("visible") && query.containsField("city")
&& query.containsField("cat") && query.get("cat") instanceof String) {
StringBuilder fq = new StringBuilder();
fq.append("+cat:").append(query.get("cat"));
fq.append(" +city:").append(query.get("city"));
fq.append(" +visible:").append(((Number) query.get("visible")).intValue());
solrQuery.addFilterQuery(fq.toString());
query.removeField("cat");
query.removeField("city");
query.removeField("visible");
}
if (null != query) {
for (String key : query.keySet()) {
if (key.charAt(0) == '$') {
continue;
}
Object value = query.get(key);
Index index = assertFieldExistence(key);
if (value instanceof BSONObject) {
BSONObject subQuery = (BSONObject) value;
Object g = subQuery.get("$gt");
boolean ge = false;
if (null == g) {
g = subQuery.get("$gte");
ge = g != null;
}
Object l = subQuery.get("$lt");
boolean le = false;
if (null == l) {
l = subQuery.get("$lte");
le = l != null;
}
if (ge && null == l) {
le = true;
}
if (le && null == g) {
ge = true;
}
if (null != g || null != l) {
queryBuilder.range(key, index.normalizeValue(g), index.normalizeValue(l), ge, le);
}
Object in = subQuery.get("$in");
if (null != in) {
queryBuilder.in(key, index.normalizeValue((List) in));
}
Object ne = subQuery.get("$ne");
if (null != ne) {
queryBuilder.ne(key, index.normalizeValue(ne));
}
} else {
queryBuilder.eq(key, index.normalizeValue(value));
}
}
}
String q = queryBuilder.toQuery().toString();
// $srģʽ
if (null != query) {
if (query.get("$sr") != null) {
Object value = query.get("$sr");
BSONObject bsonObject = (BSONObject) value;
String val = bsonObject.get("sr.query").toString();
if (val != null && !"".equals(val)) {
q += " +(" + val + ")";
}
}
if (query.get("$sd") != null) {
Object value = query.get("$sd");
BSONObject bsonObject = (BSONObject) value;
String distance = (String) bsonObject.get("sd.distance");
String field = (String) bsonObject.get("sd.sfield");
String point = (String) bsonObject.get("sd.point");
if (StringUtils.isNotBlank(point) && StringUtils.isNotBlank(distance) && StringUtils.isNotBlank(field)) {
solrQuery.setParam(SpatialParams.FIELD, field);
solrQuery.setParam(SpatialParams.DISTANCE, distance);
solrQuery.setParam(SpatialParams.POINT, point);
solrQuery.addSort("geodist()", ORDER.asc);
solrQuery.addFilterQuery("{!geofilt}");
} else {
throw new IllegalArgumentException("paramName $sd value illegal : " + value);
}
}
if (query.get("$fangjiawang_query_type") != null) {
Object value = query.get("$fangjiawang_query_type");
solrQuery.add("q.op", value.toString());
}
}
if (StringUtils.isEmpty(q)) {
solrQuery.setQuery("*:*");
} else {
solrQuery.setQuery(q);
}
return solrQuery;
}
private Index assertFieldExistence(String key) throws SolrException {
Index index = this.solrServer.getSolrSchema().getIndex(key);
if (null == index) {
throw new SolrException("index '" + key + "' is not exist!");
}
return index;
}
public QueryResponse find(BSONObject query, BSONObject sort, BSONObject facet, int skip, int limit, boolean fromMaster,
boolean bulk) throws SolrException {
long start = System.currentTimeMillis();
if (limit < 0) {
limit = serverConfig.getDefaultCursorLimit();
}
if (skip < 0) {
skip = 0;
}
SolrQuery solrQuery = toSolrQuery(query, true);
// fields, start, limit
solrQuery.setFields(SolrSchema.ID);
solrQuery.setStart(skip);
solrQuery.setRows(limit);
// sort
if (null != sort) {
for (String key : sort.keySet()) {
int value = ((Number) sort.get(key)).intValue();
if (!"score".equals(key)) { // pseudo-field score
assertFieldExistence(key);
}
solrQuery.addSort(key, value > 0 ? ORDER.asc : ORDER.desc);
}
}
// facet
if (null != facet) {
List<String> fields = (List<String>) facet.get("fields");
if (null != fields) {
solrQuery.setFacet(true);
Integer facetMinCount = null;
if (null != facet.get("minCount")) {
facetMinCount = ((Number) facet.get("minCount")).intValue();
}
solrQuery.setFacetMinCount(null == facetMinCount ? 1 : facetMinCount);
Integer facetLimit = null;
if (null != facet.get("limit")) {
facetLimit = ((Number) facet.get("limit")).intValue();
}
solrQuery.setFacetLimit(null == facetLimit ? 100 : facetLimit);
for (String field : fields) {
assertFieldExistence(field);
solrQuery.addFacetField(field);
}
}
}
org.apache.solr.client.solrj.response.QueryResponse queryResponse = this.solrServer.query(solrQuery, fromMaster, bulk);
long time = System.currentTimeMillis() - start;
if (time > 100) {
StringBuilder sb = new StringBuilder();
sb.append("query solr: ").append(solrQuery);
sb.append(", time: ").append(time);
sb.append(", qtime: ").append(queryResponse.getQTime());
sb.append(", numFound: ").append(queryResponse.getResults().getNumFound());
sb.append(", numReturned: ").append(queryResponse.getResults().size());
sb.append(", master: ").append(fromMaster);
if (time < 1000) {
PERFORMANCE_LOGGER.info(sb.toString());
} else if (time < 3000) {
PERFORMANCE_LOGGER.warn(sb.toString());
} else {
PERFORMANCE_LOGGER.error(sb.toString());
}
}
QueryResponse response = toSolrResponse(queryResponse);
return response;
}
private QueryResponse toSolrResponse(org.apache.solr.client.solrj.response.QueryResponse response) {
if (response == null) {
return QueryResponse.EMPTY_RESPONSE;
}
SolrDocumentList solrDocumentList = response.getResults();
int numReturned = solrDocumentList.size();
List<ObjectId> idList = new ArrayList<ObjectId>(numReturned);
for (int i = 0; i < numReturned; i++) {
String value = solrDocumentList.get(i).getFirstValue(SolrSchema.ID).toString();
idList.add(new ObjectId(value));
}
QueryResponse solrResponse = new QueryResponse();
solrResponse.setNumFound((int) solrDocumentList.getNumFound());
solrResponse.setNumReturned(numReturned);
solrResponse.setNumToSkip((int) solrDocumentList.getStart());
solrResponse.setIdList(idList);
solrResponse.setFacetFieldList(response.getFacetFields());
return solrResponse;
}
public String getName() {
return name;
}
private List<BSONObject> executeTrigger(List<BSONObject> list) throws TriggerExecuteException {
Trigger trigger = this.storeConfig.getTrigger();
if (null == trigger) {
if (LOGGER.isWarnEnabled()) {
LOGGER.warn("null trigger!");
}
return list;
}
if (null == trigger.getInterface()) {
return list;
}
return TriggerExecutor.getInstance().execute(trigger, list);
}
}
private class SolrAddRunnable implements Runnable
{
@Override
public void run() {
final List<BSONObject> dbObjects = new ArrayList<BSONObject>(30000);
final int maxSleepTime = 10000;
int sleepTime = 0;
while (true) {
try {
CoreImpl.this.solrAddQueue.drainTo(dbObjects, 30000);
if (!dbObjects.isEmpty()) {
CoreImpl.this.solrServer.add(dbObjects);
sleepTime = 10000 - dbObjects.size();
if (sleepTime <= 100) {
continue;
}
Thread.sleep(sleepTime);
}
else {
Thread.sleep(10000L);
}
}
catch (Exception e) {
CoreImpl.LOGGER.error("error save to solr: ", (Throwable)e);
}
finally {
dbObjects.clear();
}
}
}
}
}
@@ -2,13 +2,13 @@ package com.fangjia.datastore.solr;
import org.apache.solr.client.solrj.SolrServer;
public abstract class HttpSolrServer extends SolrServer {
private static final long serialVersionUID = -3967599614339379847L;
abstract public void reloadCore() throws SolrException;
abstract public String getCoreName();
abstract public String getHost();
public abstract class HttpSolrServer extends SolrServer
{
private static final long serialVersionUID = -3967599614339379847L;
public abstract void reloadCore() throws SolrException;
public abstract String getCoreName();
public abstract String getHost();
}
@@ -1,191 +1,196 @@
package com.fangjia.datastore.solr;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.bson.BSONObject;
import org.bson.BasicBSONObject;
import org.bson.BSONObject;
import java.util.Date;
import java.util.Iterator;
import java.util.ArrayList;
import java.util.List;
public class Index {
public static enum IndexTypeEnum {
STRING, INTEGER, INT, DOUBLE, TEXT_STD, TEXT_CJK, DATE, LONG, LOCATION;
public String getType() {
return this.name().toLowerCase();
}
public static IndexTypeEnum parse(String type) {
for (IndexTypeEnum e : IndexTypeEnum.values()) {
if (e.getType().equals(type)) {
return e;
}
}
return null;
}
}
private String name;
private IndexTypeEnum type;
private boolean stored;
private boolean indexed;
private boolean multiValued;
private String comments;
private boolean dynamic;
public Index(String name, IndexTypeEnum type, boolean stored, boolean indexed, boolean multiValued, boolean dynamic,
String comments) {
this.name = name;
this.type = type;
this.stored = stored;
this.indexed = indexed;
this.multiValued = multiValued;
this.dynamic = dynamic;
this.comments = comments;
}
public Index(String name, IndexTypeEnum type) {
this.name = name;
this.type = type;
this.stored = false;
this.indexed = true;
this.multiValued = false;
this.dynamic = false;
this.comments = "";
}
public List<Object> normalizeValue(List<Object> values) {
if (null == values) {
return null;
}
List<Object> ret = new ArrayList<Object>(values.size());
for (Object v : values) {
ret.add(normalizeValue(v));
}
return ret;
}
public Object normalizeValue(Object value) {
if (null == value) {
return null;
}
switch (this.type) {
case STRING:
case TEXT_STD:
return value.toString();
case INTEGER:
return ((Number) value).intValue();
case LONG:
return ((Number) value).longValue();
case DOUBLE:
return ((Number) value).doubleValue();
case DATE:
return (Date) value;
}
return value;
}
public Index() {
this(null, null);
}
public String getName() {
return name;
}
public IndexTypeEnum getType() {
return type;
}
public boolean isStored() {
return stored;
}
public boolean isIndexed() {
return indexed;
}
public boolean isMultiValued() {
return multiValued;
}
public String getComments() {
return comments;
}
public boolean isDynamic() {
return dynamic;
}
public void setName(String name) {
this.name = name;
}
public void setType(IndexTypeEnum type) {
this.type = type;
}
public void setStored(boolean stored) {
this.stored = stored;
}
public void setIndexed(boolean indexed) {
this.indexed = indexed;
}
public void setMultiValued(boolean multiValued) {
this.multiValued = multiValued;
}
public void setComments(String comments) {
this.comments = comments;
}
public void setDynamic(boolean dynamic) {
this.dynamic = dynamic;
}
public BSONObject toBSONObject() {
BSONObject ret = new BasicBSONObject();
ret.put("name", name);
ret.put("type", this.type.getType());
ret.put("stored", this.stored);
ret.put("indexed", this.indexed);
ret.put("multiValued", this.multiValued);
ret.put("dynamic", this.dynamic);
ret.put("comments", this.comments);
return ret;
}
@Override
public int hashCode() {
return new HashCodeBuilder().append(this.dynamic).append(this.indexed).append(this.multiValued).append(this.stored)
.append(this.comments).append(this.name).append(this.type).toHashCode();
}
@Override
public boolean equals(Object obj) {
if (null == obj) {
return false;
}
if (this == obj) {
return true;
}
if (!(obj instanceof Index)) {
return false;
}
Index other = (Index) obj;
return new EqualsBuilder().append(this.dynamic, other.dynamic).append(this.indexed, other.indexed).append(
this.multiValued, other.multiValued).append(this.stored, other.stored).append(this.comments, other.comments)
.append(this.name, other.name).append(this.type, other.type).isEquals();
}
@Override
public String toString() {
return new StringBuilder().append("{name=").append(name).append(", type=").append(type).append(", stored=")
.append(stored).append(", indexed=").append(indexed).append(", multiValued=").append(multiValued).append(
", dynamic=").append(dynamic).append(", comments=").append(this.comments).append("}").toString();
}
public class Index
{
private String name;
private IndexTypeEnum type;
private boolean stored;
private boolean indexed;
private boolean multiValued;
private String comments;
private boolean dynamic;
public Index(final String name, final IndexTypeEnum type, final boolean stored, final boolean indexed, final boolean multiValued, final boolean dynamic, final String comments) {
this.name = name;
this.type = type;
this.stored = stored;
this.indexed = indexed;
this.multiValued = multiValued;
this.dynamic = dynamic;
this.comments = comments;
}
public Index(final String name, final IndexTypeEnum type) {
this.name = name;
this.type = type;
this.stored = false;
this.indexed = true;
this.multiValued = false;
this.dynamic = false;
this.comments = "";
}
public List<Object> normalizeValue(final List<Object> values) {
if (null == values) {
return null;
}
final List<Object> ret = new ArrayList<Object>(values.size());
for (final Object v : values) {
ret.add(this.normalizeValue(v));
}
return ret;
}
public Object normalizeValue(final Object value) {
if (null == value) {
return null;
}
switch (this.type) {
case STRING:
case TEXT_STD:
return value.toString();
case INTEGER:
return ((Number)value).intValue();
case LONG:
return ((Number)value).longValue();
case DOUBLE:
return ((Number)value).doubleValue();
case DATE:
return value;
default:
return value;
}
}
public Index() {
this(null, null);
}
public String getName() {
return this.name;
}
public IndexTypeEnum getType() {
return this.type;
}
public boolean isStored() {
return this.stored;
}
public boolean isIndexed() {
return this.indexed;
}
public boolean isMultiValued() {
return this.multiValued;
}
public String getComments() {
return this.comments;
}
public boolean isDynamic() {
return this.dynamic;
}
public void setName(final String name) {
this.name = name;
}
public void setType(final IndexTypeEnum type) {
this.type = type;
}
public void setStored(final boolean stored) {
this.stored = stored;
}
public void setIndexed(final boolean indexed) {
this.indexed = indexed;
}
public void setMultiValued(final boolean multiValued) {
this.multiValued = multiValued;
}
public void setComments(final String comments) {
this.comments = comments;
}
public void setDynamic(final boolean dynamic) {
this.dynamic = dynamic;
}
public BSONObject toBSONObject() {
final BSONObject ret = (BSONObject)new BasicBSONObject();
ret.put("name", (Object)this.name);
ret.put("type", (Object)this.type.getType());
ret.put("stored", (Object)this.stored);
ret.put("indexed", (Object)this.indexed);
ret.put("multiValued", (Object)this.multiValued);
ret.put("dynamic", (Object)this.dynamic);
ret.put("comments", (Object)this.comments);
return ret;
}
@Override
public int hashCode() {
return new HashCodeBuilder().append(this.dynamic).append(this.indexed).append(this.multiValued).append(this.stored).append((Object)this.comments).append((Object)this.name).append((Object)this.type).toHashCode();
}
@Override
public boolean equals(final Object obj) {
if (null == obj) {
return false;
}
if (this == obj) {
return true;
}
if (!(obj instanceof Index)) {
return false;
}
final Index other = (Index)obj;
return new EqualsBuilder().append(this.dynamic, other.dynamic).append(this.indexed, other.indexed).append(this.multiValued, other.multiValued).append(this.stored, other.stored).append((Object)this.comments, (Object)other.comments).append((Object)this.name, (Object)other.name).append((Object)this.type, (Object)other.type).isEquals();
}
@Override
public String toString() {
return "{name=" + this.name + ", type=" + this.type + ", stored=" + this.stored + ", indexed=" + this.indexed + ", multiValued=" + this.multiValued + ", dynamic=" + this.dynamic + ", comments=" + this.comments + "}";
}
public enum IndexTypeEnum
{
STRING,
INTEGER,
INT,
DOUBLE,
TEXT_STD,
TEXT_CJK,
DATE,
LONG,
LOCATION;
public String getType() {
return this.name().toLowerCase();
}
public static IndexTypeEnum parse(final String type) {
for (final IndexTypeEnum e : values()) {
if (e.getType().equals(type)) {
return e;
}
}
return null;
}
}
}
@@ -1,262 +1,223 @@
package com.fangjia.datastore.solr;
import java.io.IOException;
import java.lang.ref.WeakReference;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.RandomUtils;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import com.fangjia.datastore.utils.LoggerFactory;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.params.ShardParams;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.client.solrj.SolrRequest;
import java.util.concurrent.TimeUnit;
import java.lang.ref.WeakReference;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.Executors;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.params.SolrParams;
import java.util.Iterator;
import org.apache.commons.lang.math.RandomUtils;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.solr.client.solrj.SolrQuery;
import org.slf4j.Logger;
import com.fangjia.datastore.utils.LoggerFactory;
/**
*
* @author velna
* @see org.apache.solr.client.solrj.impl.LBHttpSolrServer
*/
public class LBHttpSolrServer extends HttpSolrServer {
private static final long serialVersionUID = -6175098554678462150L;
private final static Logger LOGGER = LoggerFactory.getLogger(LBHttpSolrServer.class);
private static final SolrQuery SOLR_QUERY = new SolrQuery("*:*");
private static final int CHECK_INTERVAL = 10 * 1000;
private final String coreName;
private final ReentrantLock checkLock = new ReentrantLock();
private ScheduledExecutorService aliveCheckExecutor;
private final CopyOnWriteArrayList<WrappedHttpSolrServer> aliveServers = new CopyOnWriteArrayList<WrappedHttpSolrServer>();
private final CopyOnWriteArrayList<WrappedHttpSolrServer> zombieServers = new CopyOnWriteArrayList<WrappedHttpSolrServer>();
public LBHttpSolrServer(HttpSolrServer... httpSolrServers) {
if (null == httpSolrServers || httpSolrServers.length == 0) {
throw new IllegalArgumentException("length of httpSolrServers must greater than 0");
}
for (HttpSolrServer httpSolrServer : httpSolrServers) {
this.aliveServers.add(new WrappedHttpSolrServer(httpSolrServer));
}
coreName = httpSolrServers[0].getCoreName();
}
private WrappedHttpSolrServer selectRandomServer() {
for (;;) {
if (aliveServers.isEmpty()) {
throw new RuntimeException("no solr server available, all are down!");
}
try {
if (aliveServers.size() == 1) {
return aliveServers.get(0);
}
return aliveServers.get(RandomUtils.nextInt(aliveServers.size()));
} catch (IndexOutOfBoundsException e) {
continue;
}
}
}
/*@Override
public NamedList<Object> request(final SolrRequest request) throws SolrServerException, IOException {
int attempts = 0;
int startSize = aliveServers.size();
Exception ex;
String zombieHost = null;
for (;;) {
WrappedHttpSolrServer httpSolrServer = selectRandomServer();
try {
if (null != zombieHost && request.getParams() instanceof SolrQuery) {
SolrQuery query = (SolrQuery) request.getParams();
String shards = query.get(ShardParams.SHARDS);
if (null != shards) {
shards = StringUtils.replace(shards, zombieHost, httpSolrServer.httpSolrServer.getHost());
query.set(ShardParams.SHARDS, shards);
}
}
return httpSolrServer.httpSolrServer.request(request);
} catch (SolrServerException e) {
if (e.getRootCause() instanceof IOException) {
ex = e;
zombieHost = httpSolrServer.httpSolrServer.getHost();
LOGGER.error("error request solr server: " + zombieHost, e);
moveAliveToDead(httpSolrServer);
} else {
throw e;
}
}
attempts++;
if (attempts >= startSize)
throw new SolrServerException("No live SolrServers available to handle this request", ex);
}
}*/
@Override
public String getHost() {
for (WrappedHttpSolrServer server : aliveServers) {
LOGGER.info("---"+server.httpSolrServer.getHost());
}
return selectRandomServer().httpSolrServer.getHost();
}
@Override
public String getCoreName() {
return coreName;
}
@Override
public void reloadCore() throws com.fangjia.datastore.solr.SolrException {
// does noting
}
private void moveAliveToDead(WrappedHttpSolrServer solrServer) {
checkLock.lock();
try {
boolean result = aliveServers.remove(solrServer);
if (result) {
if (zombieServers.addIfAbsent(solrServer)) {
if (LOGGER.isWarnEnabled()) {
LOGGER.warn("solr server [" + solrServer.httpSolrServer.getHost() + "] is marked as down.");
}
startAliveCheckExecutor();
}
}
} finally {
checkLock.unlock();
}
}
private void checkAZombieServer(WrappedHttpSolrServer zombieServer) {
checkLock.lock();
try {
if (LOGGER.isInfoEnabled()) {
LOGGER.info("check solr server [" + zombieServer.httpSolrServer.getHost() + "] ...");
}
QueryResponse resp = zombieServer.httpSolrServer.query(SOLR_QUERY);
if (resp.getStatus() == 0) {
// server has come back up
zombieServers.remove(zombieServer);
aliveServers.add(zombieServer);
if (LOGGER.isInfoEnabled()) {
LOGGER.info("solr server [" + zombieServer.httpSolrServer.getHost() + "] has come back.");
}
zombieServer.failedPings = 0;
}
} catch (Exception e) {
zombieServer.failedPings++;
// Expected . The server is still down
} finally {
checkLock.unlock();
}
}
private void startAliveCheckExecutor() {
if (aliveCheckExecutor == null) {
synchronized (this) {
if (aliveCheckExecutor == null) {
aliveCheckExecutor = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory());
aliveCheckExecutor.scheduleAtFixedRate(getAliveCheckRunner(new WeakReference<LBHttpSolrServer>(this)),
CHECK_INTERVAL, CHECK_INTERVAL, TimeUnit.MILLISECONDS);
}
}
}
}
private static Runnable getAliveCheckRunner(final WeakReference<LBHttpSolrServer> lbHttpSolrServer) {
return new Runnable() {
public void run() {
LBHttpSolrServer solrServer = lbHttpSolrServer.get();
if (solrServer != null && solrServer.zombieServers != null) {
for (WrappedHttpSolrServer zombieServer : solrServer.zombieServers) {
solrServer.checkAZombieServer(zombieServer);
}
}
}
};
}
/**
* The default thread factory
*/
static class DefaultThreadFactory implements ThreadFactory {
static final AtomicInteger poolNumber = new AtomicInteger(1);
final ThreadGroup group;
final AtomicInteger threadNumber = new AtomicInteger(1);
final String namePrefix;
DefaultThreadFactory() {
SecurityManager s = System.getSecurityManager();
group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup();
namePrefix = "solrcheck-" + poolNumber.getAndIncrement() + "-thread-";
}
public Thread newThread(Runnable r) {
Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0);
t.setDaemon(true);
if (t.getPriority() != Thread.NORM_PRIORITY)
t.setPriority(Thread.NORM_PRIORITY);
return t;
}
}
private class WrappedHttpSolrServer {
private final HttpSolrServer httpSolrServer;
private int failedPings = 0;
public WrappedHttpSolrServer(HttpSolrServer httpSolrServer) {
this.httpSolrServer = httpSolrServer;
}
}
@Override
public NamedList<Object> request(SolrRequest paramSolrRequest,
String paramString) throws SolrServerException, IOException {
int attempts = 0;
int startSize = aliveServers.size();
Exception ex;
String zombieHost = null;
for (;;) {
WrappedHttpSolrServer httpSolrServer = selectRandomServer();
try {
if (null != zombieHost && paramSolrRequest.getParams() instanceof SolrQuery) {
SolrQuery query = (SolrQuery) paramSolrRequest.getParams();
String shards = query.get(ShardParams.SHARDS);
if (null != shards) {
shards = StringUtils.replace(shards, zombieHost, httpSolrServer.httpSolrServer.getHost());
query.set(ShardParams.SHARDS, shards);
}
}
return httpSolrServer.httpSolrServer.request(paramSolrRequest);
} catch (SolrServerException e) {
if (e.getRootCause() instanceof IOException) {
ex = e;
zombieHost = httpSolrServer.httpSolrServer.getHost();
LOGGER.error("error request solr server: " + zombieHost, e);
moveAliveToDead(httpSolrServer);
} else {
throw e;
}
}
attempts++;
if (attempts >= startSize)
throw new SolrServerException("No live SolrServers available to handle this request", ex);
}
}
@Override
public void shutdown() {
// TODO Auto-generated method stub
}
}
public class LBHttpSolrServer extends HttpSolrServer
{
private static final long serialVersionUID = -6175098554678462150L;
private static final Logger LOGGER;
private static final SolrQuery SOLR_QUERY;
private static final int CHECK_INTERVAL = 10000;
private final String coreName;
private final ReentrantLock checkLock;
private ScheduledExecutorService aliveCheckExecutor;
private final CopyOnWriteArrayList<WrappedHttpSolrServer> aliveServers;
private final CopyOnWriteArrayList<WrappedHttpSolrServer> zombieServers;
public LBHttpSolrServer(final HttpSolrServer... httpSolrServers) {
this.checkLock = new ReentrantLock();
this.aliveServers = new CopyOnWriteArrayList<WrappedHttpSolrServer>();
this.zombieServers = new CopyOnWriteArrayList<WrappedHttpSolrServer>();
if (null == httpSolrServers || httpSolrServers.length == 0) {
throw new IllegalArgumentException("length of httpSolrServers must greater than 0");
}
for (final HttpSolrServer httpSolrServer : httpSolrServers) {
this.aliveServers.add(new WrappedHttpSolrServer(httpSolrServer));
}
this.coreName = httpSolrServers[0].getCoreName();
}
private WrappedHttpSolrServer selectRandomServer() {
while (!this.aliveServers.isEmpty()) {
try {
if (this.aliveServers.size() == 1) {
return this.aliveServers.get(0);
}
return this.aliveServers.get(RandomUtils.nextInt(this.aliveServers.size()));
}
catch (IndexOutOfBoundsException e) {}
}
throw new RuntimeException("no solr server available, all are down!");
}
@Override
public String getHost() {
for (final WrappedHttpSolrServer server : this.aliveServers) {
LBHttpSolrServer.LOGGER.info("---" + server.httpSolrServer.getHost());
}
return this.selectRandomServer().httpSolrServer.getHost();
}
@Override
public String getCoreName() {
return this.coreName;
}
@Override
public void reloadCore() throws SolrException {
}
private void moveAliveToDead(final WrappedHttpSolrServer solrServer) {
this.checkLock.lock();
try {
final boolean result = this.aliveServers.remove(solrServer);
if (result && this.zombieServers.addIfAbsent(solrServer)) {
if (LBHttpSolrServer.LOGGER.isWarnEnabled()) {
LBHttpSolrServer.LOGGER.warn("solr server [" + solrServer.httpSolrServer.getHost() + "] is marked as down.");
}
this.startAliveCheckExecutor();
}
}
finally {
this.checkLock.unlock();
}
}
private void checkAZombieServer(final WrappedHttpSolrServer zombieServer) {
this.checkLock.lock();
try {
if (LBHttpSolrServer.LOGGER.isInfoEnabled()) {
LBHttpSolrServer.LOGGER.info("check solr server [" + zombieServer.httpSolrServer.getHost() + "] ...");
}
final QueryResponse resp = zombieServer.httpSolrServer.query((SolrParams)LBHttpSolrServer.SOLR_QUERY);
if (resp.getStatus() == 0) {
this.zombieServers.remove(zombieServer);
this.aliveServers.add(zombieServer);
if (LBHttpSolrServer.LOGGER.isInfoEnabled()) {
LBHttpSolrServer.LOGGER.info("solr server [" + zombieServer.httpSolrServer.getHost() + "] has come back.");
}
zombieServer.failedPings = 0;
}
}
catch (Exception e) {
zombieServer.failedPings++;
}
finally {
this.checkLock.unlock();
}
}
private void startAliveCheckExecutor() {
if (this.aliveCheckExecutor == null) {
synchronized (this) {
if (this.aliveCheckExecutor == null) {
(this.aliveCheckExecutor = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory())).scheduleAtFixedRate(getAliveCheckRunner(new WeakReference<LBHttpSolrServer>(this)), 10000L, 10000L, TimeUnit.MILLISECONDS);
}
}
}
}
private static Runnable getAliveCheckRunner(final WeakReference<LBHttpSolrServer> lbHttpSolrServer) {
return new Runnable() {
@Override
public void run() {
final LBHttpSolrServer solrServer = (LBHttpSolrServer)lbHttpSolrServer.get();
if (solrServer != null && solrServer.zombieServers != null) {
for (final WrappedHttpSolrServer zombieServer : solrServer.zombieServers) {
solrServer.checkAZombieServer(zombieServer);
}
}
}
};
}
public NamedList<Object> request(final SolrRequest paramSolrRequest, final String paramString) throws SolrServerException, IOException {
int attempts = 0;
final int startSize = this.aliveServers.size();
String zombieHost = null;
while (true) {
final WrappedHttpSolrServer httpSolrServer = this.selectRandomServer();
try {
if (null != zombieHost && paramSolrRequest.getParams() instanceof SolrQuery) {
final SolrQuery query = (SolrQuery)paramSolrRequest.getParams();
String shards = query.get("shards");
if (null != shards) {
shards = StringUtils.replace(shards, zombieHost, httpSolrServer.httpSolrServer.getHost());
query.set("shards", shards);
}
}
return httpSolrServer.httpSolrServer.request(paramSolrRequest);
}
catch (SolrServerException e) {
if (!(e.getRootCause() instanceof IOException)) {
throw e;
}
final Exception ex = (Exception)e;
zombieHost = httpSolrServer.httpSolrServer.getHost();
LBHttpSolrServer.LOGGER.error("error request solr server: " + zombieHost, (Throwable)e);
this.moveAliveToDead(httpSolrServer);
if (++attempts >= startSize) {
throw new SolrServerException("No live SolrServers available to handle this request", (Throwable)ex);
}
continue;
}
}
}
public void shutdown() {
}
static {
LOGGER = LoggerFactory.getLogger(LBHttpSolrServer.class);
SOLR_QUERY = new SolrQuery("*:*");
}
static class DefaultThreadFactory implements ThreadFactory
{
static final AtomicInteger poolNumber;
final ThreadGroup group;
final AtomicInteger threadNumber;
final String namePrefix;
DefaultThreadFactory() {
this.threadNumber = new AtomicInteger(1);
final SecurityManager s = System.getSecurityManager();
this.group = ((s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup());
this.namePrefix = "solrcheck-" + DefaultThreadFactory.poolNumber.getAndIncrement() + "-thread-";
}
@Override
public Thread newThread(final Runnable r) {
final Thread t = new Thread(this.group, r, this.namePrefix + this.threadNumber.getAndIncrement(), 0L);
t.setDaemon(true);
if (t.getPriority() != 5) {
t.setPriority(5);
}
return t;
}
static {
poolNumber = new AtomicInteger(1);
}
}
private class WrappedHttpSolrServer
{
private final HttpSolrServer httpSolrServer;
private int failedPings;
public WrappedHttpSolrServer(final HttpSolrServer httpSolrServer) {
this.failedPings = 0;
this.httpSolrServer = httpSolrServer;
}
}
}
@@ -1,23 +1,21 @@
package com.fangjia.datastore.solr;
public class SolrException extends Exception {
private static final long serialVersionUID = -1105081175581071875L;
public SolrException() {
super();
}
public SolrException(String message, Throwable cause) {
super(message, cause);
}
public SolrException(String message) {
super(message);
}
public SolrException(Throwable cause) {
super(cause);
}
public class SolrException extends Exception
{
private static final long serialVersionUID = -1105081175581071875L;
public SolrException() {
}
public SolrException(final String message, final Throwable cause) {
super(message, cause);
}
public SolrException(final String message) {
super(message);
}
public SolrException(final Throwable cause) {
super(cause);
}
}
@@ -1,177 +1,176 @@
package com.fangjia.datastore.solr;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.TimeZone;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.time.FastDateFormat;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
import java.util.ArrayList;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.commons.lang.StringUtils;
import java.util.Iterator;
import java.util.List;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermRangeQuery;
import java.util.Date;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.BooleanClause;
import org.apache.commons.lang.time.FastDateFormat;
import org.apache.lucene.search.BooleanQuery;
public class SolrQueryBuilder {
private BooleanQuery booleanQuery = new BooleanQuery();
public final static FastDateFormat DATE_FORMAT = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", TimeZone
.getTimeZone("UTC"));
/**
* 约定:以$$开头的字符串不需要特殊字符转义
*/
public final static String NO_ESCAPSE = "$$";
public SolrQueryBuilder range(String field, Object min, Object max, boolean minInclusive, boolean maxInclusive,
BooleanClause.Occur occur) {
if (min != null || max != null) {
Object obj = null == min ? max : min;
Query query = null;
if (Integer.class.isInstance(obj)) {
query = NumericRangeQuery.newIntRange(field, (Integer) min, (Integer) max, minInclusive, maxInclusive);
} else if (Long.class.isInstance(obj)) {
query = NumericRangeQuery.newLongRange(field, (Long) min, (Long) max, minInclusive, maxInclusive);
} else if (Double.class.isInstance(obj)) {
query = NumericRangeQuery.newDoubleRange(field, (Double) min, (Double) max, minInclusive, maxInclusive);
} else if (Float.class.isInstance(obj)) {
query = NumericRangeQuery.newFloatRange(field, (Float) min, (Float) max, minInclusive, maxInclusive);
} else if (Date.class.isInstance(obj)) {
query = new TermRangeQuery(field, null == min ? null : DATE_FORMAT.format((Date) min), null == max ? null
: DATE_FORMAT.format((Date) max), minInclusive, maxInclusive);
} else {
query = new TermRangeQuery(field, null == min ? (String) min : min.toString(), null == max ? (String) max : max
.toString(), minInclusive, maxInclusive);
}
booleanQuery.add(query, occur);
}
return this;
}
public SolrQueryBuilder range(String field, Object min, Object max, boolean minInclusive, boolean maxInclusive) {
return range(field, min, max, minInclusive, maxInclusive, BooleanClause.Occur.MUST);
}
public SolrQueryBuilder range(String field, Object min, Object max) {
return range(field, min, max, true, true, BooleanClause.Occur.MUST);
}
public SolrQueryBuilder range(String field, Object min, Object max, BooleanClause.Occur occur) {
return range(field, min, max, true, true, occur);
}
public SolrQueryBuilder gt(String field, Object value) {
return range(field, value, null, false, true, BooleanClause.Occur.MUST);
}
public SolrQueryBuilder gt(String field, Object value, BooleanClause.Occur occur) {
return range(field, value, null, false, true, occur);
}
public SolrQueryBuilder gte(String field, Object value) {
return range(field, value, null, true, true, BooleanClause.Occur.MUST);
}
public SolrQueryBuilder gte(String field, Object value, BooleanClause.Occur occur) {
return range(field, value, null, true, true, occur);
}
public SolrQueryBuilder lt(String field, Object value) {
return range(field, null, value, true, false, BooleanClause.Occur.MUST);
}
public SolrQueryBuilder lt(String field, Object value, BooleanClause.Occur occur) {
return range(field, null, value, true, false, occur);
}
public SolrQueryBuilder lte(String field, Object value) {
return range(field, null, value, true, true, BooleanClause.Occur.MUST);
}
public SolrQueryBuilder lte(String field, Object value, BooleanClause.Occur occur) {
return range(field, null, value, true, true, occur);
}
public SolrQueryBuilder eq(String field, Object value, BooleanClause.Occur occur) {
if (null != value) {
Query query = new TermQuery(new Term(field, value.toString()));
booleanQuery.add(query, occur);
}
return this;
}
public SolrQueryBuilder eq(String field, Object value) {
return eq(field, getRealValue(value), BooleanClause.Occur.MUST);
}
public SolrQueryBuilder eq(String field, Object value, boolean escapse) {
return eq(field, escapse ? QueryParser.escape(value.toString()) : value.toString(), BooleanClause.Occur.MUST);
}
public SolrQueryBuilder ne(String field, Object value) {
return eq(field, getRealValue(value), BooleanClause.Occur.MUST_NOT);
}
public SolrQueryBuilder in(String field, List<Object> value, boolean escapse, BooleanClause.Occur occur) {
if (null == value || value.isEmpty()) {
throw new IllegalArgumentException("in query with null or empty list!!");
}
BooleanQuery bQuery = new BooleanQuery();
for (Object v : value) {
Query query = new TermQuery(new Term(field, escapse ? QueryParser.escape(v.toString()) : v.toString()));
bQuery.add(query, BooleanClause.Occur.SHOULD);
}
booleanQuery.add(bQuery, occur);
return this;
}
public SolrQueryBuilder in(String field, List<Object> value) {
return in(field, getRealValue(value), false, BooleanClause.Occur.MUST);
}
public SolrQueryBuilder in(String field, List<Object> value, BooleanClause.Occur occur) {
return in(field, getRealValue(value), false, occur);
}
public SolrQueryBuilder in(String field, List<Object> value, boolean escapse) {
return in(field, value, escapse, BooleanClause.Occur.MUST);
}
public Query toQuery() {
return this.booleanQuery;
}
/**
* 获取真实value<br/>
* 如果是非string的value,返回转义后的原值<br/>
* 如果是string型,如果以$$开头,则截取$$后面的字符且不需要转义
*
* @param value
* @return
*/
private Object getRealValue(Object value) {
if (value instanceof String) {
if (NO_ESCAPSE.equals(value)) {
return value;
} else if (StringUtils.startsWith((String) value, NO_ESCAPSE)) {
return StringUtils.substring((String) value, NO_ESCAPSE.length());
}
}
return ClientUtils.escapeQueryChars(value.toString());
// return QueryParser.escape(value.toString());
}
private List<Object> getRealValue(List<Object> value) {
if (null == value || value.isEmpty()) {
return value;
}
List<Object> ret = new ArrayList<Object>(value.size());
for (Object v : value) {
ret.add(getRealValue(v));
}
return ret;
}
public class SolrQueryBuilder
{
private BooleanQuery booleanQuery;
public static final FastDateFormat DATE_FORMAT;
public static final String NO_ESCAPSE = "$$";
public SolrQueryBuilder() {
this.booleanQuery = new BooleanQuery();
}
public SolrQueryBuilder range(final String field, final Object min, final Object max, final boolean minInclusive, final boolean maxInclusive, final BooleanClause.Occur occur) {
if (min != null || max != null) {
final Object obj = (null == min) ? max : min;
Query query = null;
if (Integer.class.isInstance(obj)) {
query = (Query)NumericRangeQuery.newIntRange(field, (Integer)min, (Integer)max, minInclusive, maxInclusive);
}
else if (Long.class.isInstance(obj)) {
query = (Query)NumericRangeQuery.newLongRange(field, (Long)min, (Long)max, minInclusive, maxInclusive);
}
else if (Double.class.isInstance(obj)) {
query = (Query)NumericRangeQuery.newDoubleRange(field, (Double)min, (Double)max, minInclusive, maxInclusive);
}
else if (Float.class.isInstance(obj)) {
query = (Query)NumericRangeQuery.newFloatRange(field, (Float)min, (Float)max, minInclusive, maxInclusive);
}
else if (Date.class.isInstance(obj)) {
query = (Query)new TermRangeQuery(field, (null == min) ? null : SolrQueryBuilder.DATE_FORMAT.format((Date)min), (null == max) ? null : SolrQueryBuilder.DATE_FORMAT.format((Date)max), minInclusive, maxInclusive);
}
else {
query = (Query)new TermRangeQuery(field, (null == min) ? ((String)min) : min.toString(), (null == max) ? ((String)max) : max.toString(), minInclusive, maxInclusive);
}
this.booleanQuery.add(query, occur);
}
return this;
}
public SolrQueryBuilder range(final String field, final Object min, final Object max, final boolean minInclusive, final boolean maxInclusive) {
return this.range(field, min, max, minInclusive, maxInclusive, BooleanClause.Occur.MUST);
}
public SolrQueryBuilder range(final String field, final Object min, final Object max) {
return this.range(field, min, max, true, true, BooleanClause.Occur.MUST);
}
public SolrQueryBuilder range(final String field, final Object min, final Object max, final BooleanClause.Occur occur) {
return this.range(field, min, max, true, true, occur);
}
public SolrQueryBuilder gt(final String field, final Object value) {
return this.range(field, value, null, false, true, BooleanClause.Occur.MUST);
}
public SolrQueryBuilder gt(final String field, final Object value, final BooleanClause.Occur occur) {
return this.range(field, value, null, false, true, occur);
}
public SolrQueryBuilder gte(final String field, final Object value) {
return this.range(field, value, null, true, true, BooleanClause.Occur.MUST);
}
public SolrQueryBuilder gte(final String field, final Object value, final BooleanClause.Occur occur) {
return this.range(field, value, null, true, true, occur);
}
public SolrQueryBuilder lt(final String field, final Object value) {
return this.range(field, null, value, true, false, BooleanClause.Occur.MUST);
}
public SolrQueryBuilder lt(final String field, final Object value, final BooleanClause.Occur occur) {
return this.range(field, null, value, true, false, occur);
}
public SolrQueryBuilder lte(final String field, final Object value) {
return this.range(field, null, value, true, true, BooleanClause.Occur.MUST);
}
public SolrQueryBuilder lte(final String field, final Object value, final BooleanClause.Occur occur) {
return this.range(field, null, value, true, true, occur);
}
public SolrQueryBuilder eq(final String field, final Object value, final BooleanClause.Occur occur) {
if (null != value) {
final Query query = (Query)new TermQuery(new Term(field, value.toString()));
this.booleanQuery.add(query, occur);
}
return this;
}
public SolrQueryBuilder eq(final String field, final Object value) {
return this.eq(field, this.getRealValue(value), BooleanClause.Occur.MUST);
}
public SolrQueryBuilder eq(final String field, final Object value, final boolean escapse) {
return this.eq(field, escapse ? QueryParser.escape(value.toString()) : value.toString(), BooleanClause.Occur.MUST);
}
public SolrQueryBuilder ne(final String field, final Object value) {
return this.eq(field, this.getRealValue(value), BooleanClause.Occur.MUST_NOT);
}
public SolrQueryBuilder in(final String field, final List<Object> value, final boolean escapse, final BooleanClause.Occur occur) {
if (null == value || value.isEmpty()) {
throw new IllegalArgumentException("in query with null or empty list!!");
}
final BooleanQuery bQuery = new BooleanQuery();
for (final Object v : value) {
final Query query = (Query)new TermQuery(new Term(field, escapse ? QueryParser.escape(v.toString()) : v.toString()));
bQuery.add(query, BooleanClause.Occur.SHOULD);
}
this.booleanQuery.add((Query)bQuery, occur);
return this;
}
public SolrQueryBuilder in(final String field, final List<Object> value) {
return this.in(field, this.getRealValue(value), false, BooleanClause.Occur.MUST);
}
public SolrQueryBuilder in(final String field, final List<Object> value, final BooleanClause.Occur occur) {
return this.in(field, this.getRealValue(value), false, occur);
}
public SolrQueryBuilder in(final String field, final List<Object> value, final boolean escapse) {
return this.in(field, value, escapse, BooleanClause.Occur.MUST);
}
public Query toQuery() {
return (Query)this.booleanQuery;
}
private Object getRealValue(final Object value) {
if (value instanceof String) {
if ("$$".equals(value)) {
return value;
}
if (StringUtils.startsWith((String)value, "$$")) {
return StringUtils.substring((String)value, "$$".length());
}
}
return ClientUtils.escapeQueryChars(value.toString());
}
private List<Object> getRealValue(final List<Object> value) {
if (null == value || value.isEmpty()) {
return value;
}
final List<Object> ret = new ArrayList<Object>(value.size());
for (final Object v : value) {
ret.add(this.getRealValue(v));
}
return ret;
}
static {
DATE_FORMAT = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", TimeZone.getTimeZone("UTC"));
}
}
@@ -1,26 +1,26 @@
package com.fangjia.datastore.solr;
import java.io.File;
import java.util.Collection;
import java.util.Set;
import java.util.Collection;
import com.fangjia.datastore.config.Reloadable;
public interface SolrSchema extends Reloadable {
public final static String VERTION = "_version";
public final static String ID = "_id";
int saveIndex(Collection<Index> indexes) throws SolrException;
boolean removeIndex(String name) throws SolrException;
Index getIndex(String name);
Collection<Index> getIndexes();
Set<String> getIndexNames();
File getSchemaFile();
void setSchemaFile(File schemaFile);
}
public interface SolrSchema extends Reloadable
{
public static final String VERTION = "_version";
public static final String ID = "_id";
int saveIndex(final Collection<Index> p0) throws SolrException;
boolean removeIndex(final String p0) throws SolrException;
Index getIndex(final String p0);
Collection<Index> getIndexes();
Set<String> getIndexNames();
File getSchemaFile();
void setSchemaFile(final File p0);
}
@@ -1,189 +1,194 @@
package com.fangjia.datastore.solr;
import java.io.File;
import java.io.FileOutputStream;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.dom4j.Document;
import java.io.OutputStream;
import org.dom4j.io.XMLWriter;
import java.io.FileOutputStream;
import org.dom4j.DocumentException;
import java.util.Set;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.dom4j.Node;
import java.util.Iterator;
import org.dom4j.DocumentHelper;
import org.dom4j.Element;
import org.dom4j.Node;
import org.dom4j.io.SAXReader;
import org.dom4j.io.XMLWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collection;
import com.fangjia.datastore.ConfigurationException;
import com.fangjia.datastore.solr.Index.IndexTypeEnum;
import java.util.HashMap;
import org.slf4j.Logger;
import org.dom4j.Document;
import java.util.Map;
import java.io.File;
import org.dom4j.io.SAXReader;
public class SolrSchemaImpl implements SolrSchema {
private final SAXReader reader = new SAXReader();
private File schemaFile;
private final Map<String, Index> indexes = new HashMap<String, Index>();
private Document document;
private final static Logger LOGGER = LoggerFactory.getLogger(SolrSchemaImpl.class);
public static void main(String[] args) throws Exception {
new SolrSchemaImpl(new File("D:\\viewstore\\solrhome\\house\\conf\\schema.xml"));
}
public SolrSchemaImpl(File schemaFile) throws ConfigurationException {
if (!schemaFile.exists() || schemaFile.isDirectory()) {
throw new IllegalArgumentException("schema file not exists or is a directory:" + schemaFile);
}
this.schemaFile = schemaFile;
reload();
}
public int saveIndex(Collection<Index> newIndexes) throws SolrException {
int ret = 0;
try {
for (Index index : newIndexes) {
Index i = indexes.get(index.getName());
if (index.equals(i)) {
continue;
}
ret++;
Element e;
if (indexes.containsKey(index.getName())) {
String xpath = "//schema//fields//" + (index.isDynamic() ? "dynamicField" : "field") + "[name=\""
+ index.getName() + "\"]";
e = (Element) this.document.selectSingleNode(xpath);
} else {
e = DocumentHelper.createElement(index.isDynamic() ? "dynamicField" : "field");
Element fields = (Element) document.selectSingleNode("//schema//fields");
fields.add(e);
}
e.addAttribute("name", index.getName());
e.addAttribute("type", index.getType().getType());
e.addAttribute("stored", String.valueOf(index.isStored()));
e.addAttribute("indexed", String.valueOf(index.isIndexed()));
e.addAttribute("multiValued", String.valueOf(index.isMultiValued()));
e.addAttribute("comments", String.valueOf(index.getComments()));
this.indexes.put(index.getName(), index);
}
} finally {
write();
}
return ret;
}
public synchronized boolean removeIndex(String name) throws SolrException {
Index index = this.indexes.get(name);
if (null == index) {
return false;
}
if (ID.equals(name) || VERTION.equals(name)) {
throw new SolrException("delete of field \"" + name + "\" is not allowed!");
}
String xpath = "//schema//fields//" + (index.isDynamic() ? "dynamicField" : "field") + "[name=\"" + index.getName()
+ "\"]";
Element e = (Element) this.document.selectSingleNode(xpath);
if (null != e) {
e.detach();
}
write();
this.indexes.remove(name);
return true;
}
public Collection<Index> getIndexes() {
return indexes.values();
}
@SuppressWarnings("unchecked")
public void reload() throws ConfigurationException {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("refresh solr schema: {}", this.schemaFile);
}
Map<String, Index> newIndexes = new HashMap<String, Index>();
Document newDocument = read();
List<Node> list = newDocument.selectNodes("//schema");
if (null == list || list.size() > 1) {
throw new ConfigurationException("zero or more than one \"fields\" tag found, expect only one \"fields\" tag");
}
Node fieldsNode = list.get(0);
list = fieldsNode.selectNodes("field");
for (Node node : list) {
Element e = (Element) node;
String name = e.attributeValue("name");
if (StringUtils.isEmpty(name)) {
throw new ConfigurationException("invalid field name \"" + name + "\" in solr schema: " + this.schemaFile);
}
IndexTypeEnum type = IndexTypeEnum.parse(e.attributeValue("type"));
if (null == type) {
throw new ConfigurationException("invalid field type \"" + e.attributeValue("type") + "\" in solr schema: "
+ this.schemaFile);
}
boolean stored = Boolean.parseBoolean(e.attributeValue("stored"));
boolean indexed = Boolean.parseBoolean(e.attributeValue("indexed"));
boolean multiValued = Boolean.parseBoolean(e.attributeValue("multiValued"));
String comments = e.attributeValue("comments");
boolean dynamic = e.getName().equals("dynamicField");
Index index = new Index(name, type, stored, indexed, multiValued, dynamic, comments);
newIndexes.put(name, index);
}
if (!newIndexes.containsKey(ID)) {
throw new ConfigurationException("no field names \"" + ID + "\" find in schema: " + this.schemaFile);
}
if (!newIndexes.containsKey(VERTION)) {
throw new ConfigurationException("no field names \"" + VERTION + "\" find in schema: " + this.schemaFile);
}
indexes.clear();
indexes.putAll(newIndexes);
this.document = newDocument;
}
public Set<String> getIndexNames() {
return this.indexes.keySet();
}
private Document read() throws ConfigurationException {
try {
return reader.read(schemaFile);
} catch (DocumentException e) {
throw new ConfigurationException("error read solr xml file: " + this, e);
}
}
private void write() throws SolrException {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("write solr schema: {}" + this.schemaFile);
}
XMLWriter writer;
try {
writer = new XMLWriter(new FileOutputStream(schemaFile));
writer.write(document);
writer.close();
} catch (IOException e) {
throw new SolrException("error write solr xml file: " + this, e);
}
}
public Index getIndex(String name) {
return this.indexes.get(name);
}
public File getSchemaFile() {
return schemaFile;
}
public void setSchemaFile(File schemaFile) {
this.schemaFile = schemaFile;
}
}
public class SolrSchemaImpl implements SolrSchema
{
private final SAXReader reader;
private File schemaFile;
private final Map<String, Index> indexes;
private Document document;
private static final Logger LOGGER;
public static void main(final String[] args) throws Exception {
new SolrSchemaImpl(new File("D:\\viewstore\\solrhome\\house\\conf\\schema.xml"));
}
public SolrSchemaImpl(final File schemaFile) throws ConfigurationException {
this.reader = new SAXReader();
this.indexes = new HashMap<String, Index>();
if (!schemaFile.exists() || schemaFile.isDirectory()) {
throw new IllegalArgumentException("schema file not exists or is a directory:" + schemaFile);
}
this.schemaFile = schemaFile;
this.reload();
}
@Override
public int saveIndex(final Collection<Index> newIndexes) throws SolrException {
int ret = 0;
try {
for (final Index index : newIndexes) {
final Index i = this.indexes.get(index.getName());
if (index.equals(i)) {
continue;
}
++ret;
Element e;
if (this.indexes.containsKey(index.getName())) {
final String xpath = "//schema//fields//" + (index.isDynamic() ? "dynamicField" : "field") + "[name=\"" + index.getName() + "\"]";
e = (Element)this.document.selectSingleNode(xpath);
}
else {
e = DocumentHelper.createElement(index.isDynamic() ? "dynamicField" : "field");
final Element fields = (Element)this.document.selectSingleNode("//schema//fields");
fields.add(e);
}
e.addAttribute("name", index.getName());
e.addAttribute("type", index.getType().getType());
e.addAttribute("stored", String.valueOf(index.isStored()));
e.addAttribute("indexed", String.valueOf(index.isIndexed()));
e.addAttribute("multiValued", String.valueOf(index.isMultiValued()));
e.addAttribute("comments", String.valueOf(index.getComments()));
this.indexes.put(index.getName(), index);
}
}
finally {
this.write();
}
return ret;
}
@Override
public synchronized boolean removeIndex(final String name) throws SolrException {
final Index index = this.indexes.get(name);
if (null == index) {
return false;
}
if ("_id".equals(name) || "_version".equals(name)) {
throw new SolrException("delete of field \"" + name + "\" is not allowed!");
}
final String xpath = "//schema//fields//" + (index.isDynamic() ? "dynamicField" : "field") + "[name=\"" + index.getName() + "\"]";
final Element e = (Element)this.document.selectSingleNode(xpath);
if (null != e) {
e.detach();
}
this.write();
this.indexes.remove(name);
return true;
}
@Override
public Collection<Index> getIndexes() {
return this.indexes.values();
}
@Override
public void reload() throws ConfigurationException {
if (SolrSchemaImpl.LOGGER.isDebugEnabled()) {
SolrSchemaImpl.LOGGER.debug("refresh solr schema: {}", (Object)this.schemaFile);
}
final Map<String, Index> newIndexes = new HashMap<String, Index>();
final Document newDocument = this.read();
List<Node> list = (List<Node>)newDocument.selectNodes("//schema");
if (null == list || list.size() > 1) {
throw new ConfigurationException("zero or more than one \"fields\" tag found, expect only one \"fields\" tag");
}
final Node fieldsNode = list.get(0);
list = (List<Node>)fieldsNode.selectNodes("field");
for (final Node node : list) {
final Element e = (Element)node;
final String name = e.attributeValue("name");
if (StringUtils.isEmpty(name)) {
throw new ConfigurationException("invalid field name \"" + name + "\" in solr schema: " + this.schemaFile);
}
final Index.IndexTypeEnum type = Index.IndexTypeEnum.parse(e.attributeValue("type"));
if (null == type) {
throw new ConfigurationException("invalid field type \"" + e.attributeValue("type") + "\" in solr schema: " + this.schemaFile);
}
final boolean stored = Boolean.parseBoolean(e.attributeValue("stored"));
final boolean indexed = Boolean.parseBoolean(e.attributeValue("indexed"));
final boolean multiValued = Boolean.parseBoolean(e.attributeValue("multiValued"));
final String comments = e.attributeValue("comments");
final boolean dynamic = e.getName().equals("dynamicField");
final Index index = new Index(name, type, stored, indexed, multiValued, dynamic, comments);
newIndexes.put(name, index);
}
if (!newIndexes.containsKey("_id")) {
throw new ConfigurationException("no field names \"_id\" find in schema: " + this.schemaFile);
}
if (!newIndexes.containsKey("_version")) {
throw new ConfigurationException("no field names \"_version\" find in schema: " + this.schemaFile);
}
this.indexes.clear();
this.indexes.putAll(newIndexes);
this.document = newDocument;
}
@Override
public Set<String> getIndexNames() {
return this.indexes.keySet();
}
private Document read() throws ConfigurationException {
try {
return this.reader.read(this.schemaFile);
}
catch (DocumentException e) {
throw new ConfigurationException("error read solr xml file: " + this, (Throwable)e);
}
}
private void write() throws SolrException {
if (SolrSchemaImpl.LOGGER.isDebugEnabled()) {
SolrSchemaImpl.LOGGER.debug("write solr schema: {}" + this.schemaFile);
}
try {
final XMLWriter writer = new XMLWriter((OutputStream)new FileOutputStream(this.schemaFile));
writer.write(this.document);
writer.close();
}
catch (IOException e) {
throw new SolrException("error write solr xml file: " + this, e);
}
}
@Override
public Index getIndex(final String name) {
return this.indexes.get(name);
}
@Override
public File getSchemaFile() {
return this.schemaFile;
}
@Override
public void setSchemaFile(final File schemaFile) {
this.schemaFile = schemaFile;
}
static {
LOGGER = LoggerFactory.getLogger((Class)SolrSchemaImpl.class);
}
}
@@ -1,23 +1,22 @@
package com.fangjia.datastore.solr;
import java.util.List;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.bson.BSONObject;
import java.util.List;
import com.fangjia.datastore.ConfigurationException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.SolrQuery;
public interface SolrServer {
QueryResponse query(SolrQuery params, boolean fromMaster, boolean bulk) throws SolrException;
void deleteByQuery(String query) throws SolrException;
void reload() throws ConfigurationException;
void reloadCores() throws SolrException;
SolrSchema getSolrSchema();
void add(List<BSONObject> dbObjects) throws SolrException;
public interface SolrServer
{
QueryResponse query(final SolrQuery p0, final boolean p1, final boolean p2) throws SolrException;
void deleteByQuery(final String p0) throws SolrException;
void reload() throws ConfigurationException;
void reloadCores() throws SolrException;
SolrSchema getSolrSchema();
void add(final List<BSONObject> p0) throws SolrException;
}
@@ -1,475 +1,455 @@
package com.fangjia.datastore.solr;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executor;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.RandomUtils;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.params.ShardParams;
import org.bson.BSONObject;
import org.bson.types.ObjectId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sothis.core.util.BlockingThreadPoolExecutor;
import com.fangjia.datastore.ConfigurationException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.commons.lang.math.RandomUtils;
import java.io.IOException;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.commons.lang.StringUtils;
import java.util.Collections;
import org.apache.commons.collections.CollectionUtils;
import org.bson.types.ObjectId;
import org.apache.solr.common.SolrInputField;
import java.util.Iterator;
import java.util.Date;
import java.util.ArrayList;
import java.util.concurrent.CountDownLatch;
import org.bson.BSONObject;
import java.util.Collection;
import org.apache.solr.common.SolrInputDocument;
import java.util.List;
import java.util.HashMap;
import java.net.MalformedURLException;
import com.fangjia.datastore.config.SolrConfig;
import com.fangjia.datastore.ConfigurationException;
import org.sothis.core.util.BlockingThreadPoolExecutor;
import java.io.File;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.Executor;
import java.util.Map;
import org.apache.http.client.HttpClient;
import com.fangjia.datastore.config.StoreConfig;
import org.slf4j.Logger;
public class SolrServerImpl implements SolrServer {
private final static Logger LOGGER = LoggerFactory.getLogger(SolrServerImpl.class);
private final static int SOLR_COMMIT_MAX_RETRY_COUNT = 3;
private final static int SOLR_QUERY_MAX_RETRY_COUNT = 3;
private final static int MAX_NONE_BULK_SKIP = 100000;
private final String name;
private final StoreConfig storeConfig;
private final HttpClient httpClient;
private int masterCommitSolrServerCount;
private final Map<String, HttpSolrServer> allSolrServers;
private String[] masterCommitFilters;
private HttpSolrServer[][] masterCommitSolrServers;
private HttpSolrServer[] masterSolrServers;
private HttpSolrServer[] slaveSolrServers;
private SolrSchema[] solrSchemas;
private final SolrSchema solrSchema;
private final Executor solrAddExecutorService;
private final AtomicBoolean filtering = new AtomicBoolean(false);
private long lastFilterTime = 0;
public SolrServerImpl(StoreConfig storeConfig) throws ConfigurationException {
this.storeConfig = storeConfig;
this.name = storeConfig.getName();
/*MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
connectionManager.getParams().setTcpNoDelay(true);
connectionManager.getParams().setStaleCheckingEnabled(true);
this.httpClient = new HttpClient(connectionManager);
httpClient.getParams().setConnectionManagerTimeout(0);
DefaultHttpMethodRetryHandler retryhandler = new DefaultHttpMethodRetryHandler(0, false);
this.httpClient.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryhandler);
*/
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
cm.setMaxTotal(5);
/* RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(1000 * 10)
.setSocketTimeout(1000 * 10)
.setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.build();
this.httpClient = HttpClients.custom()
.setDefaultRequestConfig(requestConfig)
.setConnectionManager(cm)
.build();*/
this.httpClient = new DefaultHttpClient();
this.allSolrServers = initSolrServers(this.storeConfig, httpClient);
this.solrSchema = new SolrSchemaImpl(new File(storeConfig.getMasterSolrConfig().getHome() + "/" + name
+ "/conf/managed-schema"));
int size = (masterCommitSolrServerCount + 1) * 2;
this.solrAddExecutorService = new BlockingThreadPoolExecutor(name + "-SolrAdd", 1, size, size, 60000);
}
private CommonsHttpSolrServer createSolrServer(String coreName, SolrConfig solronfig) throws ConfigurationException {
try {
CommonsHttpSolrServer solrServer = new CommonsHttpSolrServer(solronfig.getHost(), coreName, httpClient);
return solrServer;
} catch (MalformedURLException e) {
throw new ConfigurationException("[" + this.name + "] error create solr server: ", e);
}
}
private HttpSolrServer[] initSolrServer(Map<String, HttpSolrServer> allSolrServers, SolrConfig solrConfig)
throws ConfigurationException {
SolrConfig[] shardsConfigs = solrConfig.getShardsConfigs();
if (shardsConfigs.length == 0) {
shardsConfigs = new SolrConfig[] { solrConfig };
}
HttpSolrServer[] shards = new HttpSolrServer[shardsConfigs.length];
for (int i = 0; i < shardsConfigs.length; i++) {
SolrConfig shardConfig = shardsConfigs[i];
if (!allSolrServers.containsKey(shardConfig.getId())) {
SolrConfig[] slaveConfigs = shardConfig.getSlaveConfigs();
if (slaveConfigs.length == 0) {
allSolrServers.put(shardConfig.getId(), createSolrServer(name, shardConfig));
} else {
CommonsHttpSolrServer[] slaveServers = new CommonsHttpSolrServer[slaveConfigs.length];
for (int j = 0; j < slaveConfigs.length; j++) {
SolrConfig slaveConfig = slaveConfigs[j];
if (!allSolrServers.containsKey(slaveConfig.getId())) {
allSolrServers.put(slaveConfig.getId(), createSolrServer(name, slaveConfig));
}
slaveServers[j] = (CommonsHttpSolrServer) allSolrServers.get(slaveConfig.getId());
}
allSolrServers.put(shardConfig.getId(), new LBHttpSolrServer(slaveServers));
}
}
shards[i] = allSolrServers.get(shardConfig.getId());
}
return shards;
}
private synchronized Map<String, HttpSolrServer> initSolrServers(StoreConfig storeConfig, HttpClient httpClient)
throws ConfigurationException {
// http connections
//httpClient.getHttpConnectionManager().getParams().setDefaultMaxConnectionsPerHost(
//storeConfig.getSolrMaxConnectionsPerHost());
//httpClient.getHttpConnectionManager().getParams().setMaxTotalConnections(storeConfig.getSolrMaxTotalConnections());
Map<String, HttpSolrServer> solrServers = new HashMap<String, HttpSolrServer>();
SolrConfig[] masterConfigs = storeConfig.getMasterSolrConfigs();
masterCommitSolrServers = new HttpSolrServer[masterConfigs.length][];
masterCommitFilters = new String[masterConfigs.length];
solrSchemas = new SolrSchema[masterConfigs.length];
masterCommitSolrServerCount = 0;
for (int i = 0; i < masterConfigs.length; i++) {
masterCommitSolrServers[i] = initSolrServer(solrServers, masterConfigs[i]);
masterCommitFilters[i] = masterConfigs[i].getFilter();
solrSchemas[i] = new SolrSchemaImpl(new File(masterConfigs[i].getHome() + "/" + name + "/conf/managed-schema"));
masterCommitSolrServerCount += masterCommitSolrServers[i].length;
}
masterSolrServers = initSolrServer(solrServers, storeConfig.getMasterSolrConfig());
slaveSolrServers = initSolrServer(solrServers, storeConfig.getSlaveSolrConfig());
return solrServers;
}
private void addToSolr0(HttpSolrServer solrServer, List<SolrInputDocument> documents, int errorCount) {
try {
if (null != documents && !documents.isEmpty()) {
solrServer.add(documents);
solrServer.commit();
}
} catch (Exception e) {
if (errorCount <= SOLR_COMMIT_MAX_RETRY_COUNT) {
errorCount++;
long sleepTime = errorCount * errorCount;
if (LOGGER.isInfoEnabled()) {
LOGGER.info("[" + this.name + "] error commit to solr[" + solrServer.getHost() + "], sleep " + sleepTime
+ "s and retry(" + e.getMessage() + ")");
}
try {
Thread.sleep(1000 * sleepTime);
} catch (InterruptedException e1) {
LOGGER.error("[" + this.name + "] error save to solr: ", e1);
}
addToSolr0(solrServer, documents, errorCount);
} else {
if (LOGGER.isErrorEnabled()) {
LOGGER.error("[" + this.name + "] error save to solr[" + solrServer.getHost() + "] after "
+ SOLR_COMMIT_MAX_RETRY_COUNT + " times retry: ", e);
}
}
}
}
public void add(List<BSONObject> dbObjects) throws SolrException {
if (null == dbObjects || dbObjects.size() == 0) {
return;
}
try {
long time = System.currentTimeMillis();
checkFilter();
final CountDownLatch countDownLatch = new CountDownLatch(masterCommitSolrServerCount);
for (int j = 0; j < this.masterCommitSolrServers.length; j++) {
final int jndex = j;
final Map<Integer, List<BSONObject>> shardedObjectMap = shard(dbObjects, this.masterCommitSolrServers[j].length);
for (int i = 0; i < this.masterCommitSolrServers[j].length; i++) {
final int index = i;
this.solrAddExecutorService.execute(new Runnable() {
public void run() {
try {
List<BSONObject> sharedObjects = shardedObjectMap.get(index);
if (null == sharedObjects || sharedObjects.size() == 0) {
return;
}
final List<SolrInputDocument> documents = new ArrayList<SolrInputDocument>(sharedObjects.size());
for (BSONObject obj : sharedObjects) {
SolrInputDocument document = toSolrInputDocument(obj, solrSchemas[jndex]);
if (!document.isEmpty()) {
document.addField(SolrSchema.VERTION, Core.VERSION_DATE_FORMAT.format(new Date()));
System.out.println("document:" + document.toString());
documents.add(document);
} else if (LOGGER.isDebugEnabled()) {
LOGGER.debug("solr input document is empty:" + document);
}
}
addToSolr0(masterCommitSolrServers[jndex][index], documents, 0);
} catch (Exception e) {
if (LOGGER.isErrorEnabled()) {
LOGGER.error("[" + name + "] error save to solr: ", e);
}
} finally {
countDownLatch.countDown();
}
}
});
}
}
countDownLatch.await();
time = System.currentTimeMillis() - time;
if (LOGGER.isInfoEnabled()) {
LOGGER.info("[" + this.name + "] add {} documents to solr, {}ms, {}/s.", new Object[] { dbObjects.size(), time,
dbObjects.size() * 1000 / (time) });
}
} catch (Exception e) {
if (LOGGER.isErrorEnabled()) {
LOGGER.error("[" + this.name + "] error save to solr: ", e);
}
}
}
@SuppressWarnings("unchecked")
private SolrInputDocument toSolrInputDocument(BSONObject doc, SolrSchema solrSchema) {
Collection<Index> indexes = solrSchema.getIndexes();
SolrInputDocument document = new SolrInputDocument();
Map<String, Float> boostsMap = null;
if (doc.get("$boosts") != null) {
if (doc.get("$boosts") instanceof Map) {
boostsMap = (Map<String, Float>) doc.get("$boosts");
if (boostsMap.containsKey("$doc_boost")) {
document.setDocumentBoost(((Number) boostsMap.get("$doc_boost")).floatValue());
}
}
}
for (Index index : indexes) {
String key = index.getName();
if (doc.containsField(key)) {
Object value = doc.get(key);
SolrInputField field = new SolrInputField(key);
if (value instanceof ObjectId) {
field.setValue(value.toString(), 1.0f);
} else {
if (boostsMap != null && boostsMap.containsKey(key)) {
field.setValue(value, ((Number) boostsMap.get(key)).floatValue());
} else {
field.setValue(value, 1.0f);
}
}
document.put(key, field);
}
}
return document;
}
private Map<Integer, List<BSONObject>> shard(final List<BSONObject> dbObjects, int shards) {
if (CollectionUtils.isEmpty(dbObjects)) {
return Collections.emptyMap();
}
Map<Integer, List<BSONObject>> ret = new HashMap<Integer, List<BSONObject>>();
for (BSONObject object : dbObjects) {
int hashMod = Math.abs(object.get("_id").hashCode()) % shards;
List<BSONObject> modObjects = ret.get(hashMod);
if (null == modObjects) {
modObjects = new ArrayList<BSONObject>();
ret.put(hashMod, modObjects);
}
modObjects.add(object);
}
return ret;
}
private void checkFilter() {
// 5·ÖÖÓÒ»´Î
if (System.currentTimeMillis() - lastFilterTime < 300000) {
return;
}
try {
if (this.filtering.compareAndSet(false, true)) {
final long start = System.currentTimeMillis();
for (int i = 0; i < masterCommitFilters.length; i++) {
if (StringUtils.isEmpty(masterCommitFilters[i])) {
continue;
}
for (HttpSolrServer server : masterCommitSolrServers[i]) {
try {
server.deleteByQuery(masterCommitFilters[i]);
} catch (SolrServerException e) {
if (LOGGER.isErrorEnabled()) {
LOGGER.error("[" + this.name + "] error execute filter delete: ", e);
}
} catch (IOException e) {
if (LOGGER.isErrorEnabled()) {
LOGGER.error("[" + this.name + "] error execute filter delete: ", e);
}
}
}
}
if (LOGGER.isInfoEnabled()) {
LOGGER.info("[" + this.name + "] execute filter delete, " + (System.currentTimeMillis() - start) / 1000
+ "s.");
}
lastFilterTime = System.currentTimeMillis();
}
} catch (Exception e) {
if (LOGGER.isErrorEnabled()) {
LOGGER.error("[" + this.name + "] error execute filter delete: ", e);
}
} finally {
this.filtering.compareAndSet(true, false);
}
}
public void deleteByQuery(String query) throws SolrException {
try {
for (HttpSolrServer[] servers : this.masterCommitSolrServers) {
for (HttpSolrServer server : servers) {
server.deleteByQuery(query);
}
}
} catch (SolrServerException e) {
throw new SolrException("[" + this.name + "] error execute delete[" + query + "]: ", e);
} catch (IOException e) {
throw new SolrException("[" + this.name + "] error execute delete[" + query + "]: ", e);
}
return;
}
public synchronized void reload() throws ConfigurationException {
this.allSolrServers.clear();
this.allSolrServers.putAll(initSolrServers(this.storeConfig, httpClient));
String schema = this.storeConfig.getMasterSolrConfig().getHome() + "/" + name + "/conf/managed-schema";
this.solrSchema.setSchemaFile(new File(schema));
this.solrSchema.reload();
}
public void reloadCores() throws SolrException {
if (LOGGER.isInfoEnabled()) {
LOGGER.info("[" + this.name + "] reload core...");
}
for (String id : this.allSolrServers.keySet()) {
HttpSolrServer solrServer = this.allSolrServers.get(id);
solrServer.reloadCore();
}
if (LOGGER.isInfoEnabled()) {
LOGGER.info("[" + this.name + "] core reload success");
}
}
private boolean isShardedServer() {
return this.slaveSolrServers.length > 1;
}
private HttpSolrServer selectRandomServer(HttpSolrServer[] shards) {
if (null == shards || shards.length == 0) {
throw new IllegalArgumentException("no shards");
}
if (shards.length == 1) {
LOGGER.info("Choice:" + shards[0].getHost());
return shards[0];
}
HttpSolrServer server = shards[RandomUtils.nextInt(shards.length)];
LOGGER.info("Choice:" + server.getHost());
return server;
}
private String buildShardsParam(HttpSolrServer[] solrServers) {
StringBuilder ret = new StringBuilder();
for (int i = 0; i < solrServers.length; i++) {
ret.append(solrServers[i].getHost()).append('/').append(this.name);
if (i < solrServers.length - 1) {
ret.append(',');
}
}
LOGGER.info("slaveAddress:" + ret.toString());
return ret.toString();
}
public QueryResponse query(SolrQuery solrQuery, boolean fromMaster, boolean bulk) throws SolrException {
if (isShardedServer() && !bulk && solrQuery.getStart() > MAX_NONE_BULK_SKIP) {
throw new SolrException("[" + this.name + "] skip of none bulk query must not greater than " + MAX_NONE_BULK_SKIP
+ ", try set $bulk to true in the query !");
}
return queryWithRetry(solrQuery, fromMaster, bulk, 0);
}
private QueryResponse queryWithRetry(SolrQuery solrQuery, boolean fromMaster, boolean bulk, int retryCount)
throws SolrException {
// shards params
LOGGER.info("isShardedServer:"+isShardedServer());
LOGGER.info("fromMaster:"+fromMaster);
if (isShardedServer()) {
if (fromMaster) {
solrQuery.set(ShardParams.SHARDS, buildShardsParam(this.masterSolrServers));
} else {
solrQuery.set(ShardParams.SHARDS, buildShardsParam(this.slaveSolrServers));
}
// bulk query
if (bulk) {
int numberShards = this.masterSolrServers.length;
solrQuery.set(ShardParams.SHARDS_START, solrQuery.getStart() / numberShards);
solrQuery.set(ShardParams.SHARDS_ROWS, Math.max(solrQuery.getRows() / numberShards, 1));
solrQuery.setStart(0);
}
}
LOGGER.info("solrQuery:"+solrQuery.toString());
QueryResponse queryResponse = null;
HttpSolrServer httpSolrServer = null;
try {
if (fromMaster) {
httpSolrServer = selectRandomServer(this.masterSolrServers);
try {
queryResponse = httpSolrServer.query(solrQuery);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
} else {
httpSolrServer = selectRandomServer(this.slaveSolrServers);
try {
queryResponse = httpSolrServer.query(solrQuery);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
} catch (SolrServerException ex) {
if (ex.getCause() instanceof org.apache.solr.common.SolrException) {
org.apache.solr.common.SolrException e = (org.apache.solr.common.SolrException) ex.getCause();
if (retryCount < SOLR_QUERY_MAX_RETRY_COUNT && ErrorCode.getErrorCode(e.code()) == ErrorCode.SERVER_ERROR) {
retryCount++;
if (LOGGER.isErrorEnabled()) {
LOGGER.error("#[" + this.name + "]# error query solr[" + httpSolrServer.getHost() + "], code:"
+ ErrorCode.SERVER_ERROR + "(" + ErrorCode.SERVER_ERROR.code + "), retry for the " + retryCount
+ "th time...");
}
return queryWithRetry(solrQuery, fromMaster, bulk, retryCount);
} else {
throw new SolrException("=[" + this.name + "]= error query solr[" + httpSolrServer.getHost() + "]:", e);
}
}
throw new SolrException("*[" + this.name + "]* error query solr:", ex);
}
return queryResponse;
}
public SolrSchema getSolrSchema() {
return solrSchema;
}
public class SolrServerImpl implements SolrServer
{
private static final Logger LOGGER;
private static final int SOLR_COMMIT_MAX_RETRY_COUNT = 3;
private static final int SOLR_QUERY_MAX_RETRY_COUNT = 3;
private static final int MAX_NONE_BULK_SKIP = 100000;
private final String name;
private final StoreConfig storeConfig;
private final HttpClient httpClient;
private int masterCommitSolrServerCount;
private final Map<String, HttpSolrServer> allSolrServers;
private String[] masterCommitFilters;
private HttpSolrServer[][] masterCommitSolrServers;
private HttpSolrServer[] masterSolrServers;
private HttpSolrServer[] slaveSolrServers;
private SolrSchema[] solrSchemas;
private final SolrSchema solrSchema;
private final Executor solrAddExecutorService;
private final AtomicBoolean filtering;
private long lastFilterTime;
public SolrServerImpl(final StoreConfig storeConfig) throws ConfigurationException {
this.filtering = new AtomicBoolean(false);
this.lastFilterTime = 0L;
this.storeConfig = storeConfig;
this.name = storeConfig.getName();
final PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
cm.setMaxTotal(50);
this.httpClient = (HttpClient)new DefaultHttpClient();
this.allSolrServers = this.initSolrServers(this.storeConfig, this.httpClient);
this.solrSchema = new SolrSchemaImpl(new File(storeConfig.getMasterSolrConfig().getHome() + "/" + this.name + "/conf/managed-schema"));
final int size = (this.masterCommitSolrServerCount + 1) * 2;
this.solrAddExecutorService = (Executor)new BlockingThreadPoolExecutor(this.name + "-SolrAdd", 1, size, size, 60000L);
}
private CommonsHttpSolrServer createSolrServer(final String coreName, final SolrConfig solronfig) throws ConfigurationException {
try {
final CommonsHttpSolrServer solrServer = new CommonsHttpSolrServer(solronfig.getHost(), coreName, this.httpClient);
return solrServer;
}
catch (MalformedURLException e) {
throw new ConfigurationException("[" + this.name + "] error create solr server: ", e);
}
}
private HttpSolrServer[] initSolrServer(final Map<String, HttpSolrServer> allSolrServers, final SolrConfig solrConfig) throws ConfigurationException {
SolrConfig[] shardsConfigs = solrConfig.getShardsConfigs();
if (shardsConfigs.length == 0) {
shardsConfigs = new SolrConfig[] { solrConfig };
}
final HttpSolrServer[] shards = new HttpSolrServer[shardsConfigs.length];
for (int i = 0; i < shardsConfigs.length; ++i) {
final SolrConfig shardConfig = shardsConfigs[i];
if (!allSolrServers.containsKey(shardConfig.getId())) {
final SolrConfig[] slaveConfigs = shardConfig.getSlaveConfigs();
if (slaveConfigs.length == 0) {
allSolrServers.put(shardConfig.getId(), this.createSolrServer(this.name, shardConfig));
}
else {
final CommonsHttpSolrServer[] slaveServers = new CommonsHttpSolrServer[slaveConfigs.length];
for (int j = 0; j < slaveConfigs.length; ++j) {
final SolrConfig slaveConfig = slaveConfigs[j];
if (!allSolrServers.containsKey(slaveConfig.getId())) {
allSolrServers.put(slaveConfig.getId(), this.createSolrServer(this.name, slaveConfig));
}
slaveServers[j] = (CommonsHttpSolrServer)allSolrServers.get(slaveConfig.getId());
}
allSolrServers.put(shardConfig.getId(), new LBHttpSolrServer((HttpSolrServer[])slaveServers));
}
}
shards[i] = allSolrServers.get(shardConfig.getId());
}
return shards;
}
private synchronized Map<String, HttpSolrServer> initSolrServers(final StoreConfig storeConfig, final HttpClient httpClient) throws ConfigurationException {
final Map<String, HttpSolrServer> solrServers = new HashMap<String, HttpSolrServer>();
final SolrConfig[] masterConfigs = storeConfig.getMasterSolrConfigs();
this.masterCommitSolrServers = new HttpSolrServer[masterConfigs.length][];
this.masterCommitFilters = new String[masterConfigs.length];
this.solrSchemas = new SolrSchema[masterConfigs.length];
this.masterCommitSolrServerCount = 0;
for (int i = 0; i < masterConfigs.length; ++i) {
this.masterCommitSolrServers[i] = this.initSolrServer(solrServers, masterConfigs[i]);
this.masterCommitFilters[i] = masterConfigs[i].getFilter();
this.solrSchemas[i] = new SolrSchemaImpl(new File(masterConfigs[i].getHome() + "/" + this.name + "/conf/managed-schema"));
this.masterCommitSolrServerCount += this.masterCommitSolrServers[i].length;
}
this.masterSolrServers = this.initSolrServer(solrServers, storeConfig.getMasterSolrConfig());
this.slaveSolrServers = this.initSolrServer(solrServers, storeConfig.getSlaveSolrConfig());
return solrServers;
}
private void addToSolr0(final HttpSolrServer solrServer, final List<SolrInputDocument> documents, int errorCount) {
try {
if (null != documents && !documents.isEmpty()) {
solrServer.add((Collection)documents);
solrServer.commit();
}
}
catch (Exception e) {
if (errorCount <= 3) {
final long sleepTime = ++errorCount * errorCount;
if (SolrServerImpl.LOGGER.isInfoEnabled()) {
SolrServerImpl.LOGGER.info("[" + this.name + "] error commit to solr[" + solrServer.getHost() + "], sleep " + sleepTime + "s and retry(" + e.getMessage() + ")");
}
try {
Thread.sleep(1000L * sleepTime);
}
catch (InterruptedException e2) {
SolrServerImpl.LOGGER.error("[" + this.name + "] error save to solr: ", (Throwable)e2);
}
this.addToSolr0(solrServer, documents, errorCount);
}
else if (SolrServerImpl.LOGGER.isErrorEnabled()) {
SolrServerImpl.LOGGER.error("[" + this.name + "] error save to solr[" + solrServer.getHost() + "] after " + 3 + " times retry: ", (Throwable)e);
}
}
}
@Override
public void add(final List<BSONObject> dbObjects) throws SolrException {
if (null == dbObjects || dbObjects.size() == 0) {
return;
}
try {
long time = System.currentTimeMillis();
this.checkFilter();
final CountDownLatch countDownLatch = new CountDownLatch(this.masterCommitSolrServerCount);
for (int j = 0; j < this.masterCommitSolrServers.length; ++j) {
final int jndex = j;
final Map<Integer, List<BSONObject>> shardedObjectMap = this.shard(dbObjects, this.masterCommitSolrServers[j].length);
for (int i = 0; i < this.masterCommitSolrServers[j].length; ++i) {
final int index = i;
this.solrAddExecutorService.execute(new Runnable() {
@Override
public void run() {
try {
final List<BSONObject> sharedObjects = shardedObjectMap.get(index);
if (null == sharedObjects || sharedObjects.size() == 0) {
return;
}
final List<SolrInputDocument> documents = new ArrayList<SolrInputDocument>(sharedObjects.size());
for (final BSONObject obj : sharedObjects) {
final SolrInputDocument document = SolrServerImpl.this.toSolrInputDocument(obj, SolrServerImpl.this.solrSchemas[jndex]);
if (!document.isEmpty()) {
document.addField("_version", (Object)Core.VERSION_DATE_FORMAT.format(new Date()));
System.out.println("document:" + document.toString());
documents.add(document);
}
else {
if (!SolrServerImpl.LOGGER.isDebugEnabled()) {
continue;
}
SolrServerImpl.LOGGER.debug("solr input document is empty:" + document);
}
}
SolrServerImpl.this.addToSolr0(SolrServerImpl.this.masterCommitSolrServers[jndex][index], documents, 0);
}
catch (Exception e) {
if (SolrServerImpl.LOGGER.isErrorEnabled()) {
SolrServerImpl.LOGGER.error("[" + SolrServerImpl.this.name + "] error save to solr: ", (Throwable)e);
}
}
finally {
countDownLatch.countDown();
}
}
});
}
}
countDownLatch.await();
time = System.currentTimeMillis() - time;
if (SolrServerImpl.LOGGER.isInfoEnabled()) {
SolrServerImpl.LOGGER.info("[" + this.name + "] add {} documents to solr, {}ms, {}/s.", new Object[] { dbObjects.size(), time, dbObjects.size() * 1000 / time });
}
}
catch (Exception e) {
if (SolrServerImpl.LOGGER.isErrorEnabled()) {
SolrServerImpl.LOGGER.error("[" + this.name + "] error save to solr: ", (Throwable)e);
}
}
}
private SolrInputDocument toSolrInputDocument(final BSONObject doc, final SolrSchema solrSchema) {
final Collection<Index> indexes = solrSchema.getIndexes();
final SolrInputDocument document = new SolrInputDocument();
Map<String, Float> boostsMap = null;
if (doc.get("$boosts") != null && doc.get("$boosts") instanceof Map) {
boostsMap = (Map<String, Float>)doc.get("$boosts");
if (boostsMap.containsKey("$doc_boost")) {
document.setDocumentBoost(boostsMap.get("$doc_boost").floatValue());
}
}
for (final Index index : indexes) {
final String key = index.getName();
if (doc.containsField(key)) {
final Object value = doc.get(key);
final SolrInputField field = new SolrInputField(key);
if (value instanceof ObjectId) {
field.setValue((Object)value.toString(), 1.0f);
}
else if (boostsMap != null && boostsMap.containsKey(key)) {
field.setValue(value, boostsMap.get(key).floatValue());
}
else {
field.setValue(value, 1.0f);
}
document.put(key, field);
}
}
return document;
}
private Map<Integer, List<BSONObject>> shard(final List<BSONObject> dbObjects, final int shards) {
if (CollectionUtils.isEmpty((Collection)dbObjects)) {
return Collections.<Integer, List<BSONObject>>emptyMap();
}
final Map<Integer, List<BSONObject>> ret = new HashMap<Integer, List<BSONObject>>();
for (final BSONObject object : dbObjects) {
final int hashMod = Math.abs(object.get("_id").hashCode()) % shards;
List<BSONObject> modObjects = ret.get(hashMod);
if (null == modObjects) {
modObjects = new ArrayList<BSONObject>();
ret.put(hashMod, modObjects);
}
modObjects.add(object);
}
return ret;
}
private void checkFilter() {
if (System.currentTimeMillis() - this.lastFilterTime < 300000L) {
return;
}
try {
if (this.filtering.compareAndSet(false, true)) {
final long start = System.currentTimeMillis();
for (int i = 0; i < this.masterCommitFilters.length; ++i) {
if (!StringUtils.isEmpty(this.masterCommitFilters[i])) {
for (final HttpSolrServer server : this.masterCommitSolrServers[i]) {
try {
server.deleteByQuery(this.masterCommitFilters[i]);
}
catch (SolrServerException e) {
if (SolrServerImpl.LOGGER.isErrorEnabled()) {
SolrServerImpl.LOGGER.error("[" + this.name + "] error execute filter delete: ", (Throwable)e);
}
}
catch (IOException e2) {
if (SolrServerImpl.LOGGER.isErrorEnabled()) {
SolrServerImpl.LOGGER.error("[" + this.name + "] error execute filter delete: ", (Throwable)e2);
}
}
}
}
}
if (SolrServerImpl.LOGGER.isInfoEnabled()) {
SolrServerImpl.LOGGER.info("[" + this.name + "] execute filter delete, " + (System.currentTimeMillis() - start) / 1000L + "s.");
}
this.lastFilterTime = System.currentTimeMillis();
}
}
catch (Exception e3) {
if (SolrServerImpl.LOGGER.isErrorEnabled()) {
SolrServerImpl.LOGGER.error("[" + this.name + "] error execute filter delete: ", (Throwable)e3);
}
}
finally {
this.filtering.compareAndSet(true, false);
}
}
@Override
public void deleteByQuery(final String query) throws SolrException {
try {
for (final HttpSolrServer[] array : this.masterCommitSolrServers) {
final HttpSolrServer[] servers = array;
for (final HttpSolrServer server : array) {
server.deleteByQuery(query);
}
}
}
catch (SolrServerException e) {
throw new SolrException("[" + this.name + "] error execute delete[" + query + "]: ", (Throwable)e);
}
catch (IOException e2) {
throw new SolrException("[" + this.name + "] error execute delete[" + query + "]: ", e2);
}
}
@Override
public synchronized void reload() throws ConfigurationException {
this.allSolrServers.clear();
this.allSolrServers.putAll(this.initSolrServers(this.storeConfig, this.httpClient));
final String schema = this.storeConfig.getMasterSolrConfig().getHome() + "/" + this.name + "/conf/managed-schema";
this.solrSchema.setSchemaFile(new File(schema));
this.solrSchema.reload();
}
@Override
public void reloadCores() throws SolrException {
if (SolrServerImpl.LOGGER.isInfoEnabled()) {
SolrServerImpl.LOGGER.info("[" + this.name + "] reload core...");
}
for (final String id : this.allSolrServers.keySet()) {
final HttpSolrServer solrServer = this.allSolrServers.get(id);
solrServer.reloadCore();
}
if (SolrServerImpl.LOGGER.isInfoEnabled()) {
SolrServerImpl.LOGGER.info("[" + this.name + "] core reload success");
}
}
private boolean isShardedServer() {
return this.slaveSolrServers.length > 1;
}
private HttpSolrServer selectRandomServer(final HttpSolrServer[] shards) {
if (null == shards || shards.length == 0) {
throw new IllegalArgumentException("no shards");
}
if (shards.length == 1) {
SolrServerImpl.LOGGER.info("Choice:" + shards[0].getHost());
return shards[0];
}
final HttpSolrServer server = shards[RandomUtils.nextInt(shards.length)];
SolrServerImpl.LOGGER.info("Choice:" + server.getHost());
return server;
}
private String buildShardsParam(final HttpSolrServer[] solrServers) {
final StringBuilder ret = new StringBuilder();
for (int i = 0; i < solrServers.length; ++i) {
ret.append(solrServers[i].getHost()).append('/').append(this.name);
if (i < solrServers.length - 1) {
ret.append(',');
}
}
SolrServerImpl.LOGGER.info("slaveAddress:" + ret.toString());
return ret.toString();
}
@Override
public QueryResponse query(final SolrQuery solrQuery, final boolean fromMaster, final boolean bulk) throws SolrException {
if (this.isShardedServer() && !bulk && solrQuery.getStart() > 100000) {
throw new SolrException("[" + this.name + "] skip of none bulk query must not greater than " + 100000 + ", try set $bulk to true in the query !");
}
return this.queryWithRetry(solrQuery, fromMaster, bulk, 0);
}
private QueryResponse queryWithRetry(final SolrQuery solrQuery, final boolean fromMaster, final boolean bulk, int retryCount) throws SolrException {
SolrServerImpl.LOGGER.info("isShardedServer:" + this.isShardedServer());
SolrServerImpl.LOGGER.info("fromMaster:" + fromMaster);
if (this.isShardedServer()) {
if (fromMaster) {
solrQuery.set("shards", this.buildShardsParam(this.masterSolrServers));
}
else {
solrQuery.set("shards", this.buildShardsParam(this.slaveSolrServers));
}
if (bulk) {
final int numberShards = this.masterSolrServers.length;
solrQuery.set("shards.start", solrQuery.getStart() / numberShards);
solrQuery.set("shards.rows", Math.max(solrQuery.getRows() / numberShards, 1));
solrQuery.setStart(0);
}
}
SolrServerImpl.LOGGER.info("solrQuery:" + solrQuery.toString());
QueryResponse queryResponse = null;
HttpSolrServer httpSolrServer = null;
try {
if (fromMaster) {
httpSolrServer = this.selectRandomServer(this.masterSolrServers);
try {
queryResponse = httpSolrServer.query(solrQuery);
}
catch (IOException e) {
e.printStackTrace();
}
}
else {
httpSolrServer = this.selectRandomServer(this.slaveSolrServers);
try {
queryResponse = httpSolrServer.query(solrQuery);
}
catch (IOException e) {
e.printStackTrace();
}
}
solrQuery.add("__host", httpSolrServer.getHost());
}
catch (SolrServerException ex) {
if (!(ex.getCause() instanceof org.apache.solr.common.SolrException)) {
throw new SolrException("*[" + this.name + "]* error query solr:", (Throwable)ex);
}
final org.apache.solr.common.SolrException e2 = (org.apache.solr.common.SolrException)ex.getCause();
if (retryCount < 3 && org.apache.solr.common.SolrException.ErrorCode.getErrorCode(e2.code()) == org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR) {
++retryCount;
if (SolrServerImpl.LOGGER.isErrorEnabled()) {
SolrServerImpl.LOGGER.error("#[" + this.name + "]# error query solr[" + httpSolrServer.getHost() + "], code:" + org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR + "(" + org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR.code + "), retry for the " + retryCount + "th time...");
}
return this.queryWithRetry(solrQuery, fromMaster, bulk, retryCount);
}
throw new SolrException("=[" + this.name + "]= error query solr[" + httpSolrServer.getHost() + "]:", (Throwable)e2);
}
return queryResponse;
}
@Override
public SolrSchema getSolrSchema() {
return this.solrSchema;
}
static {
LOGGER = LoggerFactory.getLogger((Class)SolrServerImpl.class);
}
}
@@ -1,53 +1,34 @@
/**
* JMongo is a mongodb driver writtern in java.
* Copyright (C) 2010 Xiaohu Huang
*
* JMongo is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* JMongo is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
*/
package com.fangjia.datastore.utils;
import java.io.IOException;
public class BSONUtils {
public static long bitSet(long holder, int i, boolean set) {
if (i < 0 || i > 63) {
throw new IllegalArgumentException("i should from 0 inclusive and 64 exclusive");
}
if (set) {
return holder | (1 << i);
} else {
int ii = 1 << i;
return (holder | ii) ^ ii;
}
}
public static boolean isBitSet(long holder, int i) {
if (i < 0 || i > 63) {
throw new IllegalArgumentException("i should from 0 inclusive and 64 exclusive");
}
return (holder & (1 << i)) > 0;
}
public static int stringByteLength(String s) throws IOException {
byte[] bs = s.getBytes("UTF-8");
return bs.length + 5;
}
public static int cstringByteLength(String s) throws IOException {
byte[] bs = s.getBytes("UTF-8");
return bs.length + 1;
}
public class BSONUtils
{
public static long bitSet(final long holder, final int i, final boolean set) {
if (i < 0 || i > 63) {
throw new IllegalArgumentException("i should from 0 inclusive and 64 exclusive");
}
if (set) {
return holder | (long)(1 << i);
}
final int ii = 1 << i;
return (holder | (long)ii) ^ (long)ii;
}
public static boolean isBitSet(final long holder, final int i) {
if (i < 0 || i > 63) {
throw new IllegalArgumentException("i should from 0 inclusive and 64 exclusive");
}
return (holder & (long)(1 << i)) > 0L;
}
public static int stringByteLength(final String s) throws IOException {
final byte[] bs = s.getBytes("UTF-8");
return bs.length + 5;
}
public static int cstringByteLength(final String s) throws IOException {
final byte[] bs = s.getBytes("UTF-8");
return bs.length + 1;
}
}
@@ -1,275 +1,106 @@
/**
* JMongo is a mongodb driver writtern in java.
* Copyright (C) 2010 Xiaohu Huang
*
* JMongo is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* JMongo is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with JMongo. If not, see <http://www.gnu.org/licenses/>.
*/
package com.fangjia.datastore.utils;
import java.io.PrintStream;
/**
* Hex dump class.
*/
public final class HexDump {
public static final void dump(byte[] byt) {
dump(byt, byt.length, 0, System.out);
}
/**
* Hex dump a byte array
*
* @param byt
* Byte array to dump
* @param len
* Length of data to dump
* @param offset
* Offset to start data dump
*/
public static final void dump(byte[] byt, int len, int offset) {
dump(byt, len, offset, System.out);
}
/**
* Hex dump a byte array
*
* @param byt
* Byte array to dump
* @param len
* Length of data to dump
* @param offset
* Offset to start data dump
* @param stream
* Output stream to dump the output to.
*/
public static final void dump(byte[] byt, int len, int offset, PrintStream stream) {
// Create buffers for the ASCII and Hex output
StringBuffer ascBuf = new StringBuffer();
StringBuffer hexBuf = new StringBuffer();
// dump 16 byte blocks from the array until the length has been
// reached
int dlen = 0;
int doff = offset;
String posStr = null;
while (dlen < len) {
// Reset the ASCII/Hex buffers
ascBuf.setLength(0);
hexBuf.setLength(0);
posStr = generatePositionString(doff);
// dump a block of data, update the data offset
doff = generateLine(byt, doff, ascBuf, hexBuf);
// Output the current record
stream.print(posStr);
stream.print(hexBuf.toString());
stream.println(ascBuf.toString());
// Update the dump length
dlen += 16;
}
}
/**
* Generate a hex string for the specified string
*
* @param str
* String
* @return String
*/
public static final String hexString(String str) {
if (str != null)
return hexString(str.getBytes());
return "";
}
/**
* Generate a hex string for the specified string
*
* @param str
* String
* @param gap
* String
* @return String
*/
public static final String hexString(String str, String gap) {
if (str != null)
return hexString(str.getBytes(), gap);
return "";
}
/**
* Generate a hex string for the specified bytes
*
* @param buf
* byte[]
* @return String
*/
public static final String hexString(byte[] buf) {
return hexString(buf, buf.length, null);
}
/**
* Generate a hex string for the specified bytes
*
* @param buf
* byte[]
* @param gap
* String
* @return String
*/
public static final String hexString(byte[] buf, String gap) {
return hexString(buf, buf.length, gap);
}
/**
* Generate a hex string for the specified bytes
*
* @param buf
* byte[]
* @param len
* int
* @param gap
* String
* @return String
*/
public static final String hexString(byte[] buf, int len, String gap) {
// Check if the buffer is valid
if (buf == null)
return "";
// Create a string buffer for the hex string
int buflen = buf.length * 2;
if (gap != null)
buflen += buf.length * gap.length();
StringBuffer hex = new StringBuffer(buflen);
// Convert the bytes to hex-ASCII
for (int i = 0; i < len; i++) {
// Get the current byte
int curbyt = (int) (buf[i] & 0x00FF);
// Output the hex string
hex.append(Integer.toHexString((curbyt & 0xF0) >> 4));
hex.append(Integer.toHexString(curbyt & 0x0F));
// Add the gap string, if specified
if (gap != null && i < (len - 1))
hex.append(gap);
}
// Return the hex-ASCII string
return hex.toString();
}
/**
* Generate a buffer position string
*
* @param off
* int
* @return String
*/
private static final String generatePositionString(int off) {
// Create a buffer position string
StringBuffer posStr = new StringBuffer("" + off + ": ");
while (posStr.length() < 10)
posStr.insert(0, " ");
// Return the string
return posStr.toString();
}
/**
* Output a single line of the hex dump to a debug device
*
* @param byt
* Byte array to dump
* @param off
* Offset to start data dump
* @param ascBuf
* Buffer for ASCII output
* @param hexBuf
* Buffer for Hex output
* @return New offset value
*/
private static final int generateLine(byte[] byt, int off, StringBuffer ascBuf, StringBuffer hexBuf) {
// Check if there is enough buffer space to dump 16 bytes
int dumplen = byt.length - off;
if (dumplen > 16)
dumplen = 16;
// dump a 16 byte block of data
for (int i = 0; i < dumplen; i++) {
// Get the current byte
int curbyt = (int) (byt[off++] & 0x00FF);
// Output the hex string
hexBuf.append(Integer.toHexString((curbyt & 0xF0) >> 4));
hexBuf.append(Integer.toHexString(curbyt & 0x0F));
hexBuf.append(" ");
// Output the character equivalent, if printable
if (Character.isLetterOrDigit((char) curbyt) || Character.getType((char) curbyt) != Character.CONTROL)
ascBuf.append((char) curbyt);
else
ascBuf.append(".");
}
for (int i = dumplen; i < 16; i++) {
hexBuf.append(" ");
}
// Output the hex dump line
hexBuf.append(" ");
// Return the new data offset
return off;
}
public final class HexDump
{
public static final void dump(final byte[] byt) {
dump(byt, byt.length, 0, System.out);
}
public static final void dump(final byte[] byt, final int len, final int offset) {
dump(byt, len, offset, System.out);
}
public static final void dump(final byte[] byt, final int len, final int offset, final PrintStream stream) {
final StringBuffer ascBuf = new StringBuffer();
final StringBuffer hexBuf = new StringBuffer();
int dlen = 0;
int doff = offset;
String posStr = null;
while (dlen < len) {
ascBuf.setLength(0);
hexBuf.setLength(0);
posStr = generatePositionString(doff);
doff = generateLine(byt, doff, ascBuf, hexBuf);
stream.print(posStr);
stream.print(hexBuf.toString());
stream.println(ascBuf.toString());
dlen += 16;
}
}
public static final String hexString(final String str) {
if (str != null) {
return hexString(str.getBytes());
}
return "";
}
public static final String hexString(final String str, final String gap) {
if (str != null) {
return hexString(str.getBytes(), gap);
}
return "";
}
public static final String hexString(final byte[] buf) {
return hexString(buf, buf.length, null);
}
public static final String hexString(final byte[] buf, final String gap) {
return hexString(buf, buf.length, gap);
}
public static final String hexString(final byte[] buf, final int len, final String gap) {
if (buf == null) {
return "";
}
int buflen = buf.length * 2;
if (gap != null) {
buflen += buf.length * gap.length();
}
final StringBuffer hex = new StringBuffer(buflen);
for (int i = 0; i < len; ++i) {
final int curbyt = buf[i] & 0xFF;
hex.append(Integer.toHexString((curbyt & 0xF0) >> 4));
hex.append(Integer.toHexString(curbyt & 0xF));
if (gap != null && i < len - 1) {
hex.append(gap);
}
}
return hex.toString();
}
private static final String generatePositionString(final int off) {
final StringBuffer posStr = new StringBuffer("" + off + ": ");
while (posStr.length() < 10) {
posStr.insert(0, " ");
}
return posStr.toString();
}
private static final int generateLine(final byte[] byt, int off, final StringBuffer ascBuf, final StringBuffer hexBuf) {
int dumplen = byt.length - off;
if (dumplen > 16) {
dumplen = 16;
}
for (int i = 0; i < dumplen; ++i) {
final int curbyt = byt[off++] & 0xFF;
hexBuf.append(Integer.toHexString((curbyt & 0xF0) >> 4));
hexBuf.append(Integer.toHexString(curbyt & 0xF));
hexBuf.append(" ");
if (Character.isLetterOrDigit((char)curbyt) || Character.getType((char)curbyt) != 15) {
ascBuf.append((char)curbyt);
}
else {
ascBuf.append(".");
}
}
for (int i = dumplen; i < 16; ++i) {
hexBuf.append(" ");
}
hexBuf.append(" ");
return off;
}
}
@@ -2,17 +2,17 @@ package com.fangjia.datastore.utils;
import org.slf4j.Logger;
public class LoggerFactory {
public static Logger getLogger(Class<?> clazz) {
return org.slf4j.LoggerFactory.getLogger(clazz);
}
public static Logger getLogger(String name) {
return org.slf4j.LoggerFactory.getLogger(name);
}
public static Logger getPerformanceLogger(Class<?> clazz) {
return org.slf4j.LoggerFactory.getLogger("performance." + clazz.getName());
}
public class LoggerFactory
{
public static Logger getLogger(final Class<?> clazz) {
return org.slf4j.LoggerFactory.getLogger((Class)clazz);
}
public static Logger getLogger(final String name) {
return org.slf4j.LoggerFactory.getLogger(name);
}
public static Logger getPerformanceLogger(final Class<?> clazz) {
return org.slf4j.LoggerFactory.getLogger("performance." + clazz.getName());
}
}
@@ -2,22 +2,23 @@ package com.fangjia.datastore.utils;
import org.apache.commons.lang.StringUtils;
public class MongoUtils {
private MongoUtils() {
}
public static String[] parseFullCollectionName(String fullCollectionName) {
if (StringUtils.isEmpty(fullCollectionName)) {
return null;
}
int i = fullCollectionName.indexOf('.');
if (i <= 0 || i >= fullCollectionName.length() - 1) {
return null;
}
return new String[] { fullCollectionName.substring(0, i), fullCollectionName.substring(i + 1) };
}
public static void main(String[] args) {
System.out.println(parseFullCollectionName(".q"));
}
public class MongoUtils
{
private MongoUtils() {
}
public static String[] parseFullCollectionName(final String fullCollectionName) {
if (StringUtils.isEmpty(fullCollectionName)) {
return null;
}
final int i = fullCollectionName.indexOf(46);
if (i <= 0 || i >= fullCollectionName.length() - 1) {
return null;
}
return new String[] { fullCollectionName.substring(0, i), fullCollectionName.substring(i + 1) };
}
public static void main(final String[] args) {
System.out.println(parseFullCollectionName(".q"));
}
}
@@ -2,45 +2,42 @@ package com.fangjia.datastore.utils;
import org.dom4j.Node;
public class XmlUtils {
public static String getNodeValue(Node parentNode, String nodeName, String defaultValue) {
Node node = parentNode.selectSingleNode(nodeName);
String ret = defaultValue;
if (null == node) {
return ret;
} else {
ret = node.getText();
if (null == ret) {
ret = defaultValue;
}
}
return ret;
}
public static int getNodeValueAsInt(Node parentNode, String nodeName, int defaultValue) {
String value = getNodeValue(parentNode, nodeName, null);
if (null == value) {
return defaultValue;
} else {
return Integer.parseInt(value);
}
}
public static long getNodeValueAsLong(Node parentNode, String nodeName, long defaultValue) {
String value = getNodeValue(parentNode, nodeName, null);
if (null == value) {
return defaultValue;
} else {
return Long.parseLong(value);
}
}
public static boolean getNodeValueAsBoolean(Node parentNode, String nodeName, boolean defaultValue) {
String value = getNodeValue(parentNode, nodeName, null);
if (null == value) {
return defaultValue;
} else {
return Boolean.parseBoolean(value);
}
}
public class XmlUtils
{
public static String getNodeValue(final Node parentNode, final String nodeName, final String defaultValue) {
final Node node = parentNode.selectSingleNode(nodeName);
String ret = defaultValue;
if (null == node) {
return ret;
}
ret = node.getText();
if (null == ret) {
ret = defaultValue;
}
return ret;
}
public static int getNodeValueAsInt(final Node parentNode, final String nodeName, final int defaultValue) {
final String value = getNodeValue(parentNode, nodeName, null);
if (null == value) {
return defaultValue;
}
return Integer.parseInt(value);
}
public static long getNodeValueAsLong(final Node parentNode, final String nodeName, final long defaultValue) {
final String value = getNodeValue(parentNode, nodeName, null);
if (null == value) {
return defaultValue;
}
return Long.parseLong(value);
}
public static boolean getNodeValueAsBoolean(final Node parentNode, final String nodeName, final boolean defaultValue) {
final String value = getNodeValue(parentNode, nodeName, null);
if (null == value) {
return defaultValue;
}
return Boolean.parseBoolean(value);
}
}
@@ -918,6 +918,19 @@ public class WeiXinEvaluationController {
return jsonMmodel;
}
public Object saveBhLoanAction(EvaluationModel model){
JsonModel jsonMmodel = new JsonModel();
if(StringUtils.isNotBlank(model.getEvaluationId())){
FjtEvaluation fjtEvaluation = fjtEvaluationService.findEvaluationById(Long.valueOf(model.getEvaluationId()));
if(fjtEvaluation != null && fjtEvaluation.getLoan()!=model.getLoan()){
fjtEvaluation.setLoan(model.getLoan());
fjtEvaluation.setLastUpdated(new Date());
fjtEvaluationService.updateEvaluation(fjtEvaluation);
}
}
return jsonMmodel;
}
//刷新审核价
public Object refreshAuditPriceAction(EvaluationModel model){
JsonModel jsonMmodel = new JsonModel();
@@ -1106,9 +1119,15 @@ public class WeiXinEvaluationController {
public Object checkImageAction(EvaluationModel model){
JsonModel jsonModel = new JsonModel();
if (StringUtils.isNotBlank(model.getEvaluationId())){
List<ReportFileAttach> datas = reportFileAttachService.findHouseImage(Integer.valueOf(model.getEvaluationId()), 3);
List<ReportFileAttach> datas3 = reportFileAttachService.findHouseImage(Integer.valueOf(model.getEvaluationId()), 3);
Map<String, Integer> data = new HashMap<String, Integer>();
data.put("count", datas.size());
data.put("count_3", datas3.size());
FjtEvaluation evaluation = fjtEvaluationService.findEvaluationById(Long.valueOf(model.getEvaluationId()));
if(evaluation!=null && evaluation.getLoan()!=null && evaluation.getLoan()==1){
List<ReportFileAttach> datas5 = reportFileAttachService.findHouseImage(Integer.valueOf(model.getEvaluationId()), 5);
data.put("count_5", datas5.size());
}
jsonModel.setJsonModel(data);
}
return jsonModel;
@@ -1287,7 +1306,7 @@ public class WeiXinEvaluationController {
FjtUsers fjtUser = fjtUsersService.findById(Long.valueOf(userId));
UsersDetail usersDetail = usersDetailService.findById(fjtUser.getId());
try {
String id = new ObjectId().toString();
List<File> uploadfiles = new ArrayList<File>();
for (CommonsMultipartFile file : files) {
if (file.getSize() == 0 ) {
@@ -1319,6 +1338,8 @@ public class WeiXinEvaluationController {
} catch (IOException e) {
//处理异常信息
}
String id = new ObjectId().toString();
reportFileAttach = new ReportFileAttach();
reportFileAttach.setEvaluationId(Integer.valueOf(model.getEvaluationId()));
reportFileAttach.setQueryId(model.getQueryId());
@@ -45,6 +45,7 @@ public class EvaluationModel extends BaseModel{
private Map<String, Object> loanInfoMap;
private Integer hopePrice;
private String remark;
private Integer loan;
// private String preCode;//渤海接口--楼盘编码
// private String preName;//渤海接口--楼盘名称
// private String countryCode;//渤海接口--区域编码
@@ -317,4 +318,12 @@ public class EvaluationModel extends BaseModel{
public void setAddress(String address) {
this.address = address;
}
public Integer getLoan() {
return loan;
}
public void setLoan(Integer loan) {
this.loan = loan;
}
}
@@ -190,6 +190,36 @@ $('#submitRemark').on('click', function () {
}
});
$('#submitLoan').on('click', function () {
var opt = $("#submitLoan").text();
if(opt == "修改"){
$(this).parent().find("span").eq(1).hide()
$("input[name=loan]").removeClass("dn");
$("#submitLoan").text("保存");
}else{
var loan = $("input[name=loan]").val();
if(loan == ""){
$.toast("请输入是否续贷");
}else{
$.ajax({
url:"saveBhLoan",
data:{
evaluationId:eId,
loan:loan
},
dataType:"json",
success:function(data){
$("#submitLoan").remove();
$("input[name=loan]").remove();
$("#refreshPrice0").show();
location.href ="bhreport?evaluationId="+eId;
}
})
}
}
});
$('#refreshPrice,#refreshPrice0').on('click', function () {
$.ajax({
url:"refreshAuditPrice",
@@ -301,21 +331,34 @@ $('.download_btn').on('click', function () {
/* 下载评估报告 */
$('.formal_btn').on('click', function () {
$.ajax({
url:"checkImage",
data:{
evaluationId:eId
},
dataType:"json",
success:function(data){
if(data && data.count>0){
clickFormalReport()
}else{
$.alert('请上传房屋照片!');
if(isLoan!=null) {
$.ajax({
url:"checkImage",
data:{
evaluationId:eId
},
dataType:"json",
success:function(data){
if(isLoan==1){
if(data.count_5==0){
$.alert('请上传产调报告或权籍!');
}else if(data.count_3==0){
$.alert('请上传房屋照片!');
}else{
clickFormalReport()
}
}else{
if(data.count_3==0){
$.alert('请上传房屋照片!');
}else{
clickFormalReport()
}
}
}
}
})
})
}else{
$.alert('请选择是否续贷!');
}
});
function clickFormalReport(){
@@ -134,7 +134,7 @@ input::-webkit-input-placeholder{
.inputTopic01{margin-bottom:-1px;}
.inputTopic01 .topicTitle,.inputTopic02 .topicTitle{margin-right:0;border:none;line-height:2.5rem;}
.inputTopic01 .topicTitle .input_price{width:20%;padding-left:5px;}
.inputTopic02 .topicTitle .input_lender{width:40%;padding-left:5px;}
.inputTopic02 .topicTitle .input_lender{width:39%;padding-left:5px;line-height: 28px;}
.inputTopic02 .topicTitle .input_cardId{width:35%;padding-left:5px;}
.submit_btn{width:90px;border-left:solid 1px #e7e7e7;float:right;padding-right:0;text-align:center;cursor:pointer;}
.left01{float:left;margin-right:8%;}
@@ -132,6 +132,18 @@
</div>
</#if>
<div class="topic inputTopic02" id="inputTopic02b" style="border-bottom:none;margin-bottom:0;">
<div class="topicTitle topicTitle01">
<span>是否续贷:</span>
<input class="input_lender" name="loan" id="loan" type="hidden" required/>
<p class="di"><a href="#" class="create-actions01"><#if fjtEvaluation.loan?? && fjtEvaluation.loan==1>是<#elseif fjtEvaluation.loan?? && fjtEvaluation.loan==0>否<#else>请选择</#if></a></p>
<a class="submit_btn" id="submitLoan" style="border-left: none;"><#if fjtEvaluation.loan??>修改<#else>提交</#if></a>
</div>
<div class="topicTitle topicTitle01" style="color: #F4A460;font-size: 12px;line-height: 20px;margin-top: -5px;">
注:如出具正式报告时,需上传产调报告或权籍
</div>
</div>
<form id="fileForm" enctype="multipart/form-data" method="post" name="fileForm">
<input type="hidden" name="evaluationId" value="${fjtEvaluation.id}">
<input type="hidden" name="queryId" value="<#if bohaiEvaluationAudit??>${bohaiEvaluationAudit.queryId}</#if>">
@@ -139,7 +151,7 @@
<#if reportFileMap??>
<div class="topicTitle topicTitle01 ">
<span>已上传文件:</span>
<#if reportFileMap["1"]??><span>询价单(${reportFileMap["1"]?size})</span></#if>
<#if reportFileMap["5"]??><span>产调报告或权籍(${reportFileMap["5"]?size})</span></#if>
<#if reportFileMap["2"]??><label style="margin-left: 10px;">产证(${reportFileMap["2"]?size})</label></#if>
<#if reportFileMap["3"]??><label style="margin-left: 10px;">房屋照片(${reportFileMap["3"]?size})</label></#if>
<#if reportFileMap["4"]??><label style="margin-left: 10px;">其它附件(${reportFileMap["4"]?size})</label></#if>
@@ -456,6 +468,10 @@
<script type="text/javascript">
var eId = '${fjtEvaluation.id!}';
var role = ${role}
var isLoan = null
<#if fjtEvaluation?? && fjtEvaluation.loan??>
isLoan = ${fjtEvaluation.loan}
</#if>
showOtherSource(eId);
//showTradeReference(eId);
showHouseReference(eId);
@@ -522,11 +538,11 @@
label: true
},
{
text: '询价单',
text: '产调报告或权籍(必填)',
bold: true,
onClick: function () {
$("a.create-actions08").empty().text(this.text).addClass("selected")
$("#fileClass").val("1")
$("#fileClass").val("5")
$("input[name=reportFiles]").attr("accept","image/*")
}
},
@@ -576,6 +592,41 @@ function changeFile(d){
// }
}
}
$(document).on('click', '.create-actions01', function () {
var buttons1 = [
{
text: '请选择是否续贷',
label: true
},
{
text: '是',
bold: true,
onClick: function () {
$("#loan").empty().text(this.text).addClass("selected")
$(".create-actions01").text(this.text)
$("#loan").val("1")
}
},
{
text: '否',
onClick: function () {
$("#loan").empty().text(this.text).addClass("selected")
$(".create-actions01").text(this.text)
$("#loan").val("0")
}
}
];
var buttons2 = [
{
text: '取消',
bg: 'gray'
}
];
var groups = [buttons1, buttons2];
$.actions(groups);
});
</script>
</body>
</html>
@@ -23,14 +23,7 @@ class CityInfoEmailSender extends BaseBatch{
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files')
def emails = [
"wangsheng@fangjia.com"
,"xuyejun@fangjia.com"
,"tiki@fangjia.com"
,"zhangzx@fangjia.com"
,"panyingjiao@fangjia.com"
,"yangqixing@fangjia.com"
,"james.lin@fangjia.com"
,"jhu@fangjia.com"
"gaojuan@fangjia.com"
]
@@ -16,6 +16,7 @@ import com.fangjia.script.util.ServerAreaMap
import com.google.common.collect.MapDifference
import com.google.common.collect.Maps
import com.mongodb.*
import org.apache.commons.lang.StringUtils
import org.bson.types.ObjectId
/**
@@ -47,7 +48,16 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
}
void process(Object args) {
def cities = ServerAreaMap.findAllCities()
String cityParam = System.getProperty("city");
def cities = new ArrayList()
if(StringUtils.isNotBlank(cityParam)){
cities = new ArrayList()
for(String c in cityParam.split(",")){
cities.add(c);
}
}else{
cities = ServerAreaMap.findAllCities()
}
println cities
Date weekend = getLastDayOfMonth()
println weekend
@@ -383,7 +383,6 @@ class DealPriceDataProcess {
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
sqlInstance236.executeUpdate(sql)
}
}
}
@@ -48,13 +48,14 @@ class AnjukeAppCrawl {
// task.run();
// task.run2()
task.addThirdPrice()
// task.addThirdPrice()
// task.saveToEs()
// task.saveToEs1()
// task.groupByCity()
// task.selectNoSave()
// task.statCount()
task.exportFzDist()
}
def saveToEs1(){
def lists = getAllAppDatas()
@@ -75,6 +76,24 @@ class AnjukeAppCrawl {
})
}
def exportFzDist(){
def q = new BasicDBObject()
q.put("city","福州")
def dbcursor = detail.find(q).batchSize(5000)
def index = 0
def rets = []
rets << ["city","region","name","_id","id","sale_num","city_short","source"]
for(def d : dbcursor){
def count = 0
if(d.propInfo.saleNum){
count = d.propInfo.saleNum.toInteger()
}
rets << [d.base.cityName, d.base.areaName, d.base.name, d._id.toString(),d.base.id, count, "fz","安居客"]
}
ExcelUtils.write("D:\\数据\\福州\\福州小区.xlsx", rets, "数据")
}
def selectNoSave(){
def esData = findAllEsData()
def q = new BasicDBObject()
@@ -25,14 +25,14 @@ class AnjukeHouseCrawl {
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def href = "https://sh.esf.fang.com/loupan/1210043376.htm"
def href = "https://fz.anjuke.com/prop/view/S2512363126645770"
def doc = getDocument(href, driver)
// def doc = ClaABuYun.getResponse(href)
// def newUrl = href.substring(0, hrefs.indexOf("?"))
sendMq(href, doc, "district_detail_fang")
sendMq(href, doc, "sell_anjuke")
// for(def i=1;i<50;i++){
// def url = "https://fz.anjuke.com/sale/guloue/p${i}/"
// for(def i=25;i<50;i++){
// def url = "https://fz.anjuke.com/sale/jinana/p${i}/"
// def doc = getDocument(url, driver)
// def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc)
// println "Ò³Êý£º"+i+" ¼Ç¼Êý£º"+eles.size()
@@ -203,7 +203,18 @@ class AnjukeHouseCrawl {
}
if(!els){
els = JQueryUtils.select(".village_info", doc)
els = JQueryUtils.select(".detail-content", doc)
}
if(!els){
els = JQueryUtils.select(".content", doc)
}
if(!els){
els = JQueryUtils.select(".wrapper", doc)
}
if(!els){
els = JQueryUtils.select(".main-wrap", doc)
}
if(els && els.size()>0){
stop = true;
@@ -23,14 +23,14 @@ class ZhugeFangDistCrawl {
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
def url = "https://fz.esfxiaoqu.zhuge.com/pingtan/"
def url = "https://fz.esfxiaoqu.zhuge.com/changle/"
WebDriver driver = new ChromeDriver()
driver.manage().deleteAllCookies()
getRegions(driver, url)
}
def getRegions(driver, url) {
for(def i=1;i<=4;i++){
for(def i=1;i<=7;i++){
def newUrl = url+"page${i}/"
println newUrl
def doc = getDocument(newUrl, driver)
@@ -0,0 +1,147 @@
package com.fangjia.script.gj.paimai
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class ExportPaimai {
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
static main(args) {
def task = new ExportPaimai();
task.run()
// task.addressStandard()
// task.addPrice()
// task.stat()
}
def stat(){
def file = "D:\\数据\\重庆农商行\\法拍\\宿迁法拍数据.xlsx"
def datas = ExcelUtils.read(file, 0, 1)
def index1 = 0
def rates1 = []
for(def d : datas){
if(d['P'] && d['P']=='已成交'){
if(d['M'] && d['M']=="住宅")
// if(d['K']>=150){
// index1++
// def sumTradePrice = d['S']
// def marketPrice = d['Y']
// rates1 << sumTradePrice/marketPrice
// }
if(d['S']>2000000){
index1++
def sumTradePrice = d['S']
def marketPrice = d['Y']
rates1 << sumTradePrice/marketPrice
}
}
}
def rate = rates1.sum()/rates1.size()
println index1 +" "+ rate
}
def addPrice(){
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\宿迁法拍数据.xlsx",{d->
if(!d['Y'] && d['Z']){
d['Y'] = d['Z']
}
})
}
def addressStandard(){
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\重庆法拍数据.xlsx",{d->
def formatAddress = AddressUtil.formatAddress(d['C'], d['A'])
if(formatAddress){
d['F'] = formatAddress
}
})
}
def run(){
def datas = findPaimaiData()
println "总量:"+datas.size()
def rows = []
def header = ["标地物地址","省","城市","区域","地址","标题提取地址","楼栋号","单元","室号","楼盘名称","建筑面积","竣工日期","房屋用途","总楼层",
"拍卖状态","是否成交","拍卖阶段","拍卖方式","成交价","拍卖开始时间","拍卖结束时间","起拍价","保证金","加价幅度","评估价","市场价",
"变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数","URL"]
rows << header
def set = new HashSet()
def index = 0
for(def d : datas){
if(!set.contains(d.title)){
index++
println index
set << d.title
def row = []
row << d.title
row << d.province
row << d.city
row << d.region
row << d.address
row << ""
// def formatAddress = AddressUtil.formatAddress(d.city, d.title)
// if(formatAddress){
// row << formatAddress
// }else{
// row << ""
// }
row << d.houseNum
row << d.unitNum
row << d.roomNum
row << d.matchName
row << d.area
row << d.buildYear
row << d.estateType
row << ""
row << ""
row << d.status
row << d.stage
row << ""
row << d.tradePrice
row << DateUtils.formatDateByPattern(d.startDate, "yyyy-MM-dd")
row << DateUtils.formatDateByPattern(d.finishDate, "yyyy-MM-dd")
row << d.startPrice
row << d.depositPrice
row << d.addPrice
row << d.evalatePrice
row << d.marketPrice
row << d.sellOffAdvancePay
row << d.pricePkWeekly
row << d.delayWeekly
row << d.preemptivePerson
row << d.enrollCount
row << d.lookCount
row << d.callCount
row << d.delayCount
row << d.url
rows << row
}
}
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\重庆法拍原数据.xlsx",rows,"数据")
}
def findPaimaiData(){
def pmquery = new BasicDBObject()
pmquery.put("city","重庆")
pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-06-01")))
pmquery.put("status","已成交")
def dbcursor = paimai43_COLL.find(pmquery).sort(new BasicDBObject("finishDate",-1))
def rets = []
for(def db : dbcursor){
// if(db.estateType && db.estateType=="车库"){
// continue
// }
rets << db
}
return rets
}
}
@@ -0,0 +1,71 @@
package com.fangjia.script.gj.paimai
import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.util.MessageUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import java.text.SimpleDateFormat
DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ali_fapai_detail", "192.168.0.235", 27018)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
//Date date=new Date(1550455200000);
//SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
//String str = format.format(date);
//System.out.println(str);
def pmquery = new BasicDBObject()
pmquery.put("city","ËÞǨ")
pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
//pmquery.put("status","Òѳɽ»")
//pmquery.put("estateType", null)
pmquery.put("url","https://sf-item.taobao.com/sf_item/665574733237.htm")
def cur = paimai43_COLL.find(pmquery)
def urls = []
for(def o : cur){
urls << o.url
}
println "´ý½âÎö£º"+urls
def index = 0
for(def url : urls){
def q = new BasicDBObject()
q.put("url", url)
def dbcursor = collection.find(q)
index++
println index
for(def d : dbcursor){
JSON json = new JSONObject()
json.put("project", "paimai_taobao")
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.content)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("trades", bid_content)
result.put("condition", d.condition)
result.put("sub_count", d.sub_count)
def data_json = net.sf.json.JSONObject.fromObject(d.json)
result.put("data_json", data_json)
result.put("desc_content", d.desc_content)
result.put("item_notice_content", d.item_notice_content)
result.put("notice_detail_content", d.notice_detail_content)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
}
}
@@ -44,9 +44,7 @@ class PdsUseStatisticBatch extends BaseBatch{
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000)
def static emails = [
"hubingliang@fangjia.com",
"james.lin@fangjia.com",
"panyingjiao@fangjia.com",
"duweiliang@fangjia.com",
"yaojingmeng@fangjia.com",
"xuqi@fangjia.com",
@@ -34,8 +34,7 @@ class DaiQianGuZhiBatch extends BaseBatch{
final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files')
def static emails = [
"hubingliang@fangjia.com",
"james.lin@fangjia.com",
"panyingjiao@fangjia.com"
"james.lin@fangjia.com"
]
static main(args) {
@@ -34,8 +34,7 @@ class KeHuFenLeiBatch extends BaseBatch{
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000)
def static emails = [
"hubingliang@fangjia.com",
"james.lin@fangjia.com",
"panyingjiao@fangjia.com"
"james.lin@fangjia.com"
]
static main(args) {
@@ -33,9 +33,7 @@ class KeHuPingJiBatch extends BaseBatch{
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files')
def static emails = [
"hubingliang@fangjia.com",
"james.lin@fangjia.com",
"panyingjiao@fangjia.com"
]
static main(args) {
@@ -34,7 +34,7 @@ sqlInstance.eachRow(sql, { r ->
if (year_month == '202205') {
fiveMap.put(key, r['avg_price'] + "|" + r['calc_type'])
} else if ((year_month == '202206')) {
fiveMap.put(key, r['avg_price'] + "|" + r['calc_type'])
sixMap.put(key, r['avg_price'] + "|" + r['calc_type'])
}
})
for (String key : sixMap.keySet()) {
@@ -47,4 +47,4 @@ for (String key : sixMap.keySet()) {
}
}
}
ExcelUtils.write("F:\\数据\\价格\\东莞上海价格偏差大.xlsx", result, "统计")
ExcelUtils.write("D:\\数据\\价格\\东莞上海价格偏差大.xlsx", result, "统计")
@@ -0,0 +1,94 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.statistics.poi.AmapPoiDetail
import com.fangjia.script.temp.shenjian.poi.entity.Community
import com.fangjia.script.temp.shenjian.util.CommonDtoUtils
import com.fangjia.script.temp.shenjian.util.EsUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import org.apache.commons.collections.CollectionUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.stream.Collectors
/**
* 从ES获取POI
*/
class GetPoiFromCommunity {
static final ExecutorService executorService = Executors.newFixedThreadPool(10)
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail"
// 住宅类
static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304";
static void main(String[] args) {
AtomicLong atomicLong = new AtomicLong(0)
Map<String, String> map = new ConcurrentHashMap<>()
def rets = this.findCommunities()
CountDownLatch countDownLatch = new CountDownLatch(rets.size())
for (def retTemp : rets) {
final def ret = retTemp
executorService.submit(new Runnable() {
@Override
void run() {
final String id = ret._id // 索引ID,并非小区ID
String city = ret.city
String region = ret.region
String seaweedName = ret.name
String address = ret.address
List<AmapPoi> pois = new ArrayList<>();
List<AmapPoi> namePois = GaoDeUtils.searchPoi(city, seaweedName, region, typecodes);
List<AmapPoi> addressPois = GaoDeUtils.searchPoi(city, address, region, typecodes);
if (CollectionUtils.isNotEmpty(namePois)) {
pois.addAll(namePois)
}
if (CollectionUtils.isNotEmpty(addressPois)) {
pois.addAll(addressPois)
}
if (CollectionUtils.isNotEmpty(pois)) {
List<AmapPoiDetail> detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois);
String jsonStr = JSONObject.toJSONString(detailList)
HttpUtils.postForJson(URL, jsonStr)
}
synchronized (map) {
map.put(id, id)
countDownLatch.countDown()
if (map.size() % 100 == 0) {
atomicLong.addAndGet(map.size())
List<String> ids = map.values().stream().collect(Collectors.toList())
EsUtils.bulkUpdateFieldByIds("community", "crawl_flag", 0, ids)
map.clear()
println(atomicLong.intValue())
}
}
}
})
}
countDownLatch.await()
if (map.size() > 0) {
atomicLong.addAndGet(map.size())
List<String> ids = map.values().stream().collect(Collectors.toList())
EsUtils.bulkUpdateFieldByIds("community", "crawl_flag", 0, ids)
map.clear()
println(atomicLong.intValue())
}
System.exit(1)
}
static def findCommunities(){
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
.mustNot(QueryBuilders.termQuery("crawl_flag", 0))
List<Community> communities = EsUtils.scrollQuery("community", queryBuilder, Community.class)
return communities
}
}
@@ -0,0 +1,49 @@
package com.fangjia.script.temp.shenjian.poi.entity;
public class Community extends MyId {
private String id;
private String city;
private String region;
private String name;
private String address;
String getId() {
return id
}
void setId(String id) {
this.id = id
}
String getCity() {
return city
}
void setCity(String city) {
this.city = city
}
String getRegion() {
return region
}
void setRegion(String region) {
this.region = region
}
String getName() {
return name
}
void setName(String name) {
this.name = name
}
String getAddress() {
return address
}
void setAddress(String address) {
this.address = address
}
}
@@ -0,0 +1,17 @@
package com.fangjia.script.temp.shenjian.poi.entity
class MyId {
/**
* Ë÷Òýid
*/
String _id;
String get_id() {
return _id
}
void set_id(String _id) {
this._id = _id
}
}
@@ -0,0 +1,116 @@
package com.fangjia.script.temp.shenjian.util
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils
import com.fangjia.script.temp.shenjian.poi.entity.MyId
import org.elasticsearch.action.DocWriteRequest
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.search.*
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.search.Scroll
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.slf4j.Logger
import org.slf4j.LoggerFactory
/**
* @author shenjian
* @since 2022/07/02
*/
class EsUtils {
private static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient()
private static final Logger logger = LoggerFactory.getLogger(EsUtils.class)
/**
* 滚动方式查询大批量es数据
*
* @param index 索引
* @param boolQuery 查询语句
* @param clazz 要返回的实体类型
* @param <T>
* @return
*/
public static <T extends MyId> List<T> scrollQuery(String index, QueryBuilder boolQuery, Class<T> clazz, def limitSize=300000) {
List<T> result = new ArrayList<>();
// 存活时间,当索引数据量特别大时,出现超时可能性大,此值适当调大
Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30L));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(boolQuery);
// TEST 用!!!!!!!!!!!!!!!!!!!!!!
//searchSourceBuilder.size(1);
searchSourceBuilder.size(10000);
SearchRequest searchRequest = new SearchRequest().indices(index).scroll(scroll).source(searchSourceBuilder);
SearchResponse searchResponse = null;
try {
searchResponse = ES_CLIENT.search(searchRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
logger.error("查询es数据异常:{}", e);
}
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
if (searchHits != null) {
for (SearchHit searchHit : searchHits) {
T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz);
t.set_id(searchHit.getId())
result.add(t);
}
}
// 遍历搜索命中的数据,直到没有数据
while (searchHits != null && searchHits.length > 0 && result.size() < limitSize) {
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
try {
searchResponse = ES_CLIENT.scroll(scrollRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
logger.error("查询es数据异常:{}", e);
}
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
if (searchHits != null && searchHits.length > 0) {
for (SearchHit searchHit : searchHits) {
T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz);
t.set_id(searchHit.getId())
result.add(t);
}
}
}
// clean scroll
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
ClearScrollResponse clearScrollResponse = null;
try {
clearScrollResponse = ES_CLIENT.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
} catch (IOException e) {}
boolean succeeded = clearScrollResponse.isSucceeded();
if (succeeded) {
return result;
}
return null;
}
/**
* 批量更新es某个字段值
* @param index 索引名
* @param field 字段名
* @param val 字段新值
* @param ids 要更新的文档id
*/
public static void bulkUpdateFieldByIds(String index, String field, Object val, List<String> ids) {
if (ids.size() > 1000) {
throw new RuntimeException("批量更新数量过多")
}
BulkRequest bulkRequest = new BulkRequest();
for (String id : ids) {
Map<String, Object> doc = new HashMap<>();
doc.put(field, val)
bulkRequest.add(new UpdateRequest(index, id).doc(doc));
}
ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT);
}
}
@@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger
class GaoDeUtils {
// 新获取key
def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418"]
def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418", "10bcb421d2ee4e51ed9268d6dca58ad1"]
def static AtomicInteger keyUsed = new AtomicInteger(0)
def static codeNameMap = new HashMap<String, String>(){{
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202204);
def count2 = findTotalCount(source, 202205);
def count1 = findTotalCount(source, 202205);
def count2 = findTotalCount(source, 202206);
println source +" "+count1 +" "+count2
}
}
@@ -132,7 +132,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
q.put("s_date", SHORT_DATE);
def ret = []
try{
def dbcursor = dataProcessColl.find(q).sort(new BasicDBObject("m_date", -1).append("type",1))
def dbcursor = dataProcessColl.find(q).sort(new BasicDBObject("type",1).append("m_date", -1))
if(dbcursor){
for(def dbo : dbcursor){
@@ -1,6 +1,39 @@
package com.fangjia.script.wenyan
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\国信达级联价格-上海.xlsx",{List item->
// if(item["AB"]){
// return
// }
// String region = null
// String name = item["F"]
// if(name.endsWith("小区")){
// name = name.replaceAll("小区\$", "")
// }
// region = RegionData.formatRegion(item["B"], item["D"])
// def results = []
// DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(item["B"], region, name)).each {
// if(!results.contains(it)){
// results.add(it)
// }
// }
// DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(item["B"], region).append("alias", name)).each {
// if(!results.contains(it)){
// results.add(it)
// }
// }
// if(results.size() == 1){
// item["Z"] = results[0]._id.toString()
// item["AA"] = results[0].region
// item["AB"] = results[0].name
// println(item)
// }
//})
import com.fangjia.core.ProvinceData
import com.fangjia.core.RegionData
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
@@ -76,7 +109,48 @@ import org.apache.commons.lang.StringUtils
// }, new BasicDBObject("township", 1))
//}
//println(count)
//def GXD_CCB = [:]
//def CCB_FJW = [:]
//ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\4月级联待优化小区_3个城市.xlsx",sheetIndex = 1).each {List item->
// if(item["V"] == "级联"){
// if(item["Y"] == "国信达"){
// GXD_CCB[item['J']] = item["E"]
// }else if(item["Y"] == "房价网"){
// CCB_FJW[item['E']] = item["J"]
// }
// }
//}
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\4月级联待优化小区_3个城市.xlsx",{List item->
// if(item["V"] == "级联" && item["Y"] == "国信达"){
// def ccbId = GXD_CCB.get(item['J'], null)
// def fjwId = CCB_FJW.get(ccbId, null)
// if(fjwId != null){
// item["AB"] = fjwId
//// BasicDBObject seaweed = DbUtils.seaweed.findById(fjwId)
//// List results = DbUtils.findAroundDistrict(seaweed)
//// long total = 0
//// long count = 0
//// for(BasicDBObject s : results){
//// def avg = DbUtils.findDistrictAveragePrice(seaweed.city, seaweed.region, seaweed.name, Date.parse('yyyyMMdd', '20220619'), seaweed.estate_type2)
//// if(avg){
//// total += avg.avg_price
//// count += 1
//// }
//// }
//// if(count > 0){
//// item['Z'] = (int)(total/count)
//// println(item)
//// }
// }
// }
//},sheetIndex = 1)
ElasticUtils.executeSqlQuery("select count(*), p_date from seawater where p_date > 20210501 group by p_date",{
println(it)
})
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\moban (3).xls",{List item->
// item["B"] = RegionData.formatRegion(item["A"], item["B"])
// println(item)
//})
int count = 0
DbUtils.seaweed.find(DbUtils.buildSeaweedQuery("东莞")).each {def a->
count += 1
}
println(count)
@@ -0,0 +1,151 @@
package com.fangjia.script.wenyan
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.bson.Document
//def data = ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\TC重新跑数据.xlsx")
//def addMap = [:]
//data.each {def list->
// if(list["C"] == "有效"){
// addMap[list["B"]] = list
// }
//}
//ExcelUtils.modify("D:\\Users\\wenyan\\Downloads\\data_gj20220616.xlsx",{List item->
// if(addMap.containsKey(item["B"])){
// List list = addMap.get(item["B"])
// item["C"] = list["D"]
// item["D"] = list["E"]
// item["E"] = list["F"]
// item["S"] = "有效"
// println(item)
// }
//})
def DISTRICT_ANALY_DATA = Mongo3Utils.getDbCollection("fangjia_stat", "district_analy_data")
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\data_gj20220616.xlsx", { List item ->
// if(item["S"] == null){
// if(item["Q"] != null || item["R"] != null){
// item["S"] = "有效"
// }
// }
// if(item["S"] == null){
// def source = null
// JSONObject resp = OpenApiClient.addressMatch(item["A"], item["B"], "property")
// source = "小区库"
// if(resp.credit == 0){
// resp = OpenApiClient.addressMatch(item["A"], item["B"], null)
// source = "小区库"
// }
// if(resp.credit == 0){
// resp = OpenApiClient.addressMatch(item["A"], item["B"], null, true)
// source = "地图"
// }
//
// if(resp.credit > 0){
// println(resp)
// item["C"] = resp.city
// item["D"] = resp.district
// item["E"] = resp.name
// item["F"] = source
// item["G"] = resp.lng
// item["H"] = resp.lat
// item["I"] = resp.propertyType
// }
// if(item["G"] != null && !["普通住宅", "商住", "别墅"].contains(item["I"])){
// List seaweeds = DbUtils.findAroundDistrict(item["G"], item["H"], 1000 ,1, 2)
// if(seaweeds.size() > 0){
// BasicDBObject s = seaweeds[0]
// item["C"] = s.city
// item["D"] = s.region
// item["E"] = s.name
// item["F"] = "周边"
// item["G"] = s.lng/10000000000d
// item["H"] = s.lat/10000000000d
// item["I"] = s.estate_type2
// }
// }
// }
// if(item["S"]|| item["T"]){
// if(!item["i"]){
// JSONObject result = GaoDeApiUtils.geo(item["A"], item["B"])
// if(result != null && result.getInt("status") == 1){
// JSONArray geocodes = result.getJSONArray("geocodes");
// if(geocodes != null && geocodes.size() > 0){
// JSONObject geocode = geocodes.getJSONObject(0);
// String location = geocode.getString("location")
// item["i"] = location.split(",")[0].toDouble()
// item["j"] = location.split(",")[1].toDouble()
// }
// }
// println(item)
// }
// }
// if(item["D"] == null && item["F"] != null){
// String kw = item["F"]
// if(item["E"]){
// kw = item["E"] + kw
// }
// JSONObject result = GaoDeApiUtils.geo(item["A"], kw)
// if(result != null && result.getInt("status") == 1){
// JSONArray geocodes = result.getJSONArray("geocodes");
// if(geocodes != null && geocodes.size() > 0){
// JSONObject geocode = geocodes.getJSONObject(0);
// String location = geocode.getString("location")
// item["i"] = location.split(",")[0].toDouble()
// item["j"] = location.split(",")[1].toDouble()
// }
// }
// println(item)
// }
// if(item["E"] != null && ["普通住宅", "商住", "别墅"].contains(item["I"])){
// Document doc = DISTRICT_ANALY_DATA.find(new Document("city", item["C"]).append("region", item["D"]).append("district_name", item["E"])).first()
// if(doc){
// item["N"] =doc["rank"]
// }
// }
// if (item["S"] != "有效" || item["F"] != null) {
// return
// }
// JSONObject resp = OpenApiClient.addressMatch(item["A"], item["E"], "property", false, item["D"], null, "strict")
// if(resp.credit > 0){
// println(resp)
// item["C"] = resp.city
// item["D"] = resp.district
// item["E"] = resp.name
// item["F"] = "小区库"
// item["G"] = resp.lng
// item["H"] = resp.lat
// item["I"] = resp.propertyType
// }
// println(resp)
// if (item["F"] == null) {
// String kw = item["E"]
// if (item["D"]) {
// kw = item["D"] + kw
// }
// JSONObject result = GaoDeApiUtils.geo(item["A"], kw)
// if (result != null && result.getInt("status") == 1) {
// JSONArray geocodes = result.getJSONArray("geocodes");
// if (geocodes != null && geocodes.size() > 0) {
// JSONObject geocode = geocodes.getJSONObject(0);
// String location = geocode.getString("location")
// item["G"] = location.split(",")[0].toDouble()
// item["H"] = location.split(",")[1].toDouble()
// item["F"] = "反查经纬度"
// }
// }
// println(item)
// }
})
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20220501", "yyyyMMdd")
def endDate = DateUtil.parseDate("20220601", "yyyyMMdd")
def startDate = DateUtil.parseDate("20220601", "yyyyMMdd")
def endDate = DateUtil.parseDate("20220701", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -62,7 +62,7 @@ class ApiProductFieldStatBatch extends BaseBatch {
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000)
def emails = [
"yinjihuan@fangjia.com","panyingjiao@fangjia.com","liyan@fangjia.com", "jiangxiaochun@fangjia.com"
"jiangxiaochun@fangjia.com"
]
static def allDis = []
static DecimalFormat df = new DecimalFormat("0.00%")
@@ -499,6 +499,7 @@ public class AddressParserServiceImpl implements AddressParserService {
addr.setAddressParserGoal(params.getAddressParserGoal());
addr.setProvince(params.getProvince());
addr.setDebugName(params.getDebugName());
addr.setUseMap(params.isUseMap());
return AddressProcessors.process(addr);
}
@@ -63,6 +63,7 @@ public class Address {
private String suggestNamePinYin;
private String polygonSeaweedId;
private String geoRegion;
private boolean useMap;
private List<String> buildingNumbers;
private PreciseLevel preciseLevel;
private boolean pinYin;
@@ -389,6 +390,14 @@ public class Address {
isStandardAddress = standardAddress;
}
public boolean isUseMap() {
return useMap;
}
public void setUseMap(boolean useMap) {
this.useMap = useMap;
}
/**
* 获得有效地址
*

Some files were not shown because too many files have changed in this diff Show More