From bfe5d87a788220fd3978616120bfbca32d37d28a Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 15 Sep 2015 06:47:41 +0900 Subject: [PATCH 01/24] initial work --- .../apache/tajo/catalog/CatalogClient.java | 2 +- .../apache/tajo/client/QueryClientImpl.java | 5 +- .../apache/tajo/client/SessionConnection.java | 40 ++++++---- .../apache/tajo/worker/NodeStatusUpdater.java | 2 +- .../apache/tajo/rpc/BlockingRpcClient.java | 35 +++++---- .../org/apache/tajo/rpc/NettyClientBase.java | 49 +++++++++---- .../org/apache/tajo/rpc/RpcClientManager.java | 73 ++++++++++--------- .../org/apache/tajo/rpc/TestAsyncRpc.java | 14 ++-- .../org/apache/tajo/rpc/TestBlockingRpc.java | 14 ++-- .../apache/tajo/rpc/TestRpcClientManager.java | 4 +- 10 files changed, 139 insertions(+), 99 deletions(-) diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java index 80ded4a6e1..3c40b2a11a 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java @@ -74,7 +74,7 @@ public synchronized NettyClientBase getCatalogConnection() throws ServiceExcepti int retry = conf.getInt(RpcConstants.RPC_CLIENT_RETRY_MAX, RpcConstants.DEFAULT_RPC_RETRIES); // Client do not closed on idle state for support high available this.client = RpcClientManager.getInstance().newClient(getCatalogServerAddr(), CatalogProtocol.class, false, - retry, 0, TimeUnit.SECONDS, false); + retry, false, 0, TimeUnit.SECONDS); } catch (Exception e) { throw new ServiceException(e); } diff --git a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java index 20e56ac2a2..412426c643 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java @@ -515,9 +515,8 @@ public QueryHistoryProto getQueryHistory(final QueryId queryId) throws QueryNotF QueryMasterClientProtocol.class, false, manager.getRetries(), - manager.getTimeoutSeconds(), - TimeUnit.SECONDS, - false + false, manager.getTimeoutSeconds(), + TimeUnit.SECONDS ); diff --git a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java index ac0ff52704..f0864749fd 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java @@ -19,7 +19,6 @@ package org.apache.tajo.client; import com.google.protobuf.ServiceException; -import io.netty.channel.EventLoopGroup; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tajo.SessionVars; @@ -38,13 +37,14 @@ import org.apache.tajo.ipc.TajoMasterClientProtocol; import org.apache.tajo.ipc.TajoMasterClientProtocol.TajoMasterClientProtocolService.BlockingInterface; import org.apache.tajo.rpc.NettyClientBase; -import org.apache.tajo.rpc.NettyUtils; +import org.apache.tajo.rpc.RpcChannelFactory; import org.apache.tajo.rpc.RpcClientManager; import org.apache.tajo.rpc.RpcConstants; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetResponse; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.ReturnState; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringResponse; import org.apache.tajo.service.ServiceTracker; +import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.ProtoUtil; @@ -55,7 +55,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static org.apache.tajo.error.Errors.ResultCode.NO_SUCH_SESSION_VARIABLE; import static org.apache.tajo.exception.ReturnStateUtil.*; @@ -66,6 +68,8 @@ public class SessionConnection implements Closeable { private final static Log LOG = LogFactory.getLog(SessionConnection.class); + private final static AtomicInteger connections = new AtomicInteger(); + final RpcClientManager manager; private String baseDatabase; @@ -81,8 +85,6 @@ public class SessionConnection implements Closeable { private final ServiceTracker serviceTracker; - private final EventLoopGroup eventLoopGroup; - private NettyClientBase client; private final KeyValueSet properties; @@ -106,13 +108,7 @@ public SessionConnection(@NotNull ServiceTracker tracker, @Nullable String baseD this.manager.setRetries(properties.getInt(RpcConstants.RPC_CLIENT_RETRY_MAX, RpcConstants.DEFAULT_RPC_RETRIES)); this.userInfo = UserRoleInfo.getCurrentUser(); - this.eventLoopGroup = NettyUtils.createEventLoopGroup(getClass().getSimpleName(), 4); - try { - this.client = getTajoMasterConnection(); - } catch (TajoRuntimeException e) { - NettyUtils.shutdown(eventLoopGroup); - throw e; - } + this.client = getTajoMasterConnection(); } public Map getClientSideSessionVars() { @@ -129,8 +125,16 @@ public synchronized NettyClientBase getTajoMasterConnection() { RpcClientManager.cleanup(client); // Client do not closed on idle state for support high available - this.client = manager.newBlockingClient(getTajoMasterAddr(), TajoMasterClientProtocol.class, - manager.getRetries(), eventLoopGroup); + this.client = manager.newClient( + getTajoMasterAddr(), + TajoMasterClientProtocol.class, + false, + manager.getRetries(), + false, 0, + TimeUnit.SECONDS + ); + connections.incrementAndGet(); + } catch (Throwable t) { throw new TajoRuntimeException(new ClientConnectionException(t)); } @@ -340,7 +344,14 @@ public void close() { // ignore } finally { RpcClientManager.cleanup(client); - NettyUtils.shutdown(eventLoopGroup); + if(connections.decrementAndGet() == 0) { + if (!System.getProperty(CommonTestingUtil.TAJO_TEST_KEY, "FALSE").equals(CommonTestingUtil.TAJO_TEST_TRUE)) { + RpcChannelFactory.shutdownGracefully(); + if (LOG.isDebugEnabled()) { + LOG.debug("RPC connection is closed"); + } + } + } } } @@ -444,4 +455,5 @@ ClientProtos.SessionedStringProto getSessionedString(String str) { } return builder.build(); } + } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java b/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java index bc4f9a1169..c1744db17d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java @@ -150,7 +150,7 @@ protected TajoResourceTrackerProtocol.TajoResourceTrackerProtocolService.Interfa RpcClientManager rpcManager = RpcClientManager.getInstance(); rmClient = rpcManager.newClient(serviceTracker.getResourceTrackerAddress(), TajoResourceTrackerProtocol.class, true, rpcManager.getRetries(), - rpcManager.getTimeoutSeconds(), TimeUnit.SECONDS, false); + false, rpcManager.getTimeoutSeconds(), TimeUnit.SECONDS); return rmClient.getStub(); } diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java index 4327003b67..36ceec8eb5 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java @@ -38,36 +38,43 @@ public class BlockingRpcClient extends NettyClientBaseconnCheckEnabled is true, this connection will + * check if the connectivity is available after connCheckDuration, + * based on timeUnit. + * @param timeUnit TimeUnit for connCheckDuration + * * @throws ClassNotFoundException * @throws NoSuchMethodException */ - BlockingRpcClient(RpcConnectionKey rpcConnectionKey, int retries, long timeout, TimeUnit timeUnit, boolean enablePing, - EventLoopGroup eventLoopGroup) throws ClassNotFoundException, NoSuchMethodException { - super(rpcConnectionKey, retries); + BlockingRpcClient(RpcConnectionKey rpcConnectionKey, + EventLoopGroup eventLoopGroup, + int retryNumToConnect, + int connTimeout, + boolean useIdleTimeout, long idleTimeout, TimeUnit timeUnit) + throws ClassNotFoundException, NoSuchMethodException { + super(rpcConnectionKey, retryNumToConnect, connTimeout); this.stubMethod = getServiceClass().getMethod("newBlockingStub", BlockingRpcChannel.class); this.rpcChannel = new ProxyRpcChannel(); this.handler = new ClientChannelInboundHandler(); init(new ProtoClientChannelInitializer(handler, RpcResponse.getDefaultInstance(), - timeUnit.toNanos(timeout), - enablePing), eventLoopGroup); + timeUnit.toNanos(idleTimeout), + useIdleTimeout), eventLoopGroup); } @Override diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java index c6d90ed53f..dcd406fc89 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java @@ -48,20 +48,40 @@ public abstract class NettyClientBase implements ProtoDeclaration, Closeable { public final static Log LOG = LogFactory.getLog(NettyClientBase.class); + private final RpcConnectionKey key; + public final static int CONNECTION_RETRY_NUM_DEFAULT = 0; + private final int maxRetryNumToConnect; + /** + * Connection Timeout milli seconds (default is 15 seconds) + */ + private final int connTimeoutMillis; + public final static int CONNECTION_TIMEOUT_DEFAULT = 15 * 1000; + + private final ConcurrentMap channelEventListeners = new ConcurrentHashMap<>(); + private final ConcurrentMap requests = new ConcurrentHashMap<>(); + private Bootstrap bootstrap; private volatile ChannelFuture channelFuture; - private final RpcConnectionKey key; - private final int maxRetries; - private boolean enableMonitor; + private boolean idleTimeoutEnabled; - private final ConcurrentMap channelEventListeners = - new ConcurrentHashMap(); - private final ConcurrentMap requests = new ConcurrentHashMap(); + public NettyClientBase(RpcConnectionKey rpcConnectionKey) throws ClassNotFoundException, NoSuchMethodException { + this(rpcConnectionKey, CONNECTION_RETRY_NUM_DEFAULT, CONNECTION_TIMEOUT_DEFAULT); + } - public NettyClientBase(RpcConnectionKey rpcConnectionKey, int numRetries) + /** + * Constructor of NettyClientBase + * + * @param rpcConnectionKey RpcConnectionKey + * @param retryNumToConnect How many number it will retry + * + * @throws ClassNotFoundException + * @throws NoSuchMethodException + */ + public NettyClientBase(RpcConnectionKey rpcConnectionKey, int retryNumToConnect, int connTimeoutMillis) throws ClassNotFoundException, NoSuchMethodException { this.key = rpcConnectionKey; - this.maxRetries = numRetries; + this.maxRetryNumToConnect = retryNumToConnect; + this.connTimeoutMillis = connTimeoutMillis; } // should be called from sub class @@ -75,7 +95,8 @@ protected void init(ChannelInitializer initializer, EventLoopGroup even .option(ChannelOption.SO_REUSEADDR, true) .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, RpcConstants.DEFAULT_CONNECT_TIMEOUT) .option(ChannelOption.SO_RCVBUF, 1048576 * 10) - .option(ChannelOption.TCP_NODELAY, true); + .option(ChannelOption.TCP_NODELAY, true) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, connTimeoutMillis); } public RpcClientManager.RpcConnectionKey getKey() { @@ -132,7 +153,7 @@ public void operationComplete(final ChannelFuture future) throws Exception { getHandler().registerCallback(rpcRequest.getId(), callback); } else { - if (!future.channel().isActive() && retry < maxRetries) { + if (!future.channel().isActive() && retry < maxRetryNumToConnect) { /* schedule the current request for the retry */ LOG.warn(future.cause() + " Try to reconnect :" + getKey().addr); @@ -186,7 +207,7 @@ public synchronized void connect() throws ConnectException { ChannelFuture f = doConnect(address).awaitUninterruptibly(); if (!f.isSuccess()) { - if (maxRetries > 0) { + if (maxRetryNumToConnect > 0) { doReconnect(address, f, ++retries); } else { throw new ConnectException(ExceptionUtils.getMessage(f.cause())); @@ -198,7 +219,7 @@ private void doReconnect(final InetSocketAddress address, ChannelFuture future, throws ConnectException { for (; ; ) { - if (maxRetries > retries) { + if (maxRetryNumToConnect > retries) { retries++; if(getChannel().eventLoop().isShuttingDown()) { @@ -288,7 +309,7 @@ protected void registerCallback(int seqId, T callback) { public void channelRegistered(ChannelHandlerContext ctx) throws Exception { MonitorClientHandler handler = ctx.pipeline().get(MonitorClientHandler.class); if (handler != null) { - enableMonitor = true; + idleTimeoutEnabled = true; } for (ChannelEventListener listener : getSubscribers()) { @@ -386,7 +407,7 @@ private void sendException(RecoverableException e) { @Override public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { - if (!enableMonitor && evt instanceof IdleStateEvent) { + if (!idleTimeoutEnabled && evt instanceof IdleStateEvent) { IdleStateEvent e = (IdleStateEvent) evt; /* If all requests is done and event is triggered, idle channel close. */ if (e.state() == IdleState.READER_IDLE && requests.isEmpty()) { diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java index aa7ba67089..d3497151c0 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java @@ -61,26 +61,26 @@ public static RpcClientManager getInstance() { } private T makeClient(RpcConnectionKey rpcConnectionKey, - int retries, - long timeout, - TimeUnit timeUnit, - boolean enablePing) + int connRetryNum, + boolean useIdleTimeout, long idleTimeout, TimeUnit timeUnit) throws NoSuchMethodException, ConnectException, ClassNotFoundException { - return makeClient(rpcConnectionKey, retries, timeout, timeUnit, enablePing, NettyUtils.getDefaultEventLoopGroup()); + return makeClient(rpcConnectionKey, NettyUtils.getDefaultEventLoopGroup(), connRetryNum, useIdleTimeout, + idleTimeout, timeUnit); } private T makeClient(RpcConnectionKey rpcConnectionKey, - int retries, - long timeout, - TimeUnit timeUnit, - boolean enablePing, - EventLoopGroup eventLoopGroup) + EventLoopGroup eventLoopGroup, + int connRetryNum, + boolean useIdleTimeout, long idleTimeout, TimeUnit timeUnit) throws NoSuchMethodException, ClassNotFoundException, ConnectException { NettyClientBase client; if (rpcConnectionKey.asyncMode) { - client = new AsyncRpcClient(rpcConnectionKey, retries, timeout, timeUnit, enablePing, eventLoopGroup); + client = new AsyncRpcClient( + rpcConnectionKey, connRetryNum, idleTimeout, timeUnit, useIdleTimeout, eventLoopGroup); + } else { - client = new BlockingRpcClient(rpcConnectionKey, retries, timeout, timeUnit, enablePing, eventLoopGroup); + client = new BlockingRpcClient( + rpcConnectionKey, eventLoopGroup, connRetryNum, useIdleTimeout, idleTimeout, timeUnit); } return (T) client; } @@ -98,7 +98,7 @@ public T getClient(InetSocketAddress addr, synchronized (clients) { client = clients.get(key); if (client == null) { - clients.put(key, client = makeClient(key, retries, getTimeoutSeconds(), TimeUnit.SECONDS, true)); + clients.put(key, client = makeClient(key, retries, true, getTimeoutSeconds(), TimeUnit.SECONDS)); } } @@ -132,13 +132,13 @@ public void channelUnregistered(ChannelHandlerContext ctx) { public synchronized T newClient(InetSocketAddress addr, Class protocolClass, boolean asyncMode, - int retries, - long timeout, - TimeUnit timeUnit, - boolean enablePing) + int connRetryNum, + boolean useIdleTimeout, + long idleTimeout, + TimeUnit timeUnit) throws NoSuchMethodException, ClassNotFoundException, ConnectException { - return newClient(new RpcConnectionKey(addr, protocolClass, asyncMode), retries, timeout, timeUnit, enablePing); + return newClient(new RpcConnectionKey(addr, protocolClass, asyncMode), connRetryNum, useIdleTimeout, idleTimeout, timeUnit); } public synchronized T newClient(InetSocketAddress addr, @@ -147,30 +147,31 @@ public synchronized T newClient(InetSocketAddress ad throws NoSuchMethodException, ClassNotFoundException, ConnectException { return newClient(new RpcConnectionKey(addr, protocolClass, asyncMode), - retries, getTimeoutSeconds(), TimeUnit.SECONDS, true); + retries, true, getTimeoutSeconds(), TimeUnit.SECONDS); } + /** + * + * @param key RpcConnectionKey + * @param connRetryNum Connection Retry Number + * @param useIdueTimeout Idle timeout is enabled if true + * @param idleTimeout Idle timeout duration based on timeUnit + * @param timeUnit Time unit + * @param Rpc Protocol Class + * @return Rpc Client Class + * @throws NoSuchMethodException + * @throws ClassNotFoundException + * @throws ConnectException + */ public synchronized T newClient(RpcConnectionKey key, - int retries, - long timeout, - TimeUnit timeUnit, - boolean enablePing) - throws NoSuchMethodException, ClassNotFoundException, ConnectException { - - T client = makeClient(key, retries, timeout, timeUnit, enablePing); - client.connect(); - assert client.isConnected(); - return client; - } + int connRetryNum, + boolean useIdueTimeout, + long idleTimeout, + TimeUnit timeUnit) - public synchronized T newBlockingClient(InetSocketAddress addr, - Class protocolClass, - int retries, - EventLoopGroup eventLoopGroup) throws NoSuchMethodException, ClassNotFoundException, ConnectException { - T client = makeClient(new RpcConnectionKey(addr, protocolClass, false), - retries, 0, TimeUnit.SECONDS, false, eventLoopGroup); + T client = makeClient(key, connRetryNum, useIdueTimeout, idleTimeout, timeUnit); client.connect(); assert client.isConnected(); return client; diff --git a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java index 4f174763b0..86611bb523 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java @@ -132,7 +132,7 @@ public void setUpRpcClient() throws Exception { rpcConnectionKey = new RpcClientManager.RpcConnectionKey( RpcUtils.getConnectAddress(server.getListenAddress()), DummyProtocol.class, true); - client = manager.newClient(rpcConnectionKey, retries, 10, TimeUnit.SECONDS, true); + client = manager.newClient(rpcConnectionKey, retries, true, 10, TimeUnit.SECONDS); assertTrue(client.isConnected()); stub = client.getStub(); } @@ -350,7 +350,7 @@ public void run() { RpcClientManager.RpcConnectionKey rpcConnectionKey = new RpcClientManager.RpcConnectionKey(address, DummyProtocol.class, true); AsyncRpcClient client = manager.newClient(rpcConnectionKey, - retries, 0, TimeUnit.MILLISECONDS, false); + retries, false, 0, TimeUnit.MILLISECONDS); assertTrue(client.isConnected()); Interface stub = client.getStub(); @@ -455,7 +455,7 @@ public void testUnresolvedAddress2() throws Exception { public void testStubRecovery() throws Exception { RpcClientManager.RpcConnectionKey rpcConnectionKey = new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); - AsyncRpcClient client = manager.newClient(rpcConnectionKey, 2, 0, TimeUnit.MILLISECONDS, false); + AsyncRpcClient client = manager.newClient(rpcConnectionKey, 2, false, 0, TimeUnit.MILLISECONDS); EchoMessage echoMessage = EchoMessage.newBuilder() .setMessage(MESSAGE).build(); @@ -487,7 +487,7 @@ public void testIdleTimeout() throws Exception { RpcClientManager.RpcConnectionKey rpcConnectionKey = new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); //500 millis idle timeout - AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, 500, TimeUnit.MILLISECONDS, false); + AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, false, 500, TimeUnit.MILLISECONDS); assertTrue(client.isConnected()); Thread.sleep(600); //timeout @@ -508,7 +508,7 @@ public void testPingOnIdle() throws Exception { new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); //500 millis request timeout - AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, 500, TimeUnit.MILLISECONDS, true); + AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, true, 500, TimeUnit.MILLISECONDS); assertTrue(client.isConnected()); Thread.sleep(600); @@ -525,7 +525,7 @@ public void testIdleTimeoutWithActiveRequest() throws Exception { RpcClientManager.RpcConnectionKey rpcConnectionKey = new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); //500 millis idle timeout - AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, 500, TimeUnit.MILLISECONDS, false); + AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, false, 500, TimeUnit.MILLISECONDS); assertTrue(client.isConnected()); Interface stub = client.getStub(); @@ -551,7 +551,7 @@ public void testRequestTimeoutOnBusy() throws Exception { new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); //500 millis request timeout - AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, 500, TimeUnit.MILLISECONDS, true); + AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, true, 500, TimeUnit.MILLISECONDS); assertTrue(client.isConnected()); Interface stub = client.getStub(); diff --git a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java index 0fae7ee093..1e2709381f 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java @@ -123,7 +123,7 @@ public void setUpRpcClient() throws Exception { new RpcClientManager.RpcConnectionKey( RpcUtils.getConnectAddress(server.getListenAddress()), DummyProtocol.class, false); - client = manager.newClient(rpcConnectionKey, retries, 10, TimeUnit.SECONDS, true); + client = manager.newClient(rpcConnectionKey, retries, true, 10, TimeUnit.SECONDS); assertTrue(client.isConnected()); stub = client.getStub(); } @@ -321,7 +321,7 @@ public void run() { new RpcClientManager.RpcConnectionKey(address, DummyProtocol.class, false); BlockingRpcClient client = manager.newClient(rpcConnectionKey, - retries, 0, TimeUnit.MILLISECONDS, false); + retries, false, 0, TimeUnit.MILLISECONDS); assertTrue(client.isConnected()); BlockingInterface stub = client.getStub(); @@ -412,7 +412,7 @@ public void testUnresolvedAddress2() throws Exception { public void testStubRecovery() throws Exception { RpcClientManager.RpcConnectionKey rpcConnectionKey = new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); - BlockingRpcClient client = manager.newClient(rpcConnectionKey, 1, 0, TimeUnit.MILLISECONDS, false); + BlockingRpcClient client = manager.newClient(rpcConnectionKey, 1, false, 0, TimeUnit.MILLISECONDS); EchoMessage echoMessage = EchoMessage.newBuilder() .setMessage(MESSAGE).build(); @@ -443,7 +443,7 @@ public void testIdleTimeout() throws Exception { RpcClientManager.RpcConnectionKey rpcConnectionKey = new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); //500 millis idle timeout - BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, 500, TimeUnit.MILLISECONDS, false); + BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, false, 500, TimeUnit.MILLISECONDS); assertTrue(client.isConnected()); Thread.sleep(600); //timeout @@ -464,7 +464,7 @@ public void testPingOnIdle() throws Exception { new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); //500 millis request timeout - BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, 500, TimeUnit.MILLISECONDS, true); + BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, true, 500, TimeUnit.MILLISECONDS); assertTrue(client.isConnected()); Thread.sleep(600); @@ -481,7 +481,7 @@ public void testIdleTimeoutWithActiveRequest() throws Exception { RpcClientManager.RpcConnectionKey rpcConnectionKey = new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); //500 millis idle timeout - BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, 500, TimeUnit.MILLISECONDS, false); + BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, false, 500, TimeUnit.MILLISECONDS); assertTrue(client.isConnected()); BlockingInterface stub = client.getStub(); @@ -504,7 +504,7 @@ public void testRequestTimeoutOnBusy() throws Exception { new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); //500 millis request timeout - BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, 500, TimeUnit.MILLISECONDS, true); + BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, true, 500, TimeUnit.MILLISECONDS); assertTrue(client.isConnected()); BlockingInterface stub = client.getStub(); diff --git a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestRpcClientManager.java b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestRpcClientManager.java index 71a2b6f51a..3834351a72 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestRpcClientManager.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestRpcClientManager.java @@ -150,11 +150,11 @@ public void testUnManagedClient() throws Exception { RpcClientManager manager = RpcClientManager.getInstance(); try { - NettyClientBase client1 = manager.newClient(key, 0, 0, TimeUnit.SECONDS, false); + NettyClientBase client1 = manager.newClient(key, 0, false, 0, TimeUnit.SECONDS); assertTrue(client1.isConnected()); assertFalse(RpcClientManager.contains(key)); - NettyClientBase client2 = manager.newClient(key, 0, 0, TimeUnit.SECONDS, false); + NettyClientBase client2 = manager.newClient(key, 0, false, 0, TimeUnit.SECONDS); assertTrue(client2.isConnected()); assertFalse(RpcClientManager.contains(key)); From 3ce490862315b783e1c9c2552473eba32061afd7 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 15 Sep 2015 17:03:22 -0700 Subject: [PATCH 02/24] Refined APIs. --- .../apache/tajo/catalog/CatalogClient.java | 5 +- .../apache/tajo/client/QueryClientImpl.java | 7 +- .../apache/tajo/client/SessionConnection.java | 9 +-- .../org/apache/tajo/master/TajoMaster.java | 2 +- .../apache/tajo/worker/NodeStatusUpdater.java | 3 +- .../org/apache/tajo/worker/TajoWorker.java | 2 +- .../org/apache/tajo/rpc/RpcConstants.java | 10 ++- .../org/apache/tajo/rpc/AsyncRpcClient.java | 35 +++++----- .../apache/tajo/rpc/BlockingRpcClient.java | 32 ++++----- .../org/apache/tajo/rpc/NettyClientBase.java | 41 ++++++------ .../rpc/ProtoClientChannelInitializer.java | 11 ++-- .../org/apache/tajo/rpc/RpcClientManager.java | 66 +++++++++---------- .../org/apache/tajo/rpc/TestAsyncRpc.java | 15 ++--- .../org/apache/tajo/rpc/TestBlockingRpc.java | 15 ++--- .../apache/tajo/rpc/TestRpcClientManager.java | 4 +- 15 files changed, 123 insertions(+), 134 deletions(-) diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java index 3c40b2a11a..643226ebcd 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java @@ -31,7 +31,6 @@ import java.io.IOException; import java.net.InetSocketAddress; -import java.util.concurrent.TimeUnit; /** * CatalogClient provides a client API to access the catalog server. @@ -71,10 +70,8 @@ public synchronized NettyClientBase getCatalogConnection() throws ServiceExcepti if (client != null && client.isConnected()) return client; RpcClientManager.cleanup(client); - int retry = conf.getInt(RpcConstants.RPC_CLIENT_RETRY_MAX, RpcConstants.DEFAULT_RPC_RETRIES); // Client do not closed on idle state for support high available - this.client = RpcClientManager.getInstance().newClient(getCatalogServerAddr(), CatalogProtocol.class, false, - retry, false, 0, TimeUnit.SECONDS); + this.client = RpcClientManager.getInstance().newClient(getCatalogServerAddr(), CatalogProtocol.class, false); } catch (Exception e) { throw new ServiceException(e); } diff --git a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java index 412426c643..a18322fad4 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java @@ -512,12 +512,7 @@ public QueryHistoryProto getQueryHistory(final QueryId queryId) throws QueryNotF qmClient = manager.newClient( qmAddress, - QueryMasterClientProtocol.class, - false, - manager.getRetries(), - false, manager.getTimeoutSeconds(), - TimeUnit.SECONDS - ); + QueryMasterClientProtocol.class, false); conn.checkSessionAndGet(conn.getTajoMasterConnection()); diff --git a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java index f0864749fd..1d260861e5 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java @@ -125,14 +125,7 @@ public synchronized NettyClientBase getTajoMasterConnection() { RpcClientManager.cleanup(client); // Client do not closed on idle state for support high available - this.client = manager.newClient( - getTajoMasterAddr(), - TajoMasterClientProtocol.class, - false, - manager.getRetries(), - false, 0, - TimeUnit.SECONDS - ); + this.client = manager.newClient(getTajoMasterAddr(), TajoMasterClientProtocol.class, false ); connections.incrementAndGet(); } catch (Throwable t) { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index 1197e988b1..1453fb05fd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -172,7 +172,7 @@ public void serviceInit(Configuration conf) throws Exception { RpcClientManager rpcManager = RpcClientManager.getInstance(); rpcManager.setRetries(systemConf.getInt(RpcConstants.RPC_CLIENT_RETRY_MAX, RpcConstants.DEFAULT_RPC_RETRIES)); - rpcManager.setTimeoutSeconds( + rpcManager.setSocketTimeout( systemConf.getInt(RpcConstants.RPC_CLIENT_TIMEOUT_SECS, RpcConstants.DEFAULT_RPC_TIMEOUT_SECONDS)); initResourceManager(); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java b/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java index c1744db17d..a1e572582e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java @@ -149,8 +149,7 @@ protected TajoResourceTrackerProtocol.TajoResourceTrackerProtocolService.Interfa RpcClientManager rpcManager = RpcClientManager.getInstance(); rmClient = rpcManager.newClient(serviceTracker.getResourceTrackerAddress(), - TajoResourceTrackerProtocol.class, true, rpcManager.getRetries(), - false, rpcManager.getTimeoutSeconds(), TimeUnit.SECONDS); + TajoResourceTrackerProtocol.class, true, rpcManager.getRetries()); return rmClient.getStub(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java index fbb8d5426c..b62bdb2ff4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java @@ -154,7 +154,7 @@ public void serviceInit(Configuration conf) throws Exception { RpcClientManager rpcManager = RpcClientManager.getInstance(); rpcManager.setRetries(systemConf.getInt(RpcConstants.RPC_CLIENT_RETRY_MAX, RpcConstants.DEFAULT_RPC_RETRIES)); - rpcManager.setTimeoutSeconds( + rpcManager.setSocketTimeout( systemConf.getInt(RpcConstants.RPC_CLIENT_TIMEOUT_SECS, RpcConstants.DEFAULT_RPC_TIMEOUT_SECONDS)); serviceTracker = ServiceTrackerFactory.get(systemConf); diff --git a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java index ab0826fbfb..a56062036e 100644 --- a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java +++ b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java @@ -18,6 +18,8 @@ package org.apache.tajo.rpc; +import java.util.concurrent.TimeUnit; + public class RpcConstants { public static final String PING_PACKET = "TAJO"; @@ -25,8 +27,12 @@ public class RpcConstants { public static final String RPC_CLIENT_TIMEOUT_SECS = "tajo.rpc.client.timeout-secs"; public static final int DEFAULT_RPC_RETRIES = 3; - public static final int DEFAULT_RPC_TIMEOUT_SECONDS = 180; - public static final int DEFAULT_CONNECT_TIMEOUT = 20000; // 20 sec + public static final int DEFAULT_PAUSE = 1000; // 1 sec public static final int DEFAULT_FUTURE_TIMEOUT_SECONDS = 10; + + /** default connection timeout 15 seconds */ + public final static long DEFAULT_CONNECTION_TIMEOUT = TimeUnit.SECONDS.toMicros(15); + /** default socket timeout - 60 seconds */ + public final static long DEFAULT_SOCKET_TIMEOUT = TimeUnit.SECONDS.toNanos(60); } diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java index 6fb62d4743..46112b480b 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java @@ -39,35 +39,40 @@ public class AsyncRpcClient extends NettyClientBaseuseIdleTimeout is true, + * this connection will check if the connectivity is available after + * idleTimeout * @throws ClassNotFoundException * @throws NoSuchMethodException */ - AsyncRpcClient(RpcConnectionKey rpcConnectionKey, int retries, long timeout, TimeUnit timeUnit, boolean enablePing, - EventLoopGroup eventLoopGroup) + AsyncRpcClient(EventLoopGroup eventLoopGroup, + RpcConnectionKey rpcConnectionKey, + int retryNum, + long connTimeout, + boolean useIdleTimeout, + long idleTimeout) throws ClassNotFoundException, NoSuchMethodException { - super(rpcConnectionKey, retries); + super(rpcConnectionKey, retryNum, connTimeout); this.stubMethod = getServiceClass().getMethod("newStub", RpcChannel.class); this.rpcChannel = new ProxyRpcChannel(); this.handler = new ClientChannelInboundHandler(); - init(new ProtoClientChannelInitializer(handler, - RpcResponse.getDefaultInstance(), - timeUnit.toNanos(timeout), - enablePing), eventLoopGroup); + + init(new ProtoClientChannelInitializer(handler, RpcResponse.getDefaultInstance(), + useIdleTimeout, TimeUnit.MILLISECONDS.toNanos(idleTimeout)), eventLoopGroup); } @Override diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java index 36ceec8eb5..b33fea2232 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java @@ -40,7 +40,7 @@ public class BlockingRpcClient extends NettyClientBaseconnCheckEnabled is true, this connection will - * check if the connectivity is available after connCheckDuration, - * based on timeUnit. - * @param timeUnit TimeUnit for connCheckDuration + * @param retryNumToConnect Retry number to connect + * @param connTimeout Connection Timeout (milliseconds) + * @param useIdleTimeout Enable idle connection check + * @param idleTimeout Socket Idle timeout (milliseconds). If useIdleTimeout is true, + * this connection will check if the connectivity is available after + * idleTimeout * * @throws ClassNotFoundException * @throws NoSuchMethodException */ - BlockingRpcClient(RpcConnectionKey rpcConnectionKey, - EventLoopGroup eventLoopGroup, + BlockingRpcClient(EventLoopGroup eventLoopGroup, + RpcConnectionKey rpcConnectionKey, int retryNumToConnect, - int connTimeout, - boolean useIdleTimeout, long idleTimeout, TimeUnit timeUnit) + long connTimeout, + boolean useIdleTimeout, + long idleTimeout) throws ClassNotFoundException, NoSuchMethodException { super(rpcConnectionKey, retryNumToConnect, connTimeout); this.stubMethod = getServiceClass().getMethod("newBlockingStub", BlockingRpcChannel.class); this.rpcChannel = new ProxyRpcChannel(); this.handler = new ClientChannelInboundHandler(); - init(new ProtoClientChannelInitializer(handler, - RpcResponse.getDefaultInstance(), - timeUnit.toNanos(idleTimeout), - useIdleTimeout), eventLoopGroup); + + init(new ProtoClientChannelInitializer(handler, RpcResponse.getDefaultInstance(), + useIdleTimeout, TimeUnit.MILLISECONDS.toNanos(idleTimeout)),eventLoopGroup); } @Override diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java index dcd406fc89..4ee6a7113d 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java @@ -18,6 +18,7 @@ package org.apache.tajo.rpc; +import com.google.common.base.Preconditions; import com.google.protobuf.Descriptors; import com.google.protobuf.Message; import com.google.protobuf.ServiceException; @@ -49,13 +50,10 @@ public abstract class NettyClientBase implements ProtoDeclaration, Closeable public final static Log LOG = LogFactory.getLog(NettyClientBase.class); private final RpcConnectionKey key; - public final static int CONNECTION_RETRY_NUM_DEFAULT = 0; - private final int maxRetryNumToConnect; - /** - * Connection Timeout milli seconds (default is 15 seconds) - */ - private final int connTimeoutMillis; - public final static int CONNECTION_TIMEOUT_DEFAULT = 15 * 1000; + /** Number to retry for connection and RPC invocation */ + private final int maxRetryNum; + /** Connection Timeout */ + private final long connTimeoutMillis; private final ConcurrentMap channelEventListeners = new ConcurrentHashMap<>(); private final ConcurrentMap requests = new ConcurrentHashMap<>(); @@ -64,24 +62,24 @@ public abstract class NettyClientBase implements ProtoDeclaration, Closeable private volatile ChannelFuture channelFuture; private boolean idleTimeoutEnabled; - public NettyClientBase(RpcConnectionKey rpcConnectionKey) throws ClassNotFoundException, NoSuchMethodException { - this(rpcConnectionKey, CONNECTION_RETRY_NUM_DEFAULT, CONNECTION_TIMEOUT_DEFAULT); - } - /** * Constructor of NettyClientBase * * @param rpcConnectionKey RpcConnectionKey - * @param retryNumToConnect How many number it will retry + * @param maxRetryNum How many number it will retry + * @param connTimeout Connection Timeout (milliseconds) * * @throws ClassNotFoundException * @throws NoSuchMethodException */ - public NettyClientBase(RpcConnectionKey rpcConnectionKey, int retryNumToConnect, int connTimeoutMillis) - throws ClassNotFoundException, NoSuchMethodException { + public NettyClientBase(RpcConnectionKey rpcConnectionKey, + int maxRetryNum, + long connTimeout) throws ClassNotFoundException, NoSuchMethodException { this.key = rpcConnectionKey; - this.maxRetryNumToConnect = retryNumToConnect; - this.connTimeoutMillis = connTimeoutMillis; + this.maxRetryNum = maxRetryNum; + this.connTimeoutMillis = connTimeout; + // Netty only takes integer value range and this is to avoid integer overflow. + Preconditions.checkArgument(this.connTimeoutMillis > Integer.MAX_VALUE, "Too long connection timeout"); } // should be called from sub class @@ -93,10 +91,9 @@ protected void init(ChannelInitializer initializer, EventLoopGroup even .handler(initializer) .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) .option(ChannelOption.SO_REUSEADDR, true) - .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, RpcConstants.DEFAULT_CONNECT_TIMEOUT) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, (int) connTimeoutMillis) .option(ChannelOption.SO_RCVBUF, 1048576 * 10) - .option(ChannelOption.TCP_NODELAY, true) - .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, connTimeoutMillis); + .option(ChannelOption.TCP_NODELAY, true); } public RpcClientManager.RpcConnectionKey getKey() { @@ -153,7 +150,7 @@ public void operationComplete(final ChannelFuture future) throws Exception { getHandler().registerCallback(rpcRequest.getId(), callback); } else { - if (!future.channel().isActive() && retry < maxRetryNumToConnect) { + if (!future.channel().isActive() && retry < maxRetryNum) { /* schedule the current request for the retry */ LOG.warn(future.cause() + " Try to reconnect :" + getKey().addr); @@ -207,7 +204,7 @@ public synchronized void connect() throws ConnectException { ChannelFuture f = doConnect(address).awaitUninterruptibly(); if (!f.isSuccess()) { - if (maxRetryNumToConnect > 0) { + if (maxRetryNum > 0) { doReconnect(address, f, ++retries); } else { throw new ConnectException(ExceptionUtils.getMessage(f.cause())); @@ -219,7 +216,7 @@ private void doReconnect(final InetSocketAddress address, ChannelFuture future, throws ConnectException { for (; ; ) { - if (maxRetryNumToConnect > retries) { + if (maxRetryNum > retries) { retries++; if(getChannel().eventLoop().isShuttingDown()) { diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java index 8787dee3cb..d5a19f5271 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java @@ -34,16 +34,15 @@ class ProtoClientChannelInitializer extends ChannelInitializer { private final MessageLite defaultInstance; private final ChannelHandler handler; + private final boolean useIdleTimeout; private final long timeoutTimeNanos; - private final boolean enablePing; public ProtoClientChannelInitializer(ChannelHandler handler, MessageLite defaultInstance, - long timeoutTimeNanos, - boolean enablePing) { + boolean useIdleTimeout, long idleTimeoutNanos) { this.handler = handler; this.defaultInstance = defaultInstance; - this.timeoutTimeNanos = timeoutTimeNanos; - this.enablePing = enablePing; + this.timeoutTimeNanos = idleTimeoutNanos; + this.useIdleTimeout = useIdleTimeout; } @Override @@ -52,7 +51,7 @@ protected void initChannel(Channel channel) throws Exception { pipeline.addLast("idleStateHandler", new IdleStateHandler(timeoutTimeNanos, timeoutTimeNanos / 2, 0, TimeUnit.NANOSECONDS)); - if (enablePing) pipeline.addLast("MonitorClientHandler", new MonitorClientHandler()); + if (useIdleTimeout) pipeline.addLast("MonitorClientHandler", new MonitorClientHandler()); pipeline.addLast("frameDecoder", new ProtobufVarint32FrameDecoder()); pipeline.addLast("protobufDecoder", new ProtobufDecoder(defaultInstance)); diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java index d3497151c0..4ad7b8315c 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java @@ -39,7 +39,8 @@ public class RpcClientManager { private static final Log LOG = LogFactory.getLog(RpcClientManager.class); - private volatile int timeoutSeconds = RpcConstants.DEFAULT_RPC_TIMEOUT_SECONDS; + private volatile long connTimeout = RpcConstants.DEFAULT_CONNECTION_TIMEOUT; + private volatile long socketTimeout = RpcConstants.DEFAULT_SOCKET_TIMEOUT; private volatile int retries = RpcConstants.DEFAULT_RPC_RETRIES; /* entries will be removed by ConnectionCloseFutureListener */ @@ -61,26 +62,28 @@ public static RpcClientManager getInstance() { } private T makeClient(RpcConnectionKey rpcConnectionKey, - int connRetryNum, - boolean useIdleTimeout, long idleTimeout, TimeUnit timeUnit) + int retryNum, + long connTimeout, + boolean useSocketTimeout, + long socketTimeout) throws NoSuchMethodException, ConnectException, ClassNotFoundException { - return makeClient(rpcConnectionKey, NettyUtils.getDefaultEventLoopGroup(), connRetryNum, useIdleTimeout, - idleTimeout, timeUnit); + return makeClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, retryNum, connTimeout, useSocketTimeout, socketTimeout); } - private T makeClient(RpcConnectionKey rpcConnectionKey, - EventLoopGroup eventLoopGroup, - int connRetryNum, - boolean useIdleTimeout, long idleTimeout, TimeUnit timeUnit) + private T makeClient(EventLoopGroup eventLoopGroup, + RpcConnectionKey rpcConnectionKey, + int retryNum, + long connTimeout, + boolean useSocketTimeout, long socketTimeout) throws NoSuchMethodException, ClassNotFoundException, ConnectException { NettyClientBase client; if (rpcConnectionKey.asyncMode) { - client = new AsyncRpcClient( - rpcConnectionKey, connRetryNum, idleTimeout, timeUnit, useIdleTimeout, eventLoopGroup); + client = new AsyncRpcClient(eventLoopGroup, rpcConnectionKey, retryNum, connTimeout, useSocketTimeout, + socketTimeout); } else { - client = new BlockingRpcClient( - rpcConnectionKey, eventLoopGroup, connRetryNum, useIdleTimeout, idleTimeout, timeUnit); + client = new BlockingRpcClient(eventLoopGroup, rpcConnectionKey, retryNum, connTimeout, useSocketTimeout, + socketTimeout); } return (T) client; } @@ -98,7 +101,7 @@ public T getClient(InetSocketAddress addr, synchronized (clients) { client = clients.get(key); if (client == null) { - clients.put(key, client = makeClient(key, retries, true, getTimeoutSeconds(), TimeUnit.SECONDS)); + clients.put(key, client = makeClient(key, retries, connTimeout, true, getSocketTimeout())); } } @@ -129,16 +132,17 @@ public void channelUnregistered(ChannelHandlerContext ctx) { * Connect a {@link NettyClientBase} to the remote {@link NettyServerBase}, and returns rpc client by protocol. * This client does not managed. It should close. */ - public synchronized T newClient(InetSocketAddress addr, + public T newClient(InetSocketAddress addr, Class protocolClass, boolean asyncMode, - int connRetryNum, + int retryNum, + long connTimeout, boolean useIdleTimeout, - long idleTimeout, - TimeUnit timeUnit) + long idleTimeout) throws NoSuchMethodException, ClassNotFoundException, ConnectException { - return newClient(new RpcConnectionKey(addr, protocolClass, asyncMode), connRetryNum, useIdleTimeout, idleTimeout, timeUnit); + return newClient(new RpcConnectionKey(addr, protocolClass, asyncMode), retryNum, connTimeout, useIdleTimeout, + idleTimeout); } public synchronized T newClient(InetSocketAddress addr, @@ -147,16 +151,12 @@ public synchronized T newClient(InetSocketAddress ad throws NoSuchMethodException, ClassNotFoundException, ConnectException { return newClient(new RpcConnectionKey(addr, protocolClass, asyncMode), - retries, true, getTimeoutSeconds(), TimeUnit.SECONDS); + retries, 0, true, TimeUnit.SECONDS.toMillis(getSocketTimeout())); } /** * * @param key RpcConnectionKey - * @param connRetryNum Connection Retry Number - * @param useIdueTimeout Idle timeout is enabled if true - * @param idleTimeout Idle timeout duration based on timeUnit - * @param timeUnit Time unit * @param Rpc Protocol Class * @return Rpc Client Class * @throws NoSuchMethodException @@ -164,14 +164,14 @@ public synchronized T newClient(InetSocketAddress ad * @throws ConnectException */ public synchronized T newClient(RpcConnectionKey key, - int connRetryNum, - boolean useIdueTimeout, - long idleTimeout, - TimeUnit timeUnit) + int retryNum, + long connTimeout, + boolean useIdleTimeout, + long idleTimeout) throws NoSuchMethodException, ClassNotFoundException, ConnectException { - T client = makeClient(key, connRetryNum, useIdueTimeout, idleTimeout, timeUnit); + T client = makeClient(key, retryNum, connTimeout, useIdleTimeout, idleTimeout); client.connect(); assert client.isConnected(); return client; @@ -220,12 +220,12 @@ public static void cleanup(NettyClientBase... clients) { } } - public int getTimeoutSeconds() { - return timeoutSeconds; + public long getSocketTimeout() { + return socketTimeout; } - public void setTimeoutSeconds(int timeoutSeconds) { - this.timeoutSeconds = timeoutSeconds; + public void setSocketTimeout(int socketTimeout) { + this.socketTimeout = socketTimeout; } public int getRetries() { diff --git a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java index 86611bb523..8efc416d44 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java @@ -132,7 +132,7 @@ public void setUpRpcClient() throws Exception { rpcConnectionKey = new RpcClientManager.RpcConnectionKey( RpcUtils.getConnectAddress(server.getListenAddress()), DummyProtocol.class, true); - client = manager.newClient(rpcConnectionKey, retries, true, 10, TimeUnit.SECONDS); + client = manager.newClient(rpcConnectionKey, retries, 1000, true, 10); assertTrue(client.isConnected()); stub = client.getStub(); } @@ -349,8 +349,7 @@ public void run() { RpcClientManager.RpcConnectionKey rpcConnectionKey = new RpcClientManager.RpcConnectionKey(address, DummyProtocol.class, true); - AsyncRpcClient client = manager.newClient(rpcConnectionKey, - retries, false, 0, TimeUnit.MILLISECONDS); + AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, false, 0); assertTrue(client.isConnected()); Interface stub = client.getStub(); @@ -455,7 +454,7 @@ public void testUnresolvedAddress2() throws Exception { public void testStubRecovery() throws Exception { RpcClientManager.RpcConnectionKey rpcConnectionKey = new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); - AsyncRpcClient client = manager.newClient(rpcConnectionKey, 2, false, 0, TimeUnit.MILLISECONDS); + AsyncRpcClient client = manager.newClient(rpcConnectionKey, 2, 1000, false, 0); EchoMessage echoMessage = EchoMessage.newBuilder() .setMessage(MESSAGE).build(); @@ -487,7 +486,7 @@ public void testIdleTimeout() throws Exception { RpcClientManager.RpcConnectionKey rpcConnectionKey = new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); //500 millis idle timeout - AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, false, 500, TimeUnit.MILLISECONDS); + AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, false, 500); assertTrue(client.isConnected()); Thread.sleep(600); //timeout @@ -508,7 +507,7 @@ public void testPingOnIdle() throws Exception { new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); //500 millis request timeout - AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, true, 500, TimeUnit.MILLISECONDS); + AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, true, 500); assertTrue(client.isConnected()); Thread.sleep(600); @@ -525,7 +524,7 @@ public void testIdleTimeoutWithActiveRequest() throws Exception { RpcClientManager.RpcConnectionKey rpcConnectionKey = new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); //500 millis idle timeout - AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, false, 500, TimeUnit.MILLISECONDS); + AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, false, 500); assertTrue(client.isConnected()); Interface stub = client.getStub(); @@ -551,7 +550,7 @@ public void testRequestTimeoutOnBusy() throws Exception { new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); //500 millis request timeout - AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, true, 500, TimeUnit.MILLISECONDS); + AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, true, 500); assertTrue(client.isConnected()); Interface stub = client.getStub(); diff --git a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java index 1e2709381f..a70353475c 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java @@ -123,7 +123,7 @@ public void setUpRpcClient() throws Exception { new RpcClientManager.RpcConnectionKey( RpcUtils.getConnectAddress(server.getListenAddress()), DummyProtocol.class, false); - client = manager.newClient(rpcConnectionKey, retries, true, 10, TimeUnit.SECONDS); + client = manager.newClient(rpcConnectionKey, retries, 1000, true, 10); assertTrue(client.isConnected()); stub = client.getStub(); } @@ -320,8 +320,7 @@ public void run() { RpcClientManager.RpcConnectionKey rpcConnectionKey = new RpcClientManager.RpcConnectionKey(address, DummyProtocol.class, false); - BlockingRpcClient client = manager.newClient(rpcConnectionKey, - retries, false, 0, TimeUnit.MILLISECONDS); + BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, false, 0); assertTrue(client.isConnected()); BlockingInterface stub = client.getStub(); @@ -412,7 +411,7 @@ public void testUnresolvedAddress2() throws Exception { public void testStubRecovery() throws Exception { RpcClientManager.RpcConnectionKey rpcConnectionKey = new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); - BlockingRpcClient client = manager.newClient(rpcConnectionKey, 1, false, 0, TimeUnit.MILLISECONDS); + BlockingRpcClient client = manager.newClient(rpcConnectionKey, 1, 1000, false, 0); EchoMessage echoMessage = EchoMessage.newBuilder() .setMessage(MESSAGE).build(); @@ -443,7 +442,7 @@ public void testIdleTimeout() throws Exception { RpcClientManager.RpcConnectionKey rpcConnectionKey = new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); //500 millis idle timeout - BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, false, 500, TimeUnit.MILLISECONDS); + BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, false, 500); assertTrue(client.isConnected()); Thread.sleep(600); //timeout @@ -464,7 +463,7 @@ public void testPingOnIdle() throws Exception { new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); //500 millis request timeout - BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, true, 500, TimeUnit.MILLISECONDS); + BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, true, 500); assertTrue(client.isConnected()); Thread.sleep(600); @@ -481,7 +480,7 @@ public void testIdleTimeoutWithActiveRequest() throws Exception { RpcClientManager.RpcConnectionKey rpcConnectionKey = new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); //500 millis idle timeout - BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, false, 500, TimeUnit.MILLISECONDS); + BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, false, 500); assertTrue(client.isConnected()); BlockingInterface stub = client.getStub(); @@ -504,7 +503,7 @@ public void testRequestTimeoutOnBusy() throws Exception { new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); //500 millis request timeout - BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, true, 500, TimeUnit.MILLISECONDS); + BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, true, 500); assertTrue(client.isConnected()); BlockingInterface stub = client.getStub(); diff --git a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestRpcClientManager.java b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestRpcClientManager.java index 3834351a72..a321ab7921 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestRpcClientManager.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestRpcClientManager.java @@ -150,11 +150,11 @@ public void testUnManagedClient() throws Exception { RpcClientManager manager = RpcClientManager.getInstance(); try { - NettyClientBase client1 = manager.newClient(key, 0, false, 0, TimeUnit.SECONDS); + NettyClientBase client1 = manager.newClient(key, 0, 1000, false, 0); assertTrue(client1.isConnected()); assertFalse(RpcClientManager.contains(key)); - NettyClientBase client2 = manager.newClient(key, 0, false, 0, TimeUnit.SECONDS); + NettyClientBase client2 = manager.newClient(key, 0, 1000, false, 0); assertTrue(client2.isConnected()); assertFalse(RpcClientManager.contains(key)); From 598267556fa7819748661a32959c5ca65c3f20d9 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Wed, 16 Sep 2015 00:49:16 -0700 Subject: [PATCH 03/24] Completed. --- .../apache/tajo/catalog/CatalogClient.java | 11 +- .../apache/tajo/client/QueryClientImpl.java | 6 +- .../apache/tajo/client/SessionConnection.java | 44 +++---- .../java/org/apache/tajo/conf/TajoConf.java | 3 + .../apache/tajo/master/QueryInProgress.java | 12 +- .../org/apache/tajo/master/TajoMaster.java | 13 +- .../querymaster/DefaultTaskScheduler.java | 21 ++- .../apache/tajo/querymaster/QueryMaster.java | 20 +-- .../tajo/querymaster/QueryMasterTask.java | 11 +- .../org/apache/tajo/querymaster/Stage.java | 7 +- .../tajo/util/RpcConnectionParamUtil.java | 50 +++++++ .../tajo/worker/ExecutionBlockContext.java | 6 +- .../apache/tajo/worker/NodeStatusUpdater.java | 20 +-- .../org/apache/tajo/worker/TajoWorker.java | 5 - .../org/apache/tajo/worker/TaskManager.java | 12 +- .../ConnectivityCheckerRuleForTajoWorker.java | 4 +- .../org/apache/tajo/rpc/RpcConstants.java | 27 ++-- .../org/apache/tajo/rpc/AsyncRpcClient.java | 29 ++--- .../apache/tajo/rpc/BlockingRpcClient.java | 41 +++--- .../org/apache/tajo/rpc/NettyClientBase.java | 23 ++-- .../rpc/ProtoClientChannelInitializer.java | 21 +-- .../org/apache/tajo/rpc/RpcClientManager.java | 122 +++++------------- .../org/apache/tajo/rpc/RpcConnectionKey.java | 56 ++++++++ .../org/apache/tajo/rpc/TestAsyncRpc.java | 114 ++++++++++------ .../org/apache/tajo/rpc/TestBlockingRpc.java | 111 +++++++++++----- .../apache/tajo/rpc/TestRpcClientManager.java | 22 ++-- 26 files changed, 481 insertions(+), 330 deletions(-) create mode 100644 tajo-core/src/main/java/org/apache/tajo/util/RpcConnectionParamUtil.java create mode 100644 tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcConnectionKey.java diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java index 643226ebcd..9bd6afd859 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java @@ -22,15 +22,14 @@ import org.apache.tajo.catalog.CatalogProtocol.CatalogProtocolService.BlockingInterface; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; -import org.apache.tajo.rpc.NettyClientBase; -import org.apache.tajo.rpc.RpcClientManager; -import org.apache.tajo.rpc.RpcConstants; +import org.apache.tajo.rpc.*; import org.apache.tajo.service.ServiceTracker; import org.apache.tajo.service.ServiceTrackerFactory; import org.apache.tajo.util.NetUtils; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.Properties; /** * CatalogClient provides a client API to access the catalog server. @@ -70,8 +69,12 @@ public synchronized NettyClientBase getCatalogConnection() throws ServiceExcepti if (client != null && client.isConnected()) return client; RpcClientManager.cleanup(client); + final Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, conf.getVar(ConfVars.RPC_CLIENT_RETRY_NUM)); + // Client do not closed on idle state for support high available - this.client = RpcClientManager.getInstance().newClient(getCatalogServerAddr(), CatalogProtocol.class, false); + this.client = RpcClientManager.getInstance().newClient(getCatalogServerAddr(), CatalogProtocol.class, + false, connParams); } catch (Exception e) { throw new ServiceException(e); } diff --git a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java index c0833c6009..c929d159a9 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java @@ -50,6 +50,7 @@ import java.sql.ResultSet; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -544,10 +545,7 @@ public QueryHistoryProto getQueryHistory(final QueryId queryId) throws QueryNotF try { - qmClient = manager.newClient( - qmAddress, - QueryMasterClientProtocol.class, false); - + qmClient = manager.newClient(qmAddress, QueryMasterClientProtocol.class, false, new Properties()); conn.checkSessionAndGet(conn.getTajoMasterConnection()); diff --git a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java index 3873e7e50c..19beb9b377 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java @@ -19,6 +19,7 @@ package org.apache.tajo.client; import com.google.protobuf.ServiceException; +import io.netty.channel.EventLoopGroup; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tajo.SessionVars; @@ -37,27 +38,21 @@ import org.apache.tajo.ipc.TajoMasterClientProtocol; import org.apache.tajo.ipc.TajoMasterClientProtocol.TajoMasterClientProtocolService.BlockingInterface; import org.apache.tajo.rpc.NettyClientBase; -import org.apache.tajo.rpc.RpcChannelFactory; +import org.apache.tajo.rpc.NettyUtils; import org.apache.tajo.rpc.RpcClientManager; import org.apache.tajo.rpc.RpcConstants; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetResponse; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.ReturnState; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringResponse; import org.apache.tajo.service.ServiceTracker; -import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.ProtoUtil; import java.io.Closeable; import java.net.InetSocketAddress; import java.sql.SQLException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import static org.apache.tajo.error.Errors.ResultCode.NO_SUCH_SESSION_VARIABLE; import static org.apache.tajo.exception.ReturnStateUtil.*; @@ -68,8 +63,6 @@ public class SessionConnection implements Closeable { private final static Log LOG = LogFactory.getLog(SessionConnection.class); - private final static AtomicInteger connections = new AtomicInteger(); - final RpcClientManager manager; private String baseDatabase; @@ -85,8 +78,12 @@ public class SessionConnection implements Closeable { private final ServiceTracker serviceTracker; + private final EventLoopGroup eventLoopGroup; + private NettyClientBase client; + private Properties clientConnParams; + private final KeyValueSet properties; /** @@ -105,10 +102,18 @@ public SessionConnection(@NotNull ServiceTracker tracker, @Nullable String baseD this.properties = properties; this.manager = RpcClientManager.getInstance(); - this.manager.setRetries(properties.getInt(RpcConstants.RPC_CLIENT_RETRY_MAX, RpcConstants.DEFAULT_RPC_RETRIES)); + this.clientConnParams = new Properties(); + this.clientConnParams.setProperty(RpcConstants.RPC_RETRY_NUM, properties.get(RpcConstants.RPC_RETRY_NUM)); + this.userInfo = UserRoleInfo.getCurrentUser(); - this.client = getTajoMasterConnection(); + this.eventLoopGroup = NettyUtils.createEventLoopGroup(getClass().getSimpleName(), 4); + try { + this.client = getTajoMasterConnection(); + } catch (TajoRuntimeException e) { + NettyUtils.shutdown(eventLoopGroup); + throw e; + } } public Map getClientSideSessionVars() { @@ -125,9 +130,8 @@ public synchronized NettyClientBase getTajoMasterConnection() { RpcClientManager.cleanup(client); // Client do not closed on idle state for support high available - this.client = manager.newClient(getTajoMasterAddr(), TajoMasterClientProtocol.class, false ); - connections.incrementAndGet(); - + this.client = manager.newBlockingClient(getTajoMasterAddr(), TajoMasterClientProtocol.class, + eventLoopGroup, clientConnParams); } catch (Throwable t) { throw new TajoRuntimeException(new ClientConnectionException(t)); } @@ -338,14 +342,7 @@ public void close() { // ignore } finally { RpcClientManager.cleanup(client); - if(connections.decrementAndGet() == 0) { - if (!System.getProperty(CommonTestingUtil.TAJO_TEST_KEY, "FALSE").equals(CommonTestingUtil.TAJO_TEST_TRUE)) { - RpcChannelFactory.shutdownGracefully(); - if (LOG.isDebugEnabled()) { - LOG.debug("RPC connection is closed"); - } - } - } + NettyUtils.shutdown(eventLoopGroup); } } @@ -449,5 +446,4 @@ ClientProtos.SessionedStringProto getSessionedString(String str) { } return builder.build(); } - } diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index f9b9201308..7e4c591ae6 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -229,6 +229,9 @@ public static enum ConfVars implements ConfigKey { // Internal RPC Client INTERNAL_RPC_CLIENT_WORKER_THREAD_NUM("tajo.internal.rpc.client.worker-thread-num", Runtime.getRuntime().availableProcessors() * 2), + RPC_CLIENT_RETRY_NUM("tajo.rpc.client.retry-num", 3, Validators.min("1")), + RPC_CLIENT_CONNECTION_TIMEOUT("tajo.rpc.client.connection-timeout-ms", 15 * 1000, Validators.min("1")), + RPC_CLIENT_SOCKET_TIMEOUT("tajo.rpc.client.socket-timeout-ms", 180 * 1000, Validators.min("1")), // Internal RPC Server MASTER_RPC_SERVER_WORKER_THREAD_NUM("tajo.master.rpc.server.worker-thread-num", diff --git a/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java b/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java index b8488768ea..a24165a78b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java @@ -39,9 +39,11 @@ import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.session.Session; import org.apache.tajo.util.NetUtils; +import org.apache.tajo.util.RpcConnectionParamUtil; import java.net.ConnectException; import java.net.InetSocketAddress; +import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; @@ -70,6 +72,8 @@ public class QueryInProgress { private AllocationResourceProto allocationResource; + private final Properties rpcClientParams; + private final Lock readLock; private final Lock writeLock; @@ -87,6 +91,8 @@ public QueryInProgress( queryInfo = new QueryInfo(queryId, queryContext, sql, jsonExpr); queryInfo.setStartTime(System.currentTimeMillis()); + rpcClientParams = RpcConnectionParamUtil.get(masterContext.getConf()); + ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); this.writeLock = readWriteLock.writeLock(); @@ -99,7 +105,7 @@ public void kill() { if (queryMasterRpcClient != null) { CallFuture callFuture = new CallFuture(); queryMasterRpcClient.killQuery(callFuture.getController(), queryId.getProto(), callFuture); - callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); } } catch (Throwable e) { catchException("Failed to kill query " + queryId + " by exception " + e, e); @@ -182,7 +188,7 @@ private void connectQueryMaster(WorkerConnectionInfo connectionInfo) InetSocketAddress addr = NetUtils.createSocketAddr(connectionInfo.getHost(), connectionInfo.getQueryMasterPort()); LOG.info("Try to connect to QueryMaster:" + addr); - queryMasterRpc = RpcClientManager.getInstance().newClient(addr, QueryMasterProtocol.class, true); + queryMasterRpc = RpcClientManager.getInstance().newClient(addr, QueryMasterProtocol.class, true, rpcClientParams); queryMasterRpcClient = queryMasterRpc.getStub(); } @@ -216,7 +222,7 @@ public boolean submitToQueryMaster() { CallFuture callFuture = new CallFuture(); queryMasterRpcClient.executeQuery(callFuture.getController(), builder.build(), callFuture); - callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); querySubmitted = true; getQueryInfo().setQueryState(TajoProtos.QueryState.QUERY_MASTER_LAUNCHED); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index de71f935ce..2b9beddc3c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -50,7 +50,6 @@ import org.apache.tajo.metrics.Master; import org.apache.tajo.plan.function.python.PythonScriptEngine; import org.apache.tajo.rpc.RpcClientManager; -import org.apache.tajo.rpc.RpcConstants; import org.apache.tajo.rule.EvaluationContext; import org.apache.tajo.rule.EvaluationFailedException; import org.apache.tajo.rule.SelfDiagnosisRuleEngine; @@ -167,22 +166,16 @@ public void serviceInit(Configuration conf) throws Exception { context = new MasterContext(systemConf); clock = new SystemClock(); - RackResolver.init(systemConf); - RpcClientManager rpcManager = RpcClientManager.getInstance(); - rpcManager.setRetries(systemConf.getInt(RpcConstants.RPC_CLIENT_RETRY_MAX, RpcConstants.DEFAULT_RPC_RETRIES)); - rpcManager.setSocketTimeout( - systemConf.getInt(RpcConstants.RPC_CLIENT_TIMEOUT_SECS, RpcConstants.DEFAULT_RPC_TIMEOUT_SECONDS)); - initResourceManager(); this.dispatcher = new AsyncDispatcher(); addIfService(dispatcher); - // check the system directory and create if they are not created. - checkAndInitializeSystemDirectories(); - diagnoseTajoMaster(); + // check the system directory and create if they are not created. + checkAndInitializeSystemDirectories(); + diagnoseTajoMaster(); catalogServer = new CatalogServer(Collections.EMPTY_SET, loadFunctions()); addIfService(catalogServer); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java index d3802955db..cdeb5988b2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java @@ -48,6 +48,7 @@ import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.NetUtils; +import org.apache.tajo.util.RpcConnectionParamUtil; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.FetchImpl; @@ -69,6 +70,7 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler { private final TaskSchedulerContext context; private Stage stage; private TajoConf tajoConf; + private Properties rpcConnParams; private Thread schedulingThread; private volatile boolean isStopped; @@ -83,7 +85,7 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler { private int schedulerDelay; private int maximumRequestContainer; - //candidate workers for locality of high priority + // candidate workers for locality of high priority private Set candidateWorkers = Sets.newHashSet(); public DefaultTaskScheduler(TaskSchedulerContext context, Stage stage) { @@ -95,6 +97,8 @@ public DefaultTaskScheduler(TaskSchedulerContext context, Stage stage) { @Override public void init(Configuration conf) { tajoConf = TUtil.checkTypeAndGet(conf, TajoConf.class); + rpcConnParams = RpcConnectionParamUtil.get(new TajoConf()); + scheduledRequests = new ScheduledRequests(); minTaskMemory = tajoConf.getIntVar(TajoConf.ConfVars.TASK_RESOURCE_MINIMUM_MEMORY); schedulerDelay= tajoConf.getIntVar(TajoConf.ConfVars.QUERYMASTER_TASK_SCHEDULER_DELAY); @@ -294,7 +298,7 @@ protected LinkedList createTaskRequest(final int incompleteTas ServiceTracker serviceTracker = context.getMasterContext().getQueryMasterContext().getWorkerContext().getServiceTracker(); NettyClientBase tmClient = RpcClientManager.getInstance(). - getClient(serviceTracker.getUmbilicalAddress(), QueryCoordinatorProtocol.class, true); + getClient(serviceTracker.getUmbilicalAddress(), QueryCoordinatorProtocol.class, true, rpcConnParams); QueryCoordinatorProtocolService masterClientService = tmClient.getStub(); CallFuture callBack = new CallFuture(); @@ -310,7 +314,7 @@ protected LinkedList createTaskRequest(final int incompleteTas .setQueue(context.getMasterContext().getQueryContext().get("queue", "default")); //TODO set queue masterClientService.reserveNodeResources(callBack.getController(), request.build(), callBack); - NodeResourceResponse response = callBack.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + NodeResourceResponse response = callBack.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); for (AllocationResourceProto resource : response.getResourceList()) { taskRequestEvents.add(new TaskRequestEvent(resource.getWorkerId(), resource, context.getBlockId())); @@ -885,12 +889,14 @@ public void assignToLeafTasks(LinkedList taskRequests) { CallFuture callFuture = new CallFuture(); totalAttempts++; try { - tajoWorkerRpc = RpcClientManager.getInstance().getClient(addr, TajoWorkerProtocol.class, true); + tajoWorkerRpc = RpcClientManager.getInstance().getClient(addr, TajoWorkerProtocol.class, true, + rpcConnParams); + TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); tajoWorkerRpcClient.allocateTasks(callFuture.getController(), requestProto.build(), callFuture); BatchAllocationResponse responseProto = - callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); if (responseProto.getCancellationTaskCount() > 0) { for (TaskAllocationProto proto : responseProto.getCancellationTaskList()) { @@ -1003,12 +1009,13 @@ public void assignToNonLeafTasks(LinkedList taskRequests) { AsyncRpcClient tajoWorkerRpc; try { - tajoWorkerRpc = RpcClientManager.getInstance().getClient(addr, TajoWorkerProtocol.class, true); + tajoWorkerRpc = RpcClientManager.getInstance().getClient(addr, TajoWorkerProtocol.class, true, + rpcConnParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); tajoWorkerRpcClient.allocateTasks(callFuture.getController(), requestProto.build(), callFuture); BatchAllocationResponse - responseProto = callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + responseProto = callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); if(responseProto.getCancellationTaskCount() > 0) { for (TaskAllocationProto proto : responseProto.getCancellationTaskList()) { diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java index cce94825f3..3d2ca75bd6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java @@ -45,6 +45,7 @@ import org.apache.tajo.rpc.*; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.service.ServiceTracker; +import org.apache.tajo.util.RpcConnectionParamUtil; import org.apache.tajo.util.TUtil; import org.apache.tajo.util.history.HistoryWriter.WriterFuture; import org.apache.tajo.util.history.HistoryWriter.WriterHolder; @@ -52,10 +53,7 @@ import org.apache.tajo.worker.TajoWorker; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -91,6 +89,8 @@ public class QueryMaster extends CompositeService implements EventHandler { private RpcClientManager manager; + private Properties rpcClientParams; + private ExecutorService eventExecutor; private ExecutorService singleEventExecutor; @@ -105,6 +105,7 @@ public void serviceInit(Configuration conf) throws Exception { this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class); this.manager = RpcClientManager.getInstance(); + this.rpcClientParams = RpcConnectionParamUtil.get(this.systemConf); querySessionTimeout = systemConf.getIntVar(TajoConf.ConfVars.QUERY_SESSION_TIMEOUT); queryMasterContext = new QueryMasterContext(systemConf); @@ -171,7 +172,8 @@ public List getAllWorker() { // update master address in worker context. ServiceTracker serviceTracker = workerContext.getServiceTracker(); - rpc = manager.getClient(serviceTracker.getUmbilicalAddress(), QueryCoordinatorProtocol.class, true); + rpc = manager.getClient(serviceTracker.getUmbilicalAddress(), QueryCoordinatorProtocol.class, true, + rpcClientParams); QueryCoordinatorProtocolService masterService = rpc.getStub(); CallFuture callBack = new CallFuture(); @@ -179,7 +181,7 @@ public List getAllWorker() { PrimitiveProtos.NullProto.getDefaultInstance(), callBack); WorkerConnectionsResponse connectionsProto = - callBack.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + callBack.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); return connectionsProto.getWorkerList(); } catch (Exception e) { LOG.error(e.getMessage(), e); @@ -294,11 +296,11 @@ public void stopQuery(final QueryId queryId) { NettyClientBase tmClient; try { tmClient = manager.getClient(workerContext.getServiceTracker().getUmbilicalAddress(), - QueryCoordinatorProtocol.class, true); + QueryCoordinatorProtocol.class, true, rpcClientParams); QueryCoordinatorProtocolService masterClientService = tmClient.getStub(); masterClientService.heartbeat(future.getController(), queryHeartbeat, future); - future.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + future.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); } catch (Exception e) { //this function will be closed in new thread. //When tajo do stop cluster, tajo master maybe throw closed connection exception @@ -404,7 +406,7 @@ public void run() { ServiceTracker serviceTracker = queryMasterContext.getWorkerContext().getServiceTracker(); tmClient = manager.getClient(serviceTracker.getUmbilicalAddress(), - QueryCoordinatorProtocol.class, true); + QueryCoordinatorProtocol.class, true, rpcClientParams); QueryCoordinatorProtocolService masterClientService = tmClient.getStub(); TajoHeartbeatRequest queryHeartbeat = buildTajoHeartBeat(eachTask); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java index 1313dade48..dbaef59102 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java @@ -55,6 +55,7 @@ import org.apache.tajo.storage.FormatProperty; import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.TablespaceManager; +import org.apache.tajo.util.RpcConnectionParamUtil; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.event.NodeResourceDeallocateEvent; import org.apache.tajo.worker.event.NodeResourceEvent; @@ -95,6 +96,8 @@ public class QueryMasterTask extends CompositeService { private TajoConf systemConf; + private Properties rpcConnParams; + private AtomicLong lastClientHeartbeat = new AtomicLong(-1); private volatile boolean isStopped; @@ -131,8 +134,8 @@ public QueryMasterTask(QueryMaster.QueryMasterContext queryMasterContext, @Override public void serviceInit(Configuration conf) throws Exception { - systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class); + rpcConnParams = RpcConnectionParamUtil.get(systemConf); queryTaskContext = new QueryMasterTaskContext(); @@ -255,7 +258,8 @@ protected void killTaskAttempt(int workerId, TaskAttemptId taskAttemptId) { InetSocketAddress workerAddress = getQuery().getStage(ebId).getAssignedWorkerMap().get(workerId); try { - tajoWorkerRpc = RpcClientManager.getInstance().getClient(workerAddress, TajoWorkerProtocol.class, true); + tajoWorkerRpc = RpcClientManager.getInstance().getClient(workerAddress, TajoWorkerProtocol.class, true, + rpcConnParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); CallFuture callFuture = new CallFuture(); tajoWorkerRpcClient.killTaskAttempt(null, taskAttemptId.getProto(), callFuture); @@ -472,7 +476,8 @@ private void cleanupQuery(final QueryId queryId) { @Override public void run() { try { - AsyncRpcClient rpc = RpcClientManager.getInstance().getClient(worker, TajoWorkerProtocol.class, true); + AsyncRpcClient rpc = RpcClientManager.getInstance().getClient(worker, TajoWorkerProtocol.class, true, + rpcConnParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerProtocolService = rpc.getStub(); tajoWorkerProtocolService.stopQuery(null, queryId.getProto(), NullCallback.get()); } catch (Throwable e) { diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index 68916e492e..ea2465b1d7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -62,6 +62,7 @@ import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.KeyValueSet; +import org.apache.tajo.util.RpcConnectionParamUtil; import org.apache.tajo.util.TUtil; import org.apache.tajo.util.history.StageHistory; import org.apache.tajo.util.history.TaskHistory; @@ -88,6 +89,8 @@ public class Stage implements EventHandler { private static final Log LOG = LogFactory.getLog(Stage.class); + private final Properties rpcClientParams; + private MasterPlan masterPlan; private ExecutionBlock block; private int priority; @@ -300,6 +303,8 @@ public Stage(QueryMasterTask.QueryMasterTaskContext context, MasterPlan masterPl this.block = block; this.eventHandler = context.getEventHandler(); + this.rpcClientParams = RpcConnectionParamUtil.get(context.getConf()); + ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); this.writeLock = readWriteLock.writeLock(); @@ -720,7 +725,7 @@ private void sendStopExecutionBlockEvent(final StopExecutionBlockRequest request public void run() { try { AsyncRpcClient tajoWorkerRpc = - RpcClientManager.getInstance().getClient(worker, TajoWorkerProtocol.class, true); + RpcClientManager.getInstance().getClient(worker, TajoWorkerProtocol.class, true, rpcClientParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); tajoWorkerRpcClient.stopExecutionBlock(null, requestProto, NullCallback.get(PrimitiveProtos.BoolProto.class)); diff --git a/tajo-core/src/main/java/org/apache/tajo/util/RpcConnectionParamUtil.java b/tajo-core/src/main/java/org/apache/tajo/util/RpcConnectionParamUtil.java new file mode 100644 index 0000000000..9603c9584e --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/util/RpcConnectionParamUtil.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.util; + +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; +import org.apache.tajo.rpc.RpcConstants; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +/** + * Helper class to get RPC Client Connection Parameters + */ +public class RpcConnectionParamUtil { + + static final Map PROPERTIES_MAP = new HashMap<>(); + + static { + PROPERTIES_MAP.put(RpcConstants.RPC_RETRY_NUM, ConfVars.RPC_CLIENT_RETRY_NUM); + PROPERTIES_MAP.put(RpcConstants.CLIENT_CONNECTION_TIMEOUT, ConfVars.RPC_CLIENT_CONNECTION_TIMEOUT); + PROPERTIES_MAP.put(RpcConstants.CLIENT_SOCKET_TIMEOUT, ConfVars.RPC_CLIENT_SOCKET_TIMEOUT); + } + + public static Properties get(TajoConf conf) { + final Properties properties = new Properties(); + + for (Map.Entry e : PROPERTIES_MAP.entrySet()) { + properties.put(e.getKey(), conf.getVar(e.getValue())); + } + + return properties; + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java index 6f923449ff..ba26e3c507 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java @@ -262,7 +262,7 @@ public void fatalError(TaskAttemptId taskAttemptId, String message) { //If QueryMaster does not responding, current execution block should be stop CallFuture callFuture = new CallFuture(); getStub().fatalError(callFuture.getController(), builder.build(), callFuture); - callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); } catch (Exception e) { getWorkerContext().getTaskManager().getDispatcher().getEventHandler() .handle(new ExecutionBlockErrorEvent(taskAttemptId.getTaskId().getExecutionBlockId(), e)); @@ -309,7 +309,7 @@ private void sendHashShuffleReport(ExecutionBlockId ebId) throws Exception { CallFuture callFuture = new CallFuture(); stub.doneExecutionBlock(callFuture.getController(), reporterBuilder.build(), callFuture); - callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); return; } @@ -364,7 +364,7 @@ private void sendHashShuffleReport(ExecutionBlockId ebId) throws Exception { try { CallFuture callFuture = new CallFuture(); stub.doneExecutionBlock(callFuture.getController(), reporterBuilder.build(), callFuture); - callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); } catch (Throwable e) { // can't send report to query master LOG.fatal(e.getMessage(), e); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java b/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java index a1e572582e..3dcb6abe7e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java @@ -35,12 +35,14 @@ import org.apache.tajo.rpc.RpcConstants; import org.apache.tajo.service.ServiceTracker; import org.apache.tajo.service.ServiceTrackerFactory; +import org.apache.tajo.util.RpcConnectionParamUtil; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.event.NodeStatusEvent; import java.net.ConnectException; import java.util.Collection; import java.util.List; +import java.util.Properties; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -55,7 +57,7 @@ public class NodeStatusUpdater extends AbstractService implements EventHandler heartBeatRequestQueue; private final TajoWorker.WorkerContext workerContext; private AsyncRpcClient rmClient; + private Properties rpcClientParams; private ServiceTracker serviceTracker; private TajoResourceTrackerProtocol.TajoResourceTrackerProtocolService.Interface resourceTracker; private int queueingThreshold; @@ -75,11 +78,12 @@ public NodeStatusUpdater(TajoWorker.WorkerContext workerContext) { @Override public void serviceInit(Configuration conf) throws Exception { - this.tajoConf = TUtil.checkTypeAndGet(conf, TajoConf.class); + this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class); + this.rpcClientParams = RpcConnectionParamUtil.get(this.systemConf); this.heartBeatRequestQueue = Queues.newLinkedBlockingQueue(); - this.serviceTracker = ServiceTrackerFactory.get(tajoConf); + this.serviceTracker = ServiceTrackerFactory.get(systemConf); this.workerContext.getNodeResourceManager().getDispatcher().register(NodeStatusEvent.EventType.class, this); - this.heartBeatInterval = tajoConf.getIntVar(TajoConf.ConfVars.WORKER_HEARTBEAT_IDLE_INTERVAL); + this.heartBeatInterval = systemConf.getIntVar(TajoConf.ConfVars.WORKER_HEARTBEAT_IDLE_INTERVAL); this.updaterThread = new StatusUpdaterThread(); this.updaterThread.setName("NodeStatusUpdater"); super.serviceInit(conf); @@ -89,10 +93,10 @@ public void serviceInit(Configuration conf) throws Exception { public void serviceStart() throws Exception { DefaultResourceCalculator calculator = new DefaultResourceCalculator(); int maxContainer = calculator.computeAvailableContainers(workerContext.getNodeResourceManager().getTotalResource(), - NodeResources.createResource(tajoConf.getIntVar(TajoConf.ConfVars.TASK_RESOURCE_MINIMUM_MEMORY), 1)); + NodeResources.createResource(systemConf.getIntVar(TajoConf.ConfVars.TASK_RESOURCE_MINIMUM_MEMORY), 1)); // if resource changed over than 30%, send reports - float queueingRate = tajoConf.getFloatVar(TajoConf.ConfVars.WORKER_HEARTBEAT_QUEUE_THRESHOLD_RATE); + float queueingRate = systemConf.getFloatVar(TajoConf.ConfVars.WORKER_HEARTBEAT_QUEUE_THRESHOLD_RATE); this.queueingThreshold = Math.max((int) Math.floor(maxContainer * queueingRate), 1); LOG.info("Queueing threshold:" + queueingThreshold); @@ -149,7 +153,7 @@ protected TajoResourceTrackerProtocol.TajoResourceTrackerProtocolService.Interfa RpcClientManager rpcManager = RpcClientManager.getInstance(); rmClient = rpcManager.newClient(serviceTracker.getResourceTrackerAddress(), - TajoResourceTrackerProtocol.class, true, rpcManager.getRetries()); + TajoResourceTrackerProtocol.class, true, rpcClientParams); return rmClient.getStub(); } @@ -164,7 +168,7 @@ protected NodeHeartbeatResponse sendHeartbeat(NodeHeartbeatRequest requestProto) CallFuture callBack = new CallFuture(); resourceTracker.nodeHeartbeat(callBack.getController(), requestProto, callBack); - response = callBack.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + response = callBack.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); } catch (InterruptedException e) { LOG.warn(e.getMessage()); } catch (TimeoutException te) { diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java index b62bdb2ff4..607e7ff05b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java @@ -152,11 +152,6 @@ public void serviceInit(Configuration conf) throws Exception { this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class); RackResolver.init(systemConf); - RpcClientManager rpcManager = RpcClientManager.getInstance(); - rpcManager.setRetries(systemConf.getInt(RpcConstants.RPC_CLIENT_RETRY_MAX, RpcConstants.DEFAULT_RPC_RETRIES)); - rpcManager.setSocketTimeout( - systemConf.getInt(RpcConstants.RPC_CLIENT_TIMEOUT_SECS, RpcConstants.DEFAULT_RPC_TIMEOUT_SECONDS)); - serviceTracker = ServiceTrackerFactory.get(systemConf); this.workerContext = new TajoWorkerContext(); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java index 42db852dc1..c3b8f0f097 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java @@ -35,6 +35,7 @@ import org.apache.tajo.rpc.RpcClientManager; import org.apache.tajo.rpc.RpcConstants; import org.apache.tajo.util.NetUtils; +import org.apache.tajo.util.RpcConnectionParamUtil; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.event.*; @@ -42,11 +43,10 @@ import java.net.InetSocketAddress; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.concurrent.TimeUnit; -import static org.apache.tajo.ResourceProtos.ExecutionBlockListProto; -import static org.apache.tajo.ResourceProtos.ExecutionBlockContextRequest; -import static org.apache.tajo.ResourceProtos.ExecutionBlockContextResponse; +import static org.apache.tajo.ResourceProtos.*; /** * A TaskManager is responsible for managing executionBlock resource and tasks. @@ -58,6 +58,7 @@ public class TaskManager extends AbstractService implements EventHandler executionBlockContextMap; private final Dispatcher dispatcher; private TaskExecutor executor; + private final Properties rpcClientParams; public TaskManager(Dispatcher dispatcher, TajoWorker.WorkerContext workerContext){ this(dispatcher, workerContext, null); @@ -70,6 +71,7 @@ public TaskManager(Dispatcher dispatcher, TajoWorker.WorkerContext workerContext this.workerContext = workerContext; this.executionBlockContextMap = Maps.newHashMap(); this.executor = executor; + this.rpcClientParams = RpcConnectionParamUtil.get(this.workerContext.getConf()); } @Override @@ -118,13 +120,13 @@ protected ExecutionBlockContext createExecutionBlock(ExecutionBlockId executionB request.setExecutionBlockId(executionBlockId.getProto()) .setWorker(getWorkerContext().getConnectionInfo().getProto()); - client = RpcClientManager.getInstance().newClient(address, QueryMasterProtocol.class, true); + client = RpcClientManager.getInstance().newClient(address, QueryMasterProtocol.class, true, rpcClientParams); QueryMasterProtocol.QueryMasterProtocolService.Interface stub = client.getStub(); CallFuture callback = new CallFuture(); stub.getExecutionBlockContext(callback.getController(), request.build(), callback); ExecutionBlockContextResponse contextProto = - callback.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + callback.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); ExecutionBlockContext context = new ExecutionBlockContext(getWorkerContext(), contextProto, client); context.init(); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java index f94bd78d23..56ed8df4f2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java @@ -26,6 +26,7 @@ import org.apache.tajo.rule.EvaluationResult.EvaluationResultCode; import org.apache.tajo.service.ServiceTracker; import org.apache.tajo.service.ServiceTrackerFactory; +import org.apache.tajo.util.RpcConnectionParamUtil; import org.apache.tajo.worker.TajoWorker; /** @@ -40,7 +41,8 @@ private void checkTajoMasterConnectivity(TajoConf tajoConf) throws Exception { RpcClientManager manager = RpcClientManager.getInstance(); ServiceTracker serviceTracker = ServiceTrackerFactory.get(tajoConf); - NettyClientBase masterClient = manager.getClient(serviceTracker.getUmbilicalAddress(), QueryCoordinatorProtocol.class, true); + NettyClientBase masterClient = manager.getClient(serviceTracker.getUmbilicalAddress(), + QueryCoordinatorProtocol.class, true, RpcConnectionParamUtil.get(tajoConf)); masterClient.getStub(); } diff --git a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java index a56062036e..467d808462 100644 --- a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java +++ b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java @@ -20,19 +20,28 @@ import java.util.concurrent.TimeUnit; +/** + * Constants for RPC + */ public class RpcConstants { public static final String PING_PACKET = "TAJO"; - public static final String RPC_CLIENT_RETRY_MAX = "tajo.rpc.client.retry.max"; - public static final String RPC_CLIENT_TIMEOUT_SECS = "tajo.rpc.client.timeout-secs"; + public static final int DEFAULT_PAUSE = 1000; // 1 sec + public static final int FUTURE_TIMEOUT_SECONDS_DEFAULT = 10; - public static final int DEFAULT_RPC_RETRIES = 3; + /** How many times the connect will retry */ + public static final String RPC_RETRY_NUM = "tajo.rpc.client.retry-num"; + public static final int RPC_RETRY_NUM_DEFAULT = 3; - public static final int DEFAULT_PAUSE = 1000; // 1 sec - public static final int DEFAULT_FUTURE_TIMEOUT_SECONDS = 10; + /** Client connection timeout (milliseconds) */ + public static final String CLIENT_CONNECTION_TIMEOUT = "tajo.rpc.client.connection-timeout-ms"; + /** Default client connection timeout 15 seconds */ + public final static long CLIENT_CONNECTION_TIMEOUT_DEFAULT = TimeUnit.SECONDS.toMicros(15); - /** default connection timeout 15 seconds */ - public final static long DEFAULT_CONNECTION_TIMEOUT = TimeUnit.SECONDS.toMicros(15); - /** default socket timeout - 60 seconds */ - public final static long DEFAULT_SOCKET_TIMEOUT = TimeUnit.SECONDS.toNanos(60); + /** + * Socket timeout (milliseconds). Client connection will check if the connectivity is available after this time. + */ + public static final String CLIENT_SOCKET_TIMEOUT = "tajo.rpc.client.socket-timeout-ms"; + /** Default socket timeout - 60 seconds */ + public final static long CLIENT_SOCKET_TIMEOUT_DEFAULT = TimeUnit.SECONDS.toNanos(180); } diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java index 46112b480b..9f90368e1c 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java @@ -23,24 +23,22 @@ import com.google.protobuf.*; import io.netty.channel.ChannelHandler; import io.netty.channel.EventLoopGroup; -import org.apache.tajo.rpc.RpcClientManager.RpcConnectionKey; import org.apache.tajo.rpc.RpcProtos.RpcResponse; import java.lang.reflect.Method; +import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import static org.apache.tajo.rpc.RpcConstants.CLIENT_SOCKET_TIMEOUT; +import static org.apache.tajo.rpc.RpcConstants.CLIENT_SOCKET_TIMEOUT_DEFAULT; + public class AsyncRpcClient extends NettyClientBase { private final Method stubMethod; private final ProxyRpcChannel rpcChannel; private final NettyChannelInboundHandler handler; - @VisibleForTesting - AsyncRpcClient(RpcConnectionKey rpcConnectionKey, int retries) - throws ClassNotFoundException, NoSuchMethodException { - this(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, retries, RpcConstants.DEFAULT_CONNECTION_TIMEOUT, false, 0); - } /** * Intentionally make this method package-private, avoiding user directly @@ -48,31 +46,26 @@ public class AsyncRpcClient extends NettyClientBaseuseIdleTimeout is true, - * this connection will check if the connectivity is available after - * idleTimeout * @throws ClassNotFoundException * @throws NoSuchMethodException */ AsyncRpcClient(EventLoopGroup eventLoopGroup, RpcConnectionKey rpcConnectionKey, - int retryNum, - long connTimeout, - boolean useIdleTimeout, - long idleTimeout) + Properties connectionParameters) throws ClassNotFoundException, NoSuchMethodException { - super(rpcConnectionKey, retryNum, connTimeout); + super(rpcConnectionKey, connectionParameters); this.stubMethod = getServiceClass().getMethod("newStub", RpcChannel.class); this.rpcChannel = new ProxyRpcChannel(); this.handler = new ClientChannelInboundHandler(); + long socketTimeoutMills = Long.parseLong( + connectionParameters.getProperty(CLIENT_SOCKET_TIMEOUT, String.valueOf(CLIENT_SOCKET_TIMEOUT_DEFAULT))); + init(new ProtoClientChannelInitializer(handler, RpcResponse.getDefaultInstance(), - useIdleTimeout, TimeUnit.MILLISECONDS.toNanos(idleTimeout)), eventLoopGroup); + TimeUnit.MILLISECONDS.toNanos(socketTimeoutMills)),eventLoopGroup); } @Override diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java index b33fea2232..c60a33beb6 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java @@ -18,63 +18,54 @@ package org.apache.tajo.rpc; -import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.*; import com.google.protobuf.Descriptors.MethodDescriptor; import io.netty.channel.ChannelHandler; import io.netty.channel.EventLoopGroup; -import org.apache.tajo.rpc.RpcClientManager.RpcConnectionKey; import org.apache.tajo.rpc.RpcProtos.RpcResponse; import java.lang.reflect.Method; import java.net.InetSocketAddress; +import java.util.Properties; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; +import static org.apache.tajo.rpc.RpcConstants.CLIENT_SOCKET_TIMEOUT; +import static org.apache.tajo.rpc.RpcConstants.CLIENT_SOCKET_TIMEOUT_DEFAULT; + public class BlockingRpcClient extends NettyClientBase { private final Method stubMethod; private final ProxyRpcChannel rpcChannel; private final NettyChannelInboundHandler handler; - @VisibleForTesting - BlockingRpcClient(RpcConnectionKey rpcConnectionKey, int connRetryNum) - throws NoSuchMethodException, ClassNotFoundException { - this(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connRetryNum, RpcConstants.DEFAULT_CONNECTION_TIMEOUT, false, 0); - } - /** * Intentionally make this method package-private, avoiding user directly * new an instance through this constructor. * - * @param rpcConnectionKey RpcConnectionKey - * @param eventLoopGroup Thread pool of netty's - * - * @param retryNumToConnect Retry number to connect - * @param connTimeout Connection Timeout (milliseconds) - * @param useIdleTimeout Enable idle connection check - * @param idleTimeout Socket Idle timeout (milliseconds). If useIdleTimeout is true, - * this connection will check if the connectivity is available after - * idleTimeout + * @param rpcConnectionKey RpcConnectionKey + * @param eventLoopGroup Thread pool of netty's + * @param connectionParameters Connection parameters (see RpcConstants) * * @throws ClassNotFoundException * @throws NoSuchMethodException + * @see RpcConstants */ - BlockingRpcClient(EventLoopGroup eventLoopGroup, - RpcConnectionKey rpcConnectionKey, - int retryNumToConnect, - long connTimeout, - boolean useIdleTimeout, - long idleTimeout) + public BlockingRpcClient(EventLoopGroup eventLoopGroup, + RpcConnectionKey rpcConnectionKey, + Properties connectionParameters) throws ClassNotFoundException, NoSuchMethodException { - super(rpcConnectionKey, retryNumToConnect, connTimeout); + super(rpcConnectionKey, connectionParameters); this.stubMethod = getServiceClass().getMethod("newBlockingStub", BlockingRpcChannel.class); this.rpcChannel = new ProxyRpcChannel(); this.handler = new ClientChannelInboundHandler(); + long socketTimeoutMills = Long.parseLong( + connectionParameters.getProperty(CLIENT_SOCKET_TIMEOUT, String.valueOf(CLIENT_SOCKET_TIMEOUT_DEFAULT))); + init(new ProtoClientChannelInitializer(handler, RpcResponse.getDefaultInstance(), - useIdleTimeout, TimeUnit.MILLISECONDS.toNanos(idleTimeout)),eventLoopGroup); + TimeUnit.MILLISECONDS.toNanos(socketTimeoutMills)),eventLoopGroup); } @Override diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java index 4ee6a7113d..8dbb6d4f28 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java @@ -32,7 +32,6 @@ import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.tajo.rpc.RpcClientManager.RpcConnectionKey; import org.apache.tajo.rpc.RpcProtos.RpcResponse; import java.io.Closeable; @@ -42,10 +41,13 @@ import java.net.SocketAddress; import java.nio.channels.UnresolvedAddressException; import java.util.Collection; +import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; +import static org.apache.tajo.rpc.RpcConstants.*; + public abstract class NettyClientBase implements ProtoDeclaration, Closeable { public final static Log LOG = LogFactory.getLog(NettyClientBase.class); @@ -66,18 +68,21 @@ public abstract class NettyClientBase implements ProtoDeclaration, Closeable * Constructor of NettyClientBase * * @param rpcConnectionKey RpcConnectionKey - * @param maxRetryNum How many number it will retry - * @param connTimeout Connection Timeout (milliseconds) + * @param connectionParameters Connection Parameters * * @throws ClassNotFoundException * @throws NoSuchMethodException */ - public NettyClientBase(RpcConnectionKey rpcConnectionKey, - int maxRetryNum, - long connTimeout) throws ClassNotFoundException, NoSuchMethodException { + public NettyClientBase(RpcConnectionKey rpcConnectionKey, Properties connectionParameters) + throws ClassNotFoundException, NoSuchMethodException { this.key = rpcConnectionKey; - this.maxRetryNum = maxRetryNum; - this.connTimeoutMillis = connTimeout; + + this.maxRetryNum = Integer.parseInt( + connectionParameters.getProperty(RPC_RETRY_NUM, String.valueOf(RPC_RETRY_NUM_DEFAULT))); + + this.connTimeoutMillis = Integer.parseInt( + connectionParameters.getProperty(CLIENT_CONNECTION_TIMEOUT, String.valueOf(CLIENT_CONNECTION_TIMEOUT_DEFAULT))); + // Netty only takes integer value range and this is to avoid integer overflow. Preconditions.checkArgument(this.connTimeoutMillis > Integer.MAX_VALUE, "Too long connection timeout"); } @@ -96,7 +101,7 @@ protected void init(ChannelInitializer initializer, EventLoopGroup even .option(ChannelOption.TCP_NODELAY, true); } - public RpcClientManager.RpcConnectionKey getKey() { + public RpcConnectionKey getKey() { return key; } diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java index d5a19f5271..63e3c218a5 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java @@ -34,25 +34,28 @@ class ProtoClientChannelInitializer extends ChannelInitializer { private final MessageLite defaultInstance; private final ChannelHandler handler; - private final boolean useIdleTimeout; - private final long timeoutTimeNanos; + private final long socketTimeout; - public ProtoClientChannelInitializer(ChannelHandler handler, MessageLite defaultInstance, - boolean useIdleTimeout, long idleTimeoutNanos) { + /** + * Channel Pipe Initializer + * + * @param handler Channel Handler + * @param defaultInstance Default Rpc Proto instance + * @param socketTimeout Socket timeout (milliseconds) + */ + public ProtoClientChannelInitializer(ChannelHandler handler, MessageLite defaultInstance, long socketTimeout) { this.handler = handler; this.defaultInstance = defaultInstance; - this.timeoutTimeNanos = idleTimeoutNanos; - this.useIdleTimeout = useIdleTimeout; + this.socketTimeout = socketTimeout; } @Override protected void initChannel(Channel channel) throws Exception { ChannelPipeline pipeline = channel.pipeline(); pipeline.addLast("idleStateHandler", - new IdleStateHandler(timeoutTimeNanos, timeoutTimeNanos / 2, 0, TimeUnit.NANOSECONDS)); - - if (useIdleTimeout) pipeline.addLast("MonitorClientHandler", new MonitorClientHandler()); + new IdleStateHandler(socketTimeout, socketTimeout / 2, 0, TimeUnit.MILLISECONDS)); + pipeline.addLast("MonitorClientHandler", new MonitorClientHandler()); pipeline.addLast("frameDecoder", new ProtobufVarint32FrameDecoder()); pipeline.addLast("protobufDecoder", new ProtobufDecoder(defaultInstance)); pipeline.addLast("frameEncoder", new ProtobufVarint32LengthFieldPrepender()); diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java index 76b7334235..87af5c2468 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java @@ -33,16 +33,12 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.TimeUnit; +import java.util.Properties; @ThreadSafe public class RpcClientManager { private static final Log LOG = LogFactory.getLog(RpcClientManager.class); - private volatile long connTimeout = RpcConstants.DEFAULT_CONNECTION_TIMEOUT; - private volatile long socketTimeout = RpcConstants.DEFAULT_SOCKET_TIMEOUT; - private volatile int retries = RpcConstants.DEFAULT_RPC_RETRIES; - /* entries will be removed by ConnectionCloseFutureListener */ private static final Map clients = Collections.synchronizedMap(new HashMap()); @@ -62,28 +58,23 @@ public static RpcClientManager getInstance() { } private T makeClient(RpcConnectionKey rpcConnectionKey, - int retryNum, - long connTimeout, - boolean useSocketTimeout, - long socketTimeout) + Properties connectionParameters) throws NoSuchMethodException, ConnectException, ClassNotFoundException { - return makeClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, retryNum, connTimeout, useSocketTimeout, socketTimeout); + + + return makeClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connectionParameters); } private T makeClient(EventLoopGroup eventLoopGroup, RpcConnectionKey rpcConnectionKey, - int retryNum, - long connTimeout, - boolean useSocketTimeout, long socketTimeout) + Properties connectionParameters) throws NoSuchMethodException, ClassNotFoundException, ConnectException { NettyClientBase client; if (rpcConnectionKey.asyncMode) { - client = new AsyncRpcClient(eventLoopGroup, rpcConnectionKey, retryNum, connTimeout, useSocketTimeout, - socketTimeout); + client = new AsyncRpcClient(eventLoopGroup, rpcConnectionKey, connectionParameters); } else { - client = new BlockingRpcClient(eventLoopGroup, rpcConnectionKey, retryNum, connTimeout, useSocketTimeout, - socketTimeout); + client = new BlockingRpcClient(eventLoopGroup, rpcConnectionKey, connectionParameters); } return (T) client; } @@ -93,7 +84,9 @@ private T makeClient(EventLoopGroup eventLoopGroup, * This client will be shared per protocol and address. Client is removed in shared map when a client is closed */ public T getClient(InetSocketAddress addr, - Class protocolClass, boolean asyncMode) + Class protocolClass, + boolean asyncMode, + Properties connectionParameters) throws NoSuchMethodException, ClassNotFoundException, ConnectException { RpcConnectionKey key = new RpcConnectionKey(addr, protocolClass, asyncMode); @@ -101,7 +94,7 @@ public T getClient(InetSocketAddress addr, synchronized (clients) { client = clients.get(key); if (client == null) { - clients.put(key, client = makeClient(key, retries, connTimeout, true, getSocketTimeout())); + clients.put(key, client = makeClient(key, connectionParameters)); } } @@ -135,23 +128,10 @@ public void channelUnregistered(ChannelHandlerContext ctx) { public T newClient(InetSocketAddress addr, Class protocolClass, boolean asyncMode, - int retryNum, - long connTimeout, - boolean useIdleTimeout, - long idleTimeout) - throws NoSuchMethodException, ClassNotFoundException, ConnectException { - - return newClient(new RpcConnectionKey(addr, protocolClass, asyncMode), retryNum, connTimeout, useIdleTimeout, - idleTimeout); - } - - public synchronized T newClient(InetSocketAddress addr, - Class protocolClass, - boolean asyncMode) + Properties connectionParameters) throws NoSuchMethodException, ClassNotFoundException, ConnectException { - return newClient(new RpcConnectionKey(addr, protocolClass, asyncMode), - retries, 0, true, TimeUnit.SECONDS.toMillis(getSocketTimeout())); + return newClient(new RpcConnectionKey(addr, protocolClass, asyncMode), connectionParameters); } /** @@ -164,14 +144,23 @@ public synchronized T newClient(InetSocketAddress ad * @throws ConnectException */ public synchronized T newClient(RpcConnectionKey key, - int retryNum, - long connTimeout, - boolean useIdleTimeout, - long idleTimeout) + Properties connectionParameters) + + throws NoSuchMethodException, ClassNotFoundException, ConnectException { + T client = makeClient(key, connectionParameters); + client.connect(); + assert client.isConnected(); + return client; + } + + public synchronized T newBlockingClient(InetSocketAddress addr, + Class protocolClass, + EventLoopGroup eventLoopGroup, + Properties connectionParameters) throws NoSuchMethodException, ClassNotFoundException, ConnectException { - T client = makeClient(key, retryNum, connTimeout, useIdleTimeout, idleTimeout); + T client = makeClient(eventLoopGroup, new RpcConnectionKey(addr, protocolClass, false), connectionParameters); client.connect(); assert client.isConnected(); return client; @@ -222,61 +211,10 @@ public static void cleanup(NettyClientBase... clients) { } } - public long getSocketTimeout() { - return socketTimeout; - } - - public void setSocketTimeout(int socketTimeout) { - this.socketTimeout = socketTimeout; - } - - public int getRetries() { - return retries; - } - - public void setRetries(int retries) { - this.retries = retries; - } - - static class RpcConnectionKey { - final InetSocketAddress addr; - final Class protocolClass; - final boolean asyncMode; - - final String description; - - public RpcConnectionKey(InetSocketAddress addr, - Class protocolClass, boolean asyncMode) { - this.addr = addr; - this.protocolClass = protocolClass; - this.asyncMode = asyncMode; - this.description = "[" + protocolClass + "] " + addr + "," + asyncMode; - } - - @Override - public String toString() { - return description; - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof RpcConnectionKey)) { - return false; - } - - return toString().equals(obj.toString()); - } - - @Override - public int hashCode() { - return description.hashCode(); - } - } - static class ClientCloseFutureListener implements GenericFutureListener { - private RpcClientManager.RpcConnectionKey key; + private RpcConnectionKey key; - public ClientCloseFutureListener(RpcClientManager.RpcConnectionKey key) { + public ClientCloseFutureListener(RpcConnectionKey key) { this.key = key; } diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcConnectionKey.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcConnectionKey.java new file mode 100644 index 0000000000..2804010853 --- /dev/null +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcConnectionKey.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.rpc; + +import java.net.InetSocketAddress; + +public class RpcConnectionKey { + final InetSocketAddress addr; + final Class protocolClass; + final boolean asyncMode; + + final String description; + + public RpcConnectionKey(InetSocketAddress addr, + Class protocolClass, boolean asyncMode) { + this.addr = addr; + this.protocolClass = protocolClass; + this.asyncMode = asyncMode; + this.description = "[" + protocolClass + "] " + addr + "," + asyncMode; + } + + @Override + public String toString() { + return description; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof RpcConnectionKey)) { + return false; + } + + return toString().equals(obj.toString()); + } + + @Override + public int hashCode() { + return description.hashCode(); + } +} diff --git a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java index e150dd7fdc..cec58917f5 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java @@ -42,6 +42,7 @@ import java.net.ConnectException; import java.net.InetSocketAddress; import java.net.ServerSocket; +import java.util.Properties; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -61,7 +62,7 @@ public class TestAsyncRpc { Interface stub; DummyProtocolAsyncImpl service; int retries; - RpcClientManager.RpcConnectionKey rpcConnectionKey; + RpcConnectionKey rpcConnectionKey; RpcClientManager manager = RpcClientManager.getInstance(); @Retention(RetentionPolicy.RUNTIME) @@ -129,10 +130,14 @@ public void setUpRpcServer() throws Exception { public void setUpRpcClient() throws Exception { retries = 1; - rpcConnectionKey = new RpcClientManager.RpcConnectionKey( - RpcUtils.getConnectAddress(server.getListenAddress()), - DummyProtocol.class, true); - client = manager.newClient(rpcConnectionKey, retries, 1000, true, 10); + rpcConnectionKey = new RpcConnectionKey( + RpcUtils.getConnectAddress(server.getListenAddress()), DummyProtocol.class, true); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(TimeUnit.SECONDS.toMillis(10))); + + client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); stub = client.getStub(); } @@ -347,9 +352,13 @@ public void run() { }); serverThread.start(); - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(address, DummyProtocol.class, true); - AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, false, 0); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(address, DummyProtocol.class, true); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); + + AsyncRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); Interface stub = client.getStub(); @@ -376,9 +385,13 @@ public void testClientRetryFailureOnStartup() throws Exception { .setMessage(MESSAGE).build(); CallFuture future = new CallFuture<>(); - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(address, DummyProtocol.class, true); - AsyncRpcClient client = new AsyncRpcClient(rpcConnectionKey, retries); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(address, DummyProtocol.class, true); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); + + AsyncRpcClient client = new AsyncRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams); try { client.connect(); fail(); @@ -408,9 +421,13 @@ public void testUnresolvedAddress() throws Exception { boolean expected = false; AsyncRpcClient client = null; try { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(address, DummyProtocol.class, true); - client = new AsyncRpcClient(rpcConnectionKey, retries); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(address, DummyProtocol.class, true); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); + + client = new AsyncRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams); client.connect(); fail(); } catch (ConnectException e) { @@ -428,10 +445,14 @@ public void testUnresolvedAddress() throws Exception { @SetupRpcConnection(setupRpcClient = false) public void testUnresolvedAddress2() throws Exception { String hostAndPort = RpcUtils.normalizeInetSocketAddress(server.getListenAddress()); - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey( + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey( RpcUtils.createUnresolved(hostAndPort), DummyProtocol.class, true); - AsyncRpcClient client = new AsyncRpcClient(rpcConnectionKey, retries); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); + + AsyncRpcClient client = new AsyncRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams); client.connect(); try { assertTrue(client.isConnected()); @@ -452,9 +473,12 @@ public void testUnresolvedAddress2() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testStubRecovery() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); - AsyncRpcClient client = manager.newClient(rpcConnectionKey, 2, 1000, false, 0); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(2)); + AsyncRpcClient client = manager.newClient(rpcConnectionKey, connParams); EchoMessage echoMessage = EchoMessage.newBuilder() .setMessage(MESSAGE).build(); @@ -483,10 +507,15 @@ public void testStubRecovery() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testIdleTimeout() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); - //500 millis idle timeout - AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, false, 500); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); + + // 500 millis idle timeout + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); + + AsyncRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); Thread.sleep(600); //timeout @@ -503,11 +532,15 @@ public void testIdleTimeout() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testPingOnIdle() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); - //500 millis request timeout - AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, true, 500); + // 500 millis idle timeout + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); + + AsyncRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); Thread.sleep(600); @@ -521,10 +554,15 @@ public void testPingOnIdle() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testIdleTimeoutWithActiveRequest() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); - //500 millis idle timeout - AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, false, 500); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); + + // 500 millis idle timeout + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); + + AsyncRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); Interface stub = client.getStub(); @@ -546,11 +584,15 @@ public void testIdleTimeoutWithActiveRequest() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testRequestTimeoutOnBusy() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); + + // 500 millis idle timeout + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); - //500 millis request timeout - AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, true, 500); + AsyncRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); Interface stub = client.getStub(); diff --git a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java index a70353475c..371864935e 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java @@ -39,6 +39,7 @@ import java.net.ConnectException; import java.net.InetSocketAddress; import java.net.ServerSocket; +import java.util.Properties; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -119,11 +120,15 @@ public void setUpRpcServer() throws Exception { public void setUpRpcClient() throws Exception { retries = 1; - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey( + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, "1"); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(TimeUnit.SECONDS.toMillis(10))); + + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey( RpcUtils.getConnectAddress(server.getListenAddress()), DummyProtocol.class, false); - client = manager.newClient(rpcConnectionKey, retries, 1000, true, 10); + client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); stub = client.getStub(); } @@ -317,10 +322,13 @@ public void run() { }); serverThread.start(); - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(address, DummyProtocol.class, false); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(address, DummyProtocol.class, false); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, retries + ""); - BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, false, 0); + BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); BlockingInterface stub = client.getStub(); @@ -341,9 +349,14 @@ public void testClientRetryFailureOnStartup() throws Exception { EchoMessage message = EchoMessage.newBuilder() .setMessage(MESSAGE).build(); - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(address, DummyProtocol.class, false); - BlockingRpcClient client = new BlockingRpcClient(rpcConnectionKey, retries); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(address, DummyProtocol.class, false); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, retries + ""); + + BlockingRpcClient client = new BlockingRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, + connParams); try { client.connect(); @@ -369,9 +382,13 @@ public void testUnresolvedAddress() throws Exception { boolean expected = false; BlockingRpcClient client = null; try { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(address, DummyProtocol.class, true); - client = new BlockingRpcClient(rpcConnectionKey, retries); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(address, DummyProtocol.class, true); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, retries + ""); + + client = new BlockingRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams); client.connect(); fail(); } catch (ConnectException e) { @@ -387,11 +404,17 @@ public void testUnresolvedAddress() throws Exception { @Test(timeout = 120000) @SetupRpcConnection(setupRpcClient = false) public void testUnresolvedAddress2() throws Exception { + String hostAndPort = RpcUtils.normalizeInetSocketAddress(server.getListenAddress()); - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey( + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey( RpcUtils.createUnresolved(hostAndPort), DummyProtocol.class, false); - BlockingRpcClient client = new BlockingRpcClient(rpcConnectionKey, retries); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, retries + ""); + + BlockingRpcClient client = + new BlockingRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams); client.connect(); assertTrue(client.isConnected()); @@ -409,9 +432,11 @@ public void testUnresolvedAddress2() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testStubRecovery() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); - BlockingRpcClient client = manager.newClient(rpcConnectionKey, 1, 1000, false, 0); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(1)); + BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams); EchoMessage echoMessage = EchoMessage.newBuilder() .setMessage(MESSAGE).build(); @@ -439,10 +464,15 @@ public void testStubRecovery() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testIdleTimeout() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); - //500 millis idle timeout - BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, false, 500); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); + + // 500 millis socket timeout + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, retries + ""); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); + + BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); Thread.sleep(600); //timeout @@ -459,11 +489,15 @@ public void testIdleTimeout() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testPingOnIdle() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); + + // 500 millis socket timeout + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, retries + ""); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); - //500 millis request timeout - BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, true, 500); + BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); Thread.sleep(600); @@ -477,10 +511,15 @@ public void testPingOnIdle() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testIdleTimeoutWithActiveRequest() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); - //500 millis idle timeout - BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, false, 500); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); + + // 500 millis socket timeout + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, retries + ""); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); + + BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); BlockingInterface stub = client.getStub(); @@ -499,11 +538,15 @@ public void testIdleTimeoutWithActiveRequest() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testRequestTimeoutOnBusy() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); + + // 500 millis socket timeout + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.RPC_RETRY_NUM, retries + ""); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); - //500 millis request timeout - BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, 1000, true, 500); + BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); BlockingInterface stub = client.getStub(); diff --git a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestRpcClientManager.java b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestRpcClientManager.java index 0055f0ff92..160c6a3074 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestRpcClientManager.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestRpcClientManager.java @@ -25,10 +25,10 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.List; +import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; import static org.junit.Assert.*; @@ -54,7 +54,7 @@ public void testRaceCondition() throws Exception { public void run() { NettyClientBase client = null; try { - client = manager.getClient(address, DummyProtocol.class, false); + client = manager.getClient(address, DummyProtocol.class, false, new Properties()); } catch (Throwable e) { fail(e.getMessage()); } @@ -68,7 +68,7 @@ public void run() { future.get(); } - NettyClientBase clientBase = manager.getClient(address, DummyProtocol.class, false); + NettyClientBase clientBase = manager.getClient(address, DummyProtocol.class, false, new Properties()); RpcClientManager.cleanup(clientBase); } finally { server.shutdown(); @@ -87,11 +87,11 @@ public void testClientCloseEvent() throws Exception { try { - NettyClientBase client = manager.getClient(server.getListenAddress(), DummyProtocol.class, true); + NettyClientBase client = manager.getClient(server.getListenAddress(), DummyProtocol.class, true, new Properties()); assertTrue(client.isConnected()); assertTrue(client.getChannel().isWritable()); - RpcClientManager.RpcConnectionKey key = client.getKey(); + RpcConnectionKey key = client.getKey(); assertTrue(RpcClientManager.contains(key)); client.close(); @@ -113,10 +113,10 @@ public void testClientCloseEventWithReconnect() throws Exception { try { - NettyClientBase client = manager.getClient(server.getListenAddress(), DummyProtocol.class, true); + NettyClientBase client = manager.getClient(server.getListenAddress(), DummyProtocol.class, true, new Properties()); assertTrue(client.isConnected()); - RpcClientManager.RpcConnectionKey key = client.getKey(); + RpcConnectionKey key = client.getKey(); assertTrue(RpcClientManager.contains(key)); client.close(); @@ -144,17 +144,17 @@ public void testUnManagedClient() throws Exception { NettyServerBase server = new AsyncRpcServer(DummyProtocol.class, service, new InetSocketAddress("127.0.0.1", 0), 3); server.start(); - RpcClientManager.RpcConnectionKey key = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); + RpcConnectionKey key = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); RpcClientManager.close(); RpcClientManager manager = RpcClientManager.getInstance(); try { - NettyClientBase client1 = manager.newClient(key, 0, 1000, false, 0); + NettyClientBase client1 = manager.newClient(key, new Properties()); assertTrue(client1.isConnected()); assertFalse(RpcClientManager.contains(key)); - NettyClientBase client2 = manager.newClient(key, 0, 1000, false, 0); + NettyClientBase client2 = manager.newClient(key, new Properties()); assertTrue(client2.isConnected()); assertFalse(RpcClientManager.contains(key)); From 53f61ed998398dfbc110f14ee69772601d325e8d Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Wed, 16 Sep 2015 01:06:20 -0700 Subject: [PATCH 04/24] Add unit tests. --- .../tajo/catalog/AbstractCatalogClient.java | 30 +++++++++----- .../java/org/apache/tajo/conf/TajoConf.java | 4 +- .../tajo/util/TestRpcConnectionParamUtil.java | 40 +++++++++++++++++++ 3 files changed, 61 insertions(+), 13 deletions(-) create mode 100644 tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcConnectionParamUtil.java diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java index f74de827fd..34942b1269 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java @@ -80,13 +80,15 @@ public final void createTablespace(final String tablespaceName, final String tab } @Override - public final void dropTablespace(final String tablespaceName) throws UndefinedTablespaceException { + public final void dropTablespace(final String tablespaceName) + throws UndefinedTablespaceException, InsufficientPrivilegeException { try { final BlockingInterface stub = getStub(); final ReturnState state = stub.dropTablespace(null, ProtoUtil.convertString(tablespaceName)); throwsIfThisError(state, UndefinedTablespaceException.class); + throwsIfThisError(state, InsufficientPrivilegeException.class); ensureOk(state); } catch (ServiceException e) { @@ -196,13 +198,15 @@ public final void createDatabase(final String databaseName, @Nullable final Stri } @Override - public final void dropDatabase(final String databaseName) throws UndefinedDatabaseException { + public final void dropDatabase(final String databaseName) + throws UndefinedDatabaseException, InsufficientPrivilegeException { try { final BlockingInterface stub = getStub(); final ReturnState state = stub.dropDatabase(null, ProtoUtil.convertString(databaseName)); throwsIfThisError(state, UndefinedDatabaseException.class); + throwsIfThisError(state, InsufficientPrivilegeException.class); ensureOk(state); } catch (ServiceException e) { @@ -598,18 +602,18 @@ public final boolean existsTable(final String tableName) { } @Override - public final boolean createIndex(final IndexDesc index) { + public final void createIndex(final IndexDesc index) + throws DuplicateIndexException, UndefinedDatabaseException, UndefinedTableException { try { final BlockingInterface stub = getStub(); final ReturnState state = stub.createIndex(null, index.getProto()); - if (isSuccess(state)) { - return true; - } else { - // TODO - return false; - } + + throwsIfThisError(state, DuplicateIndexException.class); + throwsIfThisError(state, UndefinedTableException.class); + throwsIfThisError(state, UndefinedDatabaseException.class); + ensureOk(state); } catch (ServiceException e) { throw new RuntimeException(e); @@ -745,7 +749,8 @@ public final Collection getAllIndexesByTable(final String databaseNam } @Override - public boolean dropIndex(final String dbName, final String indexName) { + public void dropIndex(final String dbName, final String indexName) + throws UndefinedIndexException, UndefinedDatabaseException { try { final IndexNameProto request = IndexNameProto.newBuilder() .setDatabaseName(dbName) @@ -753,8 +758,11 @@ public boolean dropIndex(final String dbName, final String indexName) { .build(); final BlockingInterface stub = getStub(); + final ReturnState state = stub.dropIndex(null, request); - return isSuccess(stub.dropIndex(null, request)); + throwsIfThisError(state, UndefinedIndexException.class); + throwsIfThisError(state, UndefinedDatabaseException.class); + ensureOk(state); } catch (ServiceException e) { throw new RuntimeException(e); diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 7e4c591ae6..3ec2bbc496 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -230,8 +230,8 @@ public static enum ConfVars implements ConfigKey { INTERNAL_RPC_CLIENT_WORKER_THREAD_NUM("tajo.internal.rpc.client.worker-thread-num", Runtime.getRuntime().availableProcessors() * 2), RPC_CLIENT_RETRY_NUM("tajo.rpc.client.retry-num", 3, Validators.min("1")), - RPC_CLIENT_CONNECTION_TIMEOUT("tajo.rpc.client.connection-timeout-ms", 15 * 1000, Validators.min("1")), - RPC_CLIENT_SOCKET_TIMEOUT("tajo.rpc.client.socket-timeout-ms", 180 * 1000, Validators.min("1")), + RPC_CLIENT_CONNECTION_TIMEOUT("tajo.rpc.client.connection-timeout-ms", (long)15 * 1000, Validators.min("1")), + RPC_CLIENT_SOCKET_TIMEOUT("tajo.rpc.client.socket-timeout-ms", (long)180 * 1000, Validators.min("1")), // Internal RPC Server MASTER_RPC_SERVER_WORKER_THREAD_NUM("tajo.master.rpc.server.worker-thread-num", diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcConnectionParamUtil.java b/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcConnectionParamUtil.java new file mode 100644 index 0000000000..27e47b4370 --- /dev/null +++ b/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcConnectionParamUtil.java @@ -0,0 +1,40 @@ +package org.apache.tajo.util; + +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; +import org.apache.tajo.rpc.RpcConstants; +import org.junit.Test; + +import java.util.Properties; + +import static org.apache.tajo.rpc.RpcConstants.CLIENT_CONNECTION_TIMEOUT; +import static org.apache.tajo.rpc.RpcConstants.RPC_RETRY_NUM; +import static org.junit.Assert.*; + +public class TestRpcConnectionParamUtil { + + @Test + public void testGetDefaults() throws Exception { + TajoConf conf = new TajoConf(); + Properties defaultParams = RpcConnectionParamUtil.get(conf); + assertEquals( + ConfVars.RPC_CLIENT_RETRY_NUM.defaultVal, defaultParams.getProperty(RPC_RETRY_NUM)); + assertEquals( + ConfVars.RPC_CLIENT_CONNECTION_TIMEOUT.defaultVal, defaultParams.getProperty(CLIENT_CONNECTION_TIMEOUT)); + assertEquals( + ConfVars.RPC_CLIENT_SOCKET_TIMEOUT.defaultVal, defaultParams.getProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT)); + } + + @Test + public void testGet() throws Exception { + TajoConf conf = new TajoConf(); + conf.setIntVar(ConfVars.RPC_CLIENT_RETRY_NUM, 100); + conf.setLongVar(ConfVars.RPC_CLIENT_CONNECTION_TIMEOUT, (long)(10 * 1000)); + conf.setLongVar(ConfVars.RPC_CLIENT_SOCKET_TIMEOUT, (long)60 * 1000); + + Properties defaultParams = RpcConnectionParamUtil.get(conf); + assertEquals("100", defaultParams.getProperty(RPC_RETRY_NUM)); + assertEquals(10 * 1000 + "", defaultParams.getProperty(CLIENT_CONNECTION_TIMEOUT)); + assertEquals(60 * 1000 + "", defaultParams.getProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT)); + } +} \ No newline at end of file From 03bba84b3a2375137fbc936e6e02659f46613fe4 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Wed, 16 Sep 2015 01:22:18 -0700 Subject: [PATCH 05/24] Removed unnecessary routine. --- .../java/org/apache/tajo/rpc/AsyncRpcClient.java | 3 +-- .../org/apache/tajo/rpc/NettyClientBase.java | 16 +++------------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java index 9f90368e1c..ddc7296e99 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java @@ -18,7 +18,6 @@ package org.apache.tajo.rpc; -import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.Descriptors.MethodDescriptor; import com.google.protobuf.*; import io.netty.channel.ChannelHandler; @@ -61,7 +60,7 @@ public class AsyncRpcClient extends NettyClientBase implements ProtoDeclaration, Closeable private Bootstrap bootstrap; private volatile ChannelFuture channelFuture; - private boolean idleTimeoutEnabled; /** * Constructor of NettyClientBase * * @param rpcConnectionKey RpcConnectionKey - * @param connectionParameters Connection Parameters + * @param connectionParameters Connection Parameters (see RpcConstants) * * @throws ClassNotFoundException * @throws NoSuchMethodException + * @see RpcConstants */ public NettyClientBase(RpcConnectionKey rpcConnectionKey, Properties connectionParameters) throws ClassNotFoundException, NoSuchMethodException { @@ -310,9 +310,6 @@ protected void registerCallback(int seqId, T callback) { @Override public void channelRegistered(ChannelHandlerContext ctx) throws Exception { MonitorClientHandler handler = ctx.pipeline().get(MonitorClientHandler.class); - if (handler != null) { - idleTimeoutEnabled = true; - } for (ChannelEventListener listener : getSubscribers()) { listener.channelRegistered(ctx); @@ -409,14 +406,7 @@ private void sendException(RecoverableException e) { @Override public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { - if (!idleTimeoutEnabled && evt instanceof IdleStateEvent) { - IdleStateEvent e = (IdleStateEvent) evt; - /* If all requests is done and event is triggered, idle channel close. */ - if (e.state() == IdleState.READER_IDLE && requests.isEmpty()) { - ctx.close(); - LOG.info("Idle connection closed successfully :" + ctx.channel()); - } - } else if (evt instanceof MonitorStateEvent) { + if (evt instanceof MonitorStateEvent) { MonitorStateEvent e = (MonitorStateEvent) evt; if (e.state() == MonitorStateEvent.MonitorState.PING_EXPIRED) { exceptionCaught(ctx, new ServiceException("Server has not respond: " + ctx.channel())); From 2bb9dca2290c588f30cf13b7eeb96041795e587d Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Wed, 16 Sep 2015 01:34:03 -0700 Subject: [PATCH 06/24] Fixed parameter units. --- .../src/main/java/org/apache/tajo/rpc/RpcConstants.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java index 467d808462..7652cb3054 100644 --- a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java +++ b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java @@ -36,12 +36,12 @@ public class RpcConstants { /** Client connection timeout (milliseconds) */ public static final String CLIENT_CONNECTION_TIMEOUT = "tajo.rpc.client.connection-timeout-ms"; /** Default client connection timeout 15 seconds */ - public final static long CLIENT_CONNECTION_TIMEOUT_DEFAULT = TimeUnit.SECONDS.toMicros(15); + public final static long CLIENT_CONNECTION_TIMEOUT_DEFAULT = TimeUnit.SECONDS.toMillis(15); /** * Socket timeout (milliseconds). Client connection will check if the connectivity is available after this time. */ public static final String CLIENT_SOCKET_TIMEOUT = "tajo.rpc.client.socket-timeout-ms"; /** Default socket timeout - 60 seconds */ - public final static long CLIENT_SOCKET_TIMEOUT_DEFAULT = TimeUnit.SECONDS.toNanos(180); + public final static long CLIENT_SOCKET_TIMEOUT_DEFAULT = TimeUnit.SECONDS.toMillis(180); } From 22ddc15685ae921e535a4b507583f03d65db234f Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Wed, 16 Sep 2015 01:35:26 -0700 Subject: [PATCH 07/24] fixed units. --- .../src/main/java/org/apache/tajo/rpc/NettyClientBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java index 13885911ea..ed8d2ccb3a 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java @@ -84,7 +84,7 @@ public NettyClientBase(RpcConnectionKey rpcConnectionKey, Properties connectionP connectionParameters.getProperty(CLIENT_CONNECTION_TIMEOUT, String.valueOf(CLIENT_CONNECTION_TIMEOUT_DEFAULT))); // Netty only takes integer value range and this is to avoid integer overflow. - Preconditions.checkArgument(this.connTimeoutMillis > Integer.MAX_VALUE, "Too long connection timeout"); + Preconditions.checkArgument(this.connTimeoutMillis < Integer.MAX_VALUE, "Too long connection timeout"); } // should be called from sub class From 4d91d290e28adc3df883a92a69533642b6fcf2a8 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Wed, 16 Sep 2015 15:01:45 -0700 Subject: [PATCH 08/24] Add missed license and fixed time units. --- .../tajo/util/TestRpcConnectionParamUtil.java | 18 ++++++++++++++++++ .../org/apache/tajo/rpc/AsyncRpcClient.java | 4 ++-- .../org/apache/tajo/rpc/BlockingRpcClient.java | 4 ++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcConnectionParamUtil.java b/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcConnectionParamUtil.java index 27e47b4370..f803c5bbf6 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcConnectionParamUtil.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcConnectionParamUtil.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.tajo.util; import org.apache.tajo.conf.TajoConf; diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java index ddc7296e99..b8889b3989 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java @@ -63,8 +63,8 @@ public class AsyncRpcClient extends NettyClientBase Date: Sun, 20 Sep 2015 14:57:51 -0700 Subject: [PATCH 09/24] Add hang detection option. --- .../java/org/apache/tajo/conf/TajoConf.java | 1 + .../java/org/apache/tajo/rpc/RpcConstants.java | 5 ++++- .../org/apache/tajo/rpc/AsyncRpcClient.java | 12 +++++++----- .../org/apache/tajo/rpc/BlockingRpcClient.java | 11 +++++++---- .../org/apache/tajo/rpc/NettyClientBase.java | 16 +++++++++++++--- .../tajo/rpc/ProtoClientChannelInitializer.java | 17 +++++++++++------ .../org/apache/tajo/rpc/TestBlockingRpc.java | 3 +++ 7 files changed, 46 insertions(+), 19 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 3ec2bbc496..167e364373 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -232,6 +232,7 @@ public static enum ConfVars implements ConfigKey { RPC_CLIENT_RETRY_NUM("tajo.rpc.client.retry-num", 3, Validators.min("1")), RPC_CLIENT_CONNECTION_TIMEOUT("tajo.rpc.client.connection-timeout-ms", (long)15 * 1000, Validators.min("1")), RPC_CLIENT_SOCKET_TIMEOUT("tajo.rpc.client.socket-timeout-ms", (long)180 * 1000, Validators.min("1")), + RPC_CLIENT_HANG_DETECTION_ENABLED("tajo.rpc.client.hang-detection", false, Validators.bool()), // Internal RPC Server MASTER_RPC_SERVER_WORKER_THREAD_NUM("tajo.master.rpc.server.worker-thread-num", diff --git a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java index 7652cb3054..c013dc14e3 100644 --- a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java +++ b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java @@ -39,9 +39,12 @@ public class RpcConstants { public final static long CLIENT_CONNECTION_TIMEOUT_DEFAULT = TimeUnit.SECONDS.toMillis(15); /** - * Socket timeout (milliseconds). Client connection will check if the connectivity is available after this time. + * Socket timeout (milliseconds). */ public static final String CLIENT_SOCKET_TIMEOUT = "tajo.rpc.client.socket-timeout-ms"; /** Default socket timeout - 60 seconds */ public final static long CLIENT_SOCKET_TIMEOUT_DEFAULT = TimeUnit.SECONDS.toMillis(180); + + public static final String CLIENT_HANG_DETECTION = "tajo.rpc.client.hang-detection"; + public final static boolean CLIENT_HANG_DETECTION_DEFAULT = false; } diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java index b8889b3989..4b33e7e935 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java @@ -26,11 +26,9 @@ import java.lang.reflect.Method; import java.util.Properties; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import static org.apache.tajo.rpc.RpcConstants.CLIENT_SOCKET_TIMEOUT; -import static org.apache.tajo.rpc.RpcConstants.CLIENT_SOCKET_TIMEOUT_DEFAULT; +import static org.apache.tajo.rpc.RpcConstants.*; public class AsyncRpcClient extends NettyClientBase { @@ -63,8 +61,12 @@ public class AsyncRpcClient extends NettyClientBase { @@ -64,8 +63,12 @@ public BlockingRpcClient(EventLoopGroup eventLoopGroup, long socketTimeoutMills = Long.parseLong( connectionParameters.getProperty(CLIENT_SOCKET_TIMEOUT, String.valueOf(CLIENT_SOCKET_TIMEOUT_DEFAULT))); - init(new ProtoClientChannelInitializer(handler, RpcResponse.getDefaultInstance(), socketTimeoutMills), - eventLoopGroup); + // Enable proactive hang detection + final boolean hangDetectionEnabled = Boolean.parseBoolean( + connectionParameters.getProperty(CLIENT_HANG_DETECTION, String.valueOf(CLIENT_HANG_DETECTION_DEFAULT))); + + init(new ProtoClientChannelInitializer(handler, RpcResponse.getDefaultInstance(), socketTimeoutMills, + hangDetectionEnabled), eventLoopGroup); } @Override diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java index ed8d2ccb3a..13940a0d71 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java @@ -56,7 +56,7 @@ public abstract class NettyClientBase implements ProtoDeclaration, Closeable private final int maxRetryNum; /** Connection Timeout */ private final long connTimeoutMillis; - + private boolean enableMonitor; private final ConcurrentMap channelEventListeners = new ConcurrentHashMap<>(); private final ConcurrentMap requests = new ConcurrentHashMap<>(); @@ -84,7 +84,7 @@ public NettyClientBase(RpcConnectionKey rpcConnectionKey, Properties connectionP connectionParameters.getProperty(CLIENT_CONNECTION_TIMEOUT, String.valueOf(CLIENT_CONNECTION_TIMEOUT_DEFAULT))); // Netty only takes integer value range and this is to avoid integer overflow. - Preconditions.checkArgument(this.connTimeoutMillis < Integer.MAX_VALUE, "Too long connection timeout"); + Preconditions.checkArgument(this.connTimeoutMillis <= Integer.MAX_VALUE, "Too long connection timeout"); } // should be called from sub class @@ -310,6 +310,9 @@ protected void registerCallback(int seqId, T callback) { @Override public void channelRegistered(ChannelHandlerContext ctx) throws Exception { MonitorClientHandler handler = ctx.pipeline().get(MonitorClientHandler.class); + if (handler != null) { + enableMonitor = true; + } for (ChannelEventListener listener : getSubscribers()) { listener.channelRegistered(ctx); @@ -406,7 +409,14 @@ private void sendException(RecoverableException e) { @Override public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { - if (evt instanceof MonitorStateEvent) { + if (!enableMonitor && evt instanceof IdleStateEvent) { + IdleStateEvent e = (IdleStateEvent) evt; + /* If all requests is done and event is triggered, idle channel close. */ + if (e.state() == IdleState.READER_IDLE && requests.isEmpty()) { + ctx.close(); + LOG.info("Idle connection closed successfully :" + ctx.channel()); + } + } else if (evt instanceof MonitorStateEvent) { MonitorStateEvent e = (MonitorStateEvent) evt; if (e.state() == MonitorStateEvent.MonitorState.PING_EXPIRED) { exceptionCaught(ctx, new ServiceException("Server has not respond: " + ctx.channel())); diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java index 63e3c218a5..396be36b4c 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java @@ -34,28 +34,33 @@ class ProtoClientChannelInitializer extends ChannelInitializer { private final MessageLite defaultInstance; private final ChannelHandler handler; - private final long socketTimeout; + private final long idleTimeout; + private final boolean usePing; /** * Channel Pipe Initializer * * @param handler Channel Handler * @param defaultInstance Default Rpc Proto instance - * @param socketTimeout Socket timeout (milliseconds) + * @param idleTimeout Idle timeout (milliseconds) */ - public ProtoClientChannelInitializer(ChannelHandler handler, MessageLite defaultInstance, long socketTimeout) { + public ProtoClientChannelInitializer(ChannelHandler handler, MessageLite defaultInstance, + long idleTimeout, boolean hangDetection) { this.handler = handler; this.defaultInstance = defaultInstance; - this.socketTimeout = socketTimeout; + this.idleTimeout = idleTimeout; + this.usePing = hangDetection; } @Override protected void initChannel(Channel channel) throws Exception { ChannelPipeline pipeline = channel.pipeline(); pipeline.addLast("idleStateHandler", - new IdleStateHandler(socketTimeout, socketTimeout / 2, 0, TimeUnit.MILLISECONDS)); + new IdleStateHandler(idleTimeout, idleTimeout / 2, 0, TimeUnit.MILLISECONDS)); - pipeline.addLast("MonitorClientHandler", new MonitorClientHandler()); + if (usePing) { + pipeline.addLast("MonitorClientHandler", new MonitorClientHandler()); + } pipeline.addLast("frameDecoder", new ProtobufVarint32FrameDecoder()); pipeline.addLast("protobufDecoder", new ProtobufDecoder(defaultInstance)); pipeline.addLast("frameEncoder", new ProtobufVarint32LengthFieldPrepender()); diff --git a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java index 371864935e..3024883caa 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java @@ -123,6 +123,7 @@ public void setUpRpcClient() throws Exception { Properties connParams = new Properties(); connParams.setProperty(RpcConstants.RPC_RETRY_NUM, "1"); connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(TimeUnit.SECONDS.toMillis(10))); + connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true"); RpcConnectionKey rpcConnectionKey = new RpcConnectionKey( @@ -496,6 +497,7 @@ public void testPingOnIdle() throws Exception { Properties connParams = new Properties(); connParams.setProperty(RpcConstants.RPC_RETRY_NUM, retries + ""); connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); + connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true"); BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); @@ -545,6 +547,7 @@ public void testRequestTimeoutOnBusy() throws Exception { Properties connParams = new Properties(); connParams.setProperty(RpcConstants.RPC_RETRY_NUM, retries + ""); connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); + connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true"); BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); From 14adeb927ee440b79884ce4b09b1c9aa7fab0d00 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Sun, 20 Sep 2015 21:42:54 -0700 Subject: [PATCH 10/24] Fixed unit test failure. --- .../src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java index cec58917f5..94962215b4 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java @@ -136,6 +136,8 @@ public void setUpRpcClient() throws Exception { Properties connParams = new Properties(); connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(TimeUnit.SECONDS.toMillis(10))); + connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true"); + client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); @@ -539,6 +541,7 @@ public void testPingOnIdle() throws Exception { Properties connParams = new Properties(); connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); + connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true"); AsyncRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); @@ -591,6 +594,7 @@ public void testRequestTimeoutOnBusy() throws Exception { Properties connParams = new Properties(); connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); + connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true"); AsyncRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); From 503e20d0dd9d859e209bbc6738fd3b319d64dce3 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Sun, 20 Sep 2015 23:22:51 -0700 Subject: [PATCH 11/24] Improve TajoClient and TajoDriver to take connection parameters. --- .../tajo/client/ConnectionParameters.java | 125 ++++++++++++++++++ .../apache/tajo/client/SessionConnection.java | 8 +- .../org/apache/tajo/jdbc/TestTajoJdbc.java | 2 +- 3 files changed, 130 insertions(+), 5 deletions(-) create mode 100644 tajo-client/src/main/java/org/apache/tajo/client/ConnectionParameters.java diff --git a/tajo-client/src/main/java/org/apache/tajo/client/ConnectionParameters.java b/tajo-client/src/main/java/org/apache/tajo/client/ConnectionParameters.java new file mode 100644 index 0000000000..47dab1886e --- /dev/null +++ b/tajo-client/src/main/java/org/apache/tajo/client/ConnectionParameters.java @@ -0,0 +1,125 @@ +package org.apache.tajo.client; + +import com.google.common.base.Predicate; +import com.google.common.collect.Collections2; +import org.apache.tajo.SessionVars; +import org.apache.tajo.util.Pair; + +import java.util.*; + +import static org.apache.tajo.SessionVars.COMPRESSED_RESULT_TRANSFER; +import static org.apache.tajo.SessionVars.FETCH_ROWNUM; +import static org.apache.tajo.client.ConnectionParameters.ActionType.CONNECTION_PARAM; +import static org.apache.tajo.client.ConnectionParameters.ActionType.SESSION_UPDATE; +import static org.apache.tajo.rpc.RpcConstants.CLIENT_CONNECTION_TIMEOUT; +import static org.apache.tajo.rpc.RpcConstants.CLIENT_SOCKET_TIMEOUT; + +/** + *
    + *
  • useCompression=bool - Enable compressed transfer for ResultSet.
    • + *
  • defaultRowFetchSize=int - Determine the number of rows fetched in ResultSet by + * one fetch with trip to the Server.
    • + *
  • connectTimeout=int - The timeout value used for socket connect operations. + * If connecting to the server takes longer than this value,the connection is broken. The + * timeout is specified in seconds and a value of zero means that it is disabled.
    • + *
  • socketTimeout=int
    • - The timeout value used for socket read operations. + * If reading from the server takes longer than this value, the connection is closed. + * This can be used as both a brute force global query timeout and a method of detecting + * network problems. The timeout is specified in seconds and a value of zero means that + * it is disabled. + *
+ */ +public class ConnectionParameters { + + public static Map PARAMETERS = new HashMap<>(); + + static { + PARAMETERS.put("useCompression", new SimpleSessionAction(COMPRESSED_RESULT_TRANSFER)); + PARAMETERS.put("defaultRowFetchSize", new SimpleSessionAction(FETCH_ROWNUM)); + + PARAMETERS.put("connectTimeout", new SimpleConnectionParamAction(CLIENT_CONNECTION_TIMEOUT)); + PARAMETERS.put("socketTimeout", new SimpleConnectionParamAction(CLIENT_SOCKET_TIMEOUT)); + } + + enum ActionType { + SESSION_UPDATE, + CONNECTION_PARAM + } + + interface Action { + ActionType type(); + } + + static class SimpleSessionAction extends SessionAction { + private final String sessionKey; + + SimpleSessionAction(SessionVars sessionVar) { + this.sessionKey = sessionVar.name(); + } + + Pair doAction(String param) { + return new Pair<>(sessionKey, param); + } + } + + @SuppressWarnings("unused") + static abstract class SessionAction implements Action { + + @Override + public ActionType type() { + return SESSION_UPDATE; + } + + abstract Pair doAction(String param); + } + + static class SimpleConnectionParamAction extends ConnectionParamAction { + final String connParamKey; + + SimpleConnectionParamAction(String connParamKey) { + this.connParamKey = connParamKey; + } + + public Pair doAction(String param) { + return new Pair<>(connParamKey, param); + } + } + + @SuppressWarnings("unused") + static abstract class ConnectionParamAction implements Action { + + @Override + public ActionType type() { + return ActionType.CONNECTION_PARAM; + } + + abstract Pair doAction(String param); + } + + public static Properties getConnParams(Collection> properties) { + Properties connParams = new Properties(); + for (Map.Entry entry : properties) { + if(PARAMETERS.containsKey(entry.getKey()) && PARAMETERS.get(entry.getKey()).type() == CONNECTION_PARAM) { + Pair keyValue = + ((ConnectionParamAction)PARAMETERS.get(entry.getKey())).doAction(entry.getValue()); + connParams.put(keyValue.getFirst(), keyValue.getSecond()); + } + } + + return connParams; + } + + public static Map getSessionVars(Collection> properties) { + Map sessionVars = new HashMap<>(); + + for (Map.Entry entry : properties) { + if(PARAMETERS.containsKey(entry.getKey()) && PARAMETERS.get(entry.getKey()).type() == SESSION_UPDATE) { + Pair keyValue = + ((SessionAction)PARAMETERS.get(entry.getKey())).doAction(entry.getValue()); + sessionVars.put(keyValue.getFirst(), keyValue.getSecond()); + } + } + + return sessionVars; + } +} diff --git a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java index 19beb9b377..0539b5a28d 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java @@ -102,10 +102,9 @@ public SessionConnection(@NotNull ServiceTracker tracker, @Nullable String baseD this.properties = properties; this.manager = RpcClientManager.getInstance(); - this.clientConnParams = new Properties(); - this.clientConnParams.setProperty(RpcConstants.RPC_RETRY_NUM, properties.get(RpcConstants.RPC_RETRY_NUM)); - this.userInfo = UserRoleInfo.getCurrentUser(); + // update the connection parameters to RPC client from connection properties + this.clientConnParams = ConnectionParameters.getConnParams(properties.getAllKeyValus().entrySet()); this.eventLoopGroup = NettyUtils.createEventLoopGroup(getClass().getSimpleName(), 4); try { @@ -114,6 +113,8 @@ public SessionConnection(@NotNull ServiceTracker tracker, @Nullable String baseD NettyUtils.shutdown(eventLoopGroup); throw e; } + // update the session variables from connection parameters + updateSessionVariables(ConnectionParameters.getSessionVars(properties.getAllKeyValus().entrySet())); } public Map getClientSideSessionVars() { @@ -218,7 +219,6 @@ public Map updateSessionVariables(final Map vari ensureOk(response.getState()); updateSessionVarsCache(ProtoUtil.convertToMap(response.getSessionVars())); - properties.putAll(sessionVarsCache); return Collections.unmodifiableMap(sessionVarsCache); } diff --git a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java index a860d5119e..48d615830e 100644 --- a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java +++ b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java @@ -197,7 +197,7 @@ public void testPreparedStatement() throws Exception { public void testResultSetCompression() throws Exception { String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(), TajoConstants.DEFAULT_DATABASE_NAME); - connUri = connUri + "?" + SessionVars.COMPRESSED_RESULT_TRANSFER.keyname() + "=true"; + connUri = connUri + "?useCompression=true"; Connection conn = DriverManager.getConnection(connUri); assertTrue(conn.isValid(100)); From e3a5c8abadd59e10609de84bd85df291e2b7bc09 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Sun, 20 Sep 2015 23:57:34 -0700 Subject: [PATCH 12/24] Add missed license. --- .../java/org/apache/tajo/client/ConnectionParameters.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tajo-client/src/main/java/org/apache/tajo/client/ConnectionParameters.java b/tajo-client/src/main/java/org/apache/tajo/client/ConnectionParameters.java index 47dab1886e..ad32ed0249 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/ConnectionParameters.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/ConnectionParameters.java @@ -1,11 +1,12 @@ package org.apache.tajo.client; -import com.google.common.base.Predicate; -import com.google.common.collect.Collections2; import org.apache.tajo.SessionVars; import org.apache.tajo.util.Pair; -import java.util.*; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; import static org.apache.tajo.SessionVars.COMPRESSED_RESULT_TRANSFER; import static org.apache.tajo.SessionVars.FETCH_ROWNUM; From 2fa3e3aa2e2bfd44e7b182b2e20cf9a9eef014ec Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 21 Sep 2015 00:11:14 -0700 Subject: [PATCH 13/24] add missed license. --- .../tajo/client/ConnectionParameters.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tajo-client/src/main/java/org/apache/tajo/client/ConnectionParameters.java b/tajo-client/src/main/java/org/apache/tajo/client/ConnectionParameters.java index ad32ed0249..34d016d60b 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/ConnectionParameters.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/ConnectionParameters.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.tajo.client; import org.apache.tajo.SessionVars; From b274af70c920a33d2295918f97589c1871c0a3c1 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 21 Sep 2015 02:57:39 -0700 Subject: [PATCH 14/24] Add connection parameters to javadoc documentation and add unit tests to verify connection parameters. --- .../apache/tajo/catalog/CatalogClient.java | 2 +- ...ers.java => ConnectionPropertyHelper.java} | 29 ++++++++-- .../apache/tajo/client/SessionConnection.java | 7 +-- .../java/org/apache/tajo/conf/TajoConf.java | 6 +- .../tajo/exception/SQLExceptionUtil.java | 4 +- tajo-common/src/main/proto/errors.proto | 4 +- .../tajo/util/TestRpcConnectionParamUtil.java | 6 +- .../tajo/util/RpcConnectionParamUtil.java | 2 +- tajo-docs/src/main/sphinx/jdbc_driver.rst | 43 ++++++++++++++ .../org/apache/tajo/jdbc/TestTajoJdbc.java | 56 ++++++++++++++++++- .../org/apache/tajo/rpc/RpcConstants.java | 4 +- .../org/apache/tajo/rpc/NettyClientBase.java | 2 +- .../org/apache/tajo/rpc/TestAsyncRpc.java | 20 +++---- .../org/apache/tajo/rpc/TestBlockingRpc.java | 20 +++---- 14 files changed, 159 insertions(+), 46 deletions(-) rename tajo-client/src/main/java/org/apache/tajo/client/{ConnectionParameters.java => ConnectionPropertyHelper.java} (79%) diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java index 9bd6afd859..c59058eb5b 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java @@ -70,7 +70,7 @@ public synchronized NettyClientBase getCatalogConnection() throws ServiceExcepti RpcClientManager.cleanup(client); final Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, conf.getVar(ConfVars.RPC_CLIENT_RETRY_NUM)); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, conf.getVar(ConfVars.RPC_CLIENT_RETRY_NUM)); // Client do not closed on idle state for support high available this.client = RpcClientManager.getInstance().newClient(getCatalogServerAddr(), CatalogProtocol.class, diff --git a/tajo-client/src/main/java/org/apache/tajo/client/ConnectionParameters.java b/tajo-client/src/main/java/org/apache/tajo/client/ConnectionPropertyHelper.java similarity index 79% rename from tajo-client/src/main/java/org/apache/tajo/client/ConnectionParameters.java rename to tajo-client/src/main/java/org/apache/tajo/client/ConnectionPropertyHelper.java index 34d016d60b..467db41c33 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/ConnectionParameters.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/ConnectionPropertyHelper.java @@ -19,17 +19,19 @@ package org.apache.tajo.client; import org.apache.tajo.SessionVars; +import org.apache.tajo.rpc.RpcConstants; import org.apache.tajo.util.Pair; import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Properties; +import java.util.concurrent.TimeUnit; import static org.apache.tajo.SessionVars.COMPRESSED_RESULT_TRANSFER; import static org.apache.tajo.SessionVars.FETCH_ROWNUM; -import static org.apache.tajo.client.ConnectionParameters.ActionType.CONNECTION_PARAM; -import static org.apache.tajo.client.ConnectionParameters.ActionType.SESSION_UPDATE; +import static org.apache.tajo.client.ConnectionPropertyHelper.ActionType.CONNECTION_PARAM; +import static org.apache.tajo.client.ConnectionPropertyHelper.ActionType.SESSION_UPDATE; import static org.apache.tajo.rpc.RpcConstants.CLIENT_CONNECTION_TIMEOUT; import static org.apache.tajo.rpc.RpcConstants.CLIENT_SOCKET_TIMEOUT; @@ -46,18 +48,33 @@ * This can be used as both a brute force global query timeout and a method of detecting * network problems. The timeout is specified in seconds and a value of zero means that * it is disabled. + *
  • retry=intNumber of retry operation. Tajo JDBC driver is resilient + * against some network or connection problems. It determines how many times the connection will retry.
    • * */ -public class ConnectionParameters { +class ConnectionPropertyHelper { public static Map PARAMETERS = new HashMap<>(); static { PARAMETERS.put("useCompression", new SimpleSessionAction(COMPRESSED_RESULT_TRANSFER)); PARAMETERS.put("defaultRowFetchSize", new SimpleSessionAction(FETCH_ROWNUM)); - - PARAMETERS.put("connectTimeout", new SimpleConnectionParamAction(CLIENT_CONNECTION_TIMEOUT)); - PARAMETERS.put("socketTimeout", new SimpleConnectionParamAction(CLIENT_SOCKET_TIMEOUT)); + PARAMETERS.put("connectTimeout", new ConnectionParamAction() { + @Override + Pair doAction(String param) { + int seconds = Integer.parseInt(param); + // convert seconds into mili seconds + return new Pair<>(CLIENT_CONNECTION_TIMEOUT, String.valueOf(TimeUnit.SECONDS.toMillis(seconds))); + } + }); + PARAMETERS.put("socketTimeout", new ConnectionParamAction() { + @Override + Pair doAction(String param) { + int seconds = Integer.parseInt(param); + return new Pair<>(CLIENT_SOCKET_TIMEOUT, String.valueOf(TimeUnit.SECONDS.toMillis(seconds))); + } + }); + PARAMETERS.put("retry", new SimpleConnectionParamAction(RpcConstants.CLIENT_RETRY_NUM)); } enum ActionType { diff --git a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java index 0539b5a28d..1688235e69 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java @@ -40,7 +40,6 @@ import org.apache.tajo.rpc.NettyClientBase; import org.apache.tajo.rpc.NettyUtils; import org.apache.tajo.rpc.RpcClientManager; -import org.apache.tajo.rpc.RpcConstants; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetResponse; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.ReturnState; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringResponse; @@ -104,7 +103,7 @@ public SessionConnection(@NotNull ServiceTracker tracker, @Nullable String baseD this.manager = RpcClientManager.getInstance(); this.userInfo = UserRoleInfo.getCurrentUser(); // update the connection parameters to RPC client from connection properties - this.clientConnParams = ConnectionParameters.getConnParams(properties.getAllKeyValus().entrySet()); + this.clientConnParams = ConnectionPropertyHelper.getConnParams(properties.getAllKeyValus().entrySet()); this.eventLoopGroup = NettyUtils.createEventLoopGroup(getClass().getSimpleName(), 4); try { @@ -113,8 +112,9 @@ public SessionConnection(@NotNull ServiceTracker tracker, @Nullable String baseD NettyUtils.shutdown(eventLoopGroup); throw e; } + // update the session variables from connection parameters - updateSessionVariables(ConnectionParameters.getSessionVars(properties.getAllKeyValus().entrySet())); + updateSessionVariables(ConnectionPropertyHelper.getSessionVars(properties.getAllKeyValus().entrySet())); } public Map getClientSideSessionVars() { @@ -363,7 +363,6 @@ protected void checkSessionAndGet(NettyClientBase client) { } CreateSessionResponse response = null; - try { response = tajoMasterService.createSession(null, builder.build()); } catch (ServiceException se) { diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 167e364373..cd0535bb03 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -230,9 +230,9 @@ public static enum ConfVars implements ConfigKey { INTERNAL_RPC_CLIENT_WORKER_THREAD_NUM("tajo.internal.rpc.client.worker-thread-num", Runtime.getRuntime().availableProcessors() * 2), RPC_CLIENT_RETRY_NUM("tajo.rpc.client.retry-num", 3, Validators.min("1")), - RPC_CLIENT_CONNECTION_TIMEOUT("tajo.rpc.client.connection-timeout-ms", (long)15 * 1000, Validators.min("1")), - RPC_CLIENT_SOCKET_TIMEOUT("tajo.rpc.client.socket-timeout-ms", (long)180 * 1000, Validators.min("1")), - RPC_CLIENT_HANG_DETECTION_ENABLED("tajo.rpc.client.hang-detection", false, Validators.bool()), + RPC_CLIENT_CONNECTION_TIMEOUT("tajo.rpc.client.connection-timeout-ms", (long)15 * 1000, Validators.min("0")), + RPC_CLIENT_SOCKET_TIMEOUT("tajo.rpc.client.socket-timeout-ms", (long)180 * 1000, Validators.min("0")), + RPC_CLIENT_HANG_DETECTION_ENABLED("tajo.rpc.client.hang-detection", true, Validators.bool()), // Internal RPC Server MASTER_RPC_SERVER_WORKER_THREAD_NUM("tajo.master.rpc.server.worker-thread-num", diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/SQLExceptionUtil.java b/tajo-common/src/main/java/org/apache/tajo/exception/SQLExceptionUtil.java index b5708e13b4..19a2f000f5 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/SQLExceptionUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/SQLExceptionUtil.java @@ -103,9 +103,9 @@ public class SQLExceptionUtil { SQLSTATES.put(ResultCode.INDETERMINATE_DATATYPE, "42P18"); // Client Connection - SQLSTATES.put(ResultCode.CLIENT_CONNECTION_EXCEPTION, "08001"); - SQLSTATES.put(ResultCode.CLIENT_UNABLE_TO_ESTABLISH_CONNECTION, "08002"); + SQLSTATES.put(ResultCode.CLIENT_CONNECTION_EXCEPTION, "08000"); SQLSTATES.put(ResultCode.CLIENT_CONNECTION_DOES_NOT_EXIST, "08003"); + SQLSTATES.put(ResultCode.CLIENT_UNABLE_TO_ESTABLISH_CONNECTION, "08006"); } public static boolean isThisError(SQLException e, ResultCode code) { diff --git a/tajo-common/src/main/proto/errors.proto b/tajo-common/src/main/proto/errors.proto index 6a1780be17..01522686d0 100644 --- a/tajo-common/src/main/proto/errors.proto +++ b/tajo-common/src/main/proto/errors.proto @@ -171,9 +171,9 @@ enum ResultCode { // Client Connection CLIENT_CONNECTION_EXCEPTION = 1101; // SQLState: 08000 - Client connection error - CLIENT_UNABLE_TO_ESTABLISH_CONNECTION = 1102; // SQLState: 08001 - CLIENT_CONNECTION_DOES_NOT_EXIST = 1103; // SQLState: 08003 - Client connection has been closed. - CLIENT_PROTOCOL_PROTOCOL_VIOLATION = 1104; // SQLState: ? + CLIENT_UNABLE_TO_ESTABLISH_CONNECTION = 1102; // SQLState: 08006 - Client connection failure + CLIENT_PROTOCOL_PROTOCOL_VIOLATION = 1104; // SQLState: 08P01 - Protocol violation // 53 - Invalid Operand or Inconsistent Specification INSUFFICIENT_RESOURCE = 53000; diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcConnectionParamUtil.java b/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcConnectionParamUtil.java index f803c5bbf6..1a506fdf21 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcConnectionParamUtil.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcConnectionParamUtil.java @@ -26,7 +26,7 @@ import java.util.Properties; import static org.apache.tajo.rpc.RpcConstants.CLIENT_CONNECTION_TIMEOUT; -import static org.apache.tajo.rpc.RpcConstants.RPC_RETRY_NUM; +import static org.apache.tajo.rpc.RpcConstants.CLIENT_RETRY_NUM; import static org.junit.Assert.*; public class TestRpcConnectionParamUtil { @@ -36,7 +36,7 @@ public void testGetDefaults() throws Exception { TajoConf conf = new TajoConf(); Properties defaultParams = RpcConnectionParamUtil.get(conf); assertEquals( - ConfVars.RPC_CLIENT_RETRY_NUM.defaultVal, defaultParams.getProperty(RPC_RETRY_NUM)); + ConfVars.RPC_CLIENT_RETRY_NUM.defaultVal, defaultParams.getProperty(CLIENT_RETRY_NUM)); assertEquals( ConfVars.RPC_CLIENT_CONNECTION_TIMEOUT.defaultVal, defaultParams.getProperty(CLIENT_CONNECTION_TIMEOUT)); assertEquals( @@ -51,7 +51,7 @@ public void testGet() throws Exception { conf.setLongVar(ConfVars.RPC_CLIENT_SOCKET_TIMEOUT, (long)60 * 1000); Properties defaultParams = RpcConnectionParamUtil.get(conf); - assertEquals("100", defaultParams.getProperty(RPC_RETRY_NUM)); + assertEquals("100", defaultParams.getProperty(CLIENT_RETRY_NUM)); assertEquals(10 * 1000 + "", defaultParams.getProperty(CLIENT_CONNECTION_TIMEOUT)); assertEquals(60 * 1000 + "", defaultParams.getProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT)); } diff --git a/tajo-core/src/main/java/org/apache/tajo/util/RpcConnectionParamUtil.java b/tajo-core/src/main/java/org/apache/tajo/util/RpcConnectionParamUtil.java index 9603c9584e..3dff61fe55 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/RpcConnectionParamUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/RpcConnectionParamUtil.java @@ -33,7 +33,7 @@ public class RpcConnectionParamUtil { static final Map PROPERTIES_MAP = new HashMap<>(); static { - PROPERTIES_MAP.put(RpcConstants.RPC_RETRY_NUM, ConfVars.RPC_CLIENT_RETRY_NUM); + PROPERTIES_MAP.put(RpcConstants.CLIENT_RETRY_NUM, ConfVars.RPC_CLIENT_RETRY_NUM); PROPERTIES_MAP.put(RpcConstants.CLIENT_CONNECTION_TIMEOUT, ConfVars.RPC_CLIENT_CONNECTION_TIMEOUT); PROPERTIES_MAP.put(RpcConstants.CLIENT_SOCKET_TIMEOUT, ConfVars.RPC_CLIENT_SOCKET_TIMEOUT); } diff --git a/tajo-docs/src/main/sphinx/jdbc_driver.rst b/tajo-docs/src/main/sphinx/jdbc_driver.rst index 6c7371bf4e..176477aba9 100644 --- a/tajo-docs/src/main/sphinx/jdbc_driver.rst +++ b/tajo-docs/src/main/sphinx/jdbc_driver.rst @@ -42,6 +42,49 @@ In order to use the JDBC driver, you should add ``tajo-jdbc-x.y.z.jar`` in your CLASSPATH=path/to/tajo-jdbc-x.y.z.jar:$CLASSPATH +Connecting to the Tajo cluster instance +======================================= +A Tajo cluster is represented by a URL. Tajo JDBC driver can take the following URL forms: + + * ``jdbc:tajo://host/`` + * ``jdbc:tajo://host/database`` + * ``jdbc:tajo://host:port/`` + * ``jdbc:tajo://host:port/database`` + +Each part of URL has the following meanings: + + * ``host`` - The hostname of the TajoMaster. You can put hostname or ip address here. + * ``port`` - The port number that server is listening. Default port number is 26002. + * ``database`` - The database name. The default database name is ``default``. + + To connect, you need to get ``Connection`` instance from Java JDBC Driver Manager as follows: + +.. code-block:: java + + Connection db = DriverManager.getConnection(url); + + +Connection Parameters +===================== +Connection parameters lets the JDBC Copnnection to enable or disable additional features. You should use ``java.util.Properties`` to pass your connection parameters into ``Connection``. The following example means that the transmission of ResultSet uses compression and its connection timeout is 15 seconds. + +.. code-block:: java + + String url = "jdbc:tajo://localhost/test"; + Properties props = new Properties(); + props.setProperty("useCompression","true"); // use compression for ResultSet + props.setProperty("connectTimeout","15000"); // 15 seconds + Connection conn = DriverManager.getConnection(url, props); + +The connection parameters that Tajo currently supports are as follows: + + * ``useCompression = bool`` - Enable compressed transfer for ResultSet. + * ``defaultRowFetchSize = int`` - Determine the number of rows fetched in ResultSet by one fetch with trip to the Server. + * ``connectTimeout = int (seconds)`` - The timeout value used for socket connect operations. If connecting to the server takes longer than this value, the connection is broken. The timeout is specified in seconds and a value of zero means that it is disabled. + * ``socketTimeout = int (seconds)`` - The timeout value used for socket read operations. If reading from the server takes longer than this value, the connection is closed. This can be used as both a brute force global query timeout and a method of detecting network problems. The timeout is specified in seconds and a value of zero means that it is disabled. + * ``retry = int`` - Number of retry operation. Tajo JDBC driver is resilient against some network or connection problems. It determines how many times the connection will retry. + + An Example JDBC Client ======================= diff --git a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java index 48d615830e..a613e91a53 100644 --- a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java +++ b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java @@ -19,21 +19,32 @@ package org.apache.tajo.jdbc; import com.google.common.collect.Maps; +import io.netty.channel.ConnectTimeoutException; import org.apache.tajo.*; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.client.QueryStatus; +import org.apache.tajo.error.Errors; +import org.apache.tajo.exception.SQLExceptionUtil; +import org.apache.tajo.util.UriUtil; import org.junit.AfterClass; +import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import java.io.IOException; import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketTimeoutException; import java.sql.*; import java.util.*; import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; +import static org.apache.tajo.error.Errors.ResultCode.CLIENT_CONNECTION_EXCEPTION; +import static org.apache.tajo.error.Errors.ResultCode.CLIENT_UNABLE_TO_ESTABLISH_CONNECTION; import static org.junit.Assert.*; @Category(IntegrationTest.class) @@ -593,7 +604,7 @@ public void testAlterTableAddPartition() throws Exception { try { if (!testingCluster.isHiveCatalogStoreRunning()) { String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), - tajoMasterAddress.getPort(), "TestTajoJdbc"); + tajoMasterAddress.getPort(), "TestTajoJdbc"); conn = DriverManager.getConnection(connUri); assertTrue(conn.isValid(100)); @@ -682,4 +693,47 @@ public final void testCancel() throws Exception { } } } + + private void assumeConnectTimeout(String host, int port, int connectTimeout) throws IOException { + try (Socket socket = new Socket()) { + // Try to connect to a private address in the 10.x.y.z range. + // These addresses are usually not routed, so an attempt to + // connect to them will hang the connection attempt, which is + // what we want to simulate in this test. + socket.connect(new InetSocketAddress(host, port), connectTimeout); + // Abort the test if we can connect. + Assume.assumeTrue(false); + } catch (SocketTimeoutException x) { + // Expected timeout during connect, continue the test. + Assume.assumeTrue(true); + } catch (Throwable x) { + // Abort if any other exception happens. + Assume.assumeTrue(false); + } + } + + @Test(timeout = 5000) + public final void testConnectTimeout() throws Exception { + final String host = "10.255.255.1"; + final int port = 80; + int connectTimeout = 1000; + assumeConnectTimeout(host, port, connectTimeout); + + long startTime = Long.MIN_VALUE; + long endTime; + try { + // artificially cause connection timeout + String connUri = buildConnectionUri(host, port, DEFAULT_DATABASE_NAME); + connUri = UriUtil.addParam(connUri, "connectTimeout", "1"); // 1 seconds + startTime = System.currentTimeMillis(); + new JdbcConnection(connUri, new Properties()); + fail("Must be failed"); + } catch (SQLException t) { + endTime = System.currentTimeMillis(); + assertEquals(t.getSQLState(), SQLExceptionUtil.toSQLState(CLIENT_CONNECTION_EXCEPTION)); + // default is 15 seconds. So, if timeout is shorter than 1~2 seconds. + // We can ensure the parameter was effective. + assertTrue(((endTime - startTime) / 1000) < 2); + } + } } diff --git a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java index c013dc14e3..601f3d2156 100644 --- a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java +++ b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java @@ -30,8 +30,8 @@ public class RpcConstants { public static final int FUTURE_TIMEOUT_SECONDS_DEFAULT = 10; /** How many times the connect will retry */ - public static final String RPC_RETRY_NUM = "tajo.rpc.client.retry-num"; - public static final int RPC_RETRY_NUM_DEFAULT = 3; + public static final String CLIENT_RETRY_NUM = "tajo.rpc.client.retry-num"; + public static final int CLIENT_RETRY_NUM_DEFAULT = 0; /** Client connection timeout (milliseconds) */ public static final String CLIENT_CONNECTION_TIMEOUT = "tajo.rpc.client.connection-timeout-ms"; diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java index 13940a0d71..39340c0033 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java @@ -78,7 +78,7 @@ public NettyClientBase(RpcConnectionKey rpcConnectionKey, Properties connectionP this.key = rpcConnectionKey; this.maxRetryNum = Integer.parseInt( - connectionParameters.getProperty(RPC_RETRY_NUM, String.valueOf(RPC_RETRY_NUM_DEFAULT))); + connectionParameters.getProperty(CLIENT_RETRY_NUM, String.valueOf(CLIENT_RETRY_NUM_DEFAULT))); this.connTimeoutMillis = Integer.parseInt( connectionParameters.getProperty(CLIENT_CONNECTION_TIMEOUT, String.valueOf(CLIENT_CONNECTION_TIMEOUT_DEFAULT))); diff --git a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java index 94962215b4..6427ffe6c0 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java @@ -134,7 +134,7 @@ public void setUpRpcClient() throws Exception { RpcUtils.getConnectAddress(server.getListenAddress()), DummyProtocol.class, true); Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(retries)); connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(TimeUnit.SECONDS.toMillis(10))); connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true"); @@ -358,7 +358,7 @@ public void run() { new RpcConnectionKey(address, DummyProtocol.class, true); Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(retries)); AsyncRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); @@ -391,7 +391,7 @@ public void testClientRetryFailureOnStartup() throws Exception { new RpcConnectionKey(address, DummyProtocol.class, true); Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(retries)); AsyncRpcClient client = new AsyncRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams); try { @@ -427,7 +427,7 @@ public void testUnresolvedAddress() throws Exception { new RpcConnectionKey(address, DummyProtocol.class, true); Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(retries)); client = new AsyncRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams); client.connect(); @@ -452,7 +452,7 @@ public void testUnresolvedAddress2() throws Exception { RpcUtils.createUnresolved(hostAndPort), DummyProtocol.class, true); Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(retries)); AsyncRpcClient client = new AsyncRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams); client.connect(); @@ -479,7 +479,7 @@ public void testStubRecovery() throws Exception { new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(2)); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(2)); AsyncRpcClient client = manager.newClient(rpcConnectionKey, connParams); EchoMessage echoMessage = EchoMessage.newBuilder() @@ -514,7 +514,7 @@ public void testIdleTimeout() throws Exception { // 500 millis idle timeout Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(retries)); connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); AsyncRpcClient client = manager.newClient(rpcConnectionKey, connParams); @@ -539,7 +539,7 @@ public void testPingOnIdle() throws Exception { // 500 millis idle timeout Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(retries)); connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true"); @@ -562,7 +562,7 @@ public void testIdleTimeoutWithActiveRequest() throws Exception { // 500 millis idle timeout Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(retries)); connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); AsyncRpcClient client = manager.newClient(rpcConnectionKey, connParams); @@ -592,7 +592,7 @@ public void testRequestTimeoutOnBusy() throws Exception { // 500 millis idle timeout Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(retries)); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(retries)); connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true"); diff --git a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java index 3024883caa..0687d0ba35 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java @@ -121,7 +121,7 @@ public void setUpRpcClient() throws Exception { retries = 1; Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, "1"); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, "1"); connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(TimeUnit.SECONDS.toMillis(10))); connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true"); @@ -327,7 +327,7 @@ public void run() { new RpcConnectionKey(address, DummyProtocol.class, false); Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, retries + ""); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + ""); BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); @@ -354,7 +354,7 @@ public void testClientRetryFailureOnStartup() throws Exception { new RpcConnectionKey(address, DummyProtocol.class, false); Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, retries + ""); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + ""); BlockingRpcClient client = new BlockingRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams); @@ -387,7 +387,7 @@ public void testUnresolvedAddress() throws Exception { new RpcConnectionKey(address, DummyProtocol.class, true); Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, retries + ""); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + ""); client = new BlockingRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams); client.connect(); @@ -412,7 +412,7 @@ public void testUnresolvedAddress2() throws Exception { RpcUtils.createUnresolved(hostAndPort), DummyProtocol.class, false); Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, retries + ""); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + ""); BlockingRpcClient client = new BlockingRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams); @@ -436,7 +436,7 @@ public void testStubRecovery() throws Exception { RpcConnectionKey rpcConnectionKey = new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, String.valueOf(1)); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(1)); BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams); EchoMessage echoMessage = EchoMessage.newBuilder() @@ -470,7 +470,7 @@ public void testIdleTimeout() throws Exception { // 500 millis socket timeout Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, retries + ""); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + ""); connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams); @@ -495,7 +495,7 @@ public void testPingOnIdle() throws Exception { // 500 millis socket timeout Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, retries + ""); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + ""); connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true"); @@ -518,7 +518,7 @@ public void testIdleTimeoutWithActiveRequest() throws Exception { // 500 millis socket timeout Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, retries + ""); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + ""); connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams); @@ -545,7 +545,7 @@ public void testRequestTimeoutOnBusy() throws Exception { // 500 millis socket timeout Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.RPC_RETRY_NUM, retries + ""); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + ""); connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true"); From fcf9cef53fa1b0854e6fd2e21fbf5190c74ead40 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 21 Sep 2015 03:16:56 -0700 Subject: [PATCH 15/24] Updated client v2 to take Properties. --- .../tajo/client/v2/ClientDelegateFactory.java | 5 ++-- .../tajo/client/v2/LegacyClientDelegate.java | 11 +++++--- .../org/apache/tajo/client/v2/TajoClient.java | 27 ++++++++++--------- .../sphinx/table_management/tablespaces.rst | 2 +- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/tajo-client/src/main/java/org/apache/tajo/client/v2/ClientDelegateFactory.java b/tajo-client/src/main/java/org/apache/tajo/client/v2/ClientDelegateFactory.java index 44721b3c3f..cf4e87a633 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/v2/ClientDelegateFactory.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/v2/ClientDelegateFactory.java @@ -22,19 +22,20 @@ import org.apache.tajo.client.v2.exception.ClientUnableToConnectException; import java.util.Map; +import java.util.Properties; public class ClientDelegateFactory { public static ClientDelegate newDefaultDelegate(String host, int port, - @Nullable Map props) + @Nullable Properties props) throws ClientUnableToConnectException { return new LegacyClientDelegate(host, port, props); } public static ClientDelegate newDefaultDelegate(ServiceDiscovery discovery, - @Nullable Map props) + @Nullable Properties props) throws ClientUnableToConnectException { return new LegacyClientDelegate(discovery, props); diff --git a/tajo-client/src/main/java/org/apache/tajo/client/v2/LegacyClientDelegate.java b/tajo-client/src/main/java/org/apache/tajo/client/v2/LegacyClientDelegate.java index 0a2c6defb9..56c0142304 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/v2/LegacyClientDelegate.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/v2/LegacyClientDelegate.java @@ -18,6 +18,7 @@ package org.apache.tajo.client.v2; +import com.google.common.collect.Maps; import com.google.common.util.concurrent.AbstractFuture; import org.apache.tajo.QueryId; import org.apache.tajo.TajoProtos; @@ -42,6 +43,7 @@ import java.sql.ResultSet; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -55,13 +57,14 @@ public class LegacyClientDelegate extends SessionConnection implements ClientDel private QueryClientImpl queryClient; private final ExecutorService executor = Executors.newFixedThreadPool(8); - public LegacyClientDelegate(String host, int port, Map props) { - super(new DummyServiceTracker(NetUtils.createSocketAddr(host, port)), null, new KeyValueSet(props)); + public LegacyClientDelegate(String host, int port, Properties clientParams) { + super(new DummyServiceTracker(NetUtils.createSocketAddr(host, port)), null, + new KeyValueSet(Maps.fromProperties(clientParams))); queryClient = new QueryClientImpl(this); } - public LegacyClientDelegate(ServiceDiscovery discovery, Map props) { - super(new DelegateServiceTracker(discovery), null, new KeyValueSet(props)); + public LegacyClientDelegate(ServiceDiscovery discovery, Properties clientParams) { + super(new DelegateServiceTracker(discovery), null, new KeyValueSet(Maps.fromProperties(clientParams))); queryClient = new QueryClientImpl(this); } diff --git a/tajo-client/src/main/java/org/apache/tajo/client/v2/TajoClient.java b/tajo-client/src/main/java/org/apache/tajo/client/v2/TajoClient.java index 2b4a15042e..dc81742e12 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/v2/TajoClient.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/v2/TajoClient.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.sql.ResultSet; import java.util.Map; +import java.util.Properties; public class TajoClient implements Closeable { private static Log LOG = LogFactory.getLog(TajoClient.class); @@ -52,11 +53,11 @@ public TajoClient(String host) throws ClientUnableToConnectException { /** * Initialize TajoClient with a hostname and default port 26002. * - * @param host Hostname to connect - * @param properties Connection properties + * @param host Hostname to connect + * @param clientParams Client connection parameters */ - public TajoClient(String host, Map properties) throws ClientUnableToConnectException { - delegate = ClientDelegateFactory.newDefaultDelegate(host, DEFAULT_PORT, properties); + public TajoClient(String host, Properties clientParams) throws ClientUnableToConnectException { + delegate = ClientDelegateFactory.newDefaultDelegate(host, DEFAULT_PORT, clientParams); } /** @@ -72,12 +73,12 @@ public TajoClient(String host, int port) throws ClientUnableToConnectException { /** * Initialize TajoClient with a hostname and port * - * @param host Hostname to connect - * @param port Port number to connect - * @param properties Connection properties + * @param host Hostname to connect + * @param port Port number to connect + * @param clientParams Client connection parameters */ - public TajoClient(String host, int port, Map properties) throws ClientUnableToConnectException { - delegate = ClientDelegateFactory.newDefaultDelegate(host, port, properties); + public TajoClient(String host, int port, Properties clientParams) throws ClientUnableToConnectException { + delegate = ClientDelegateFactory.newDefaultDelegate(host, port, clientParams); } /** @@ -92,11 +93,11 @@ public TajoClient(ServiceDiscovery discovery) throws ClientUnableToConnectExcept /** * Initialize TajoClient via service discovery protocol * - * @param discovery Service discovery - * @param properties Connection properties + * @param discovery Service discovery + * @param clientParams Client connection parameters */ - public TajoClient(ServiceDiscovery discovery, Map properties) throws ClientUnableToConnectException { - delegate = ClientDelegateFactory.newDefaultDelegate(discovery, properties); + public TajoClient(ServiceDiscovery discovery, Properties clientParams) throws ClientUnableToConnectException { + delegate = ClientDelegateFactory.newDefaultDelegate(discovery, clientParams); } /** diff --git a/tajo-docs/src/main/sphinx/table_management/tablespaces.rst b/tajo-docs/src/main/sphinx/table_management/tablespaces.rst index 964491c7b9..79ea65f1f5 100644 --- a/tajo-docs/src/main/sphinx/table_management/tablespaces.rst +++ b/tajo-docs/src/main/sphinx/table_management/tablespaces.rst @@ -42,4 +42,4 @@ The following is an example for two tablespaces for hbase and hdfs: .. note:: - Also, each tablespace can use different storage type. Please see :doc:`/storage_plugin` if you want to know more information about it. \ No newline at end of file + Also, each tablespace can use different storage type. Please see :doc:`/storage_plugins` if you want to know more information about it. \ No newline at end of file From e5bc8a679f204f5f9d6afc350ec9c1e7cf9deadf Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 21 Sep 2015 03:27:01 -0700 Subject: [PATCH 16/24] Improve error messages. --- .../org/apache/tajo/jdbc/TestTajoJdbc.java | 43 --------------- .../tajo/jdbc/TestTajoJdbcNegative.java | 53 ++++++++++++++++++- .../org/apache/tajo/rpc/NettyClientBase.java | 17 +++--- 3 files changed, 62 insertions(+), 51 deletions(-) diff --git a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java index a613e91a53..f8d5c4576b 100644 --- a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java +++ b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java @@ -693,47 +693,4 @@ public final void testCancel() throws Exception { } } } - - private void assumeConnectTimeout(String host, int port, int connectTimeout) throws IOException { - try (Socket socket = new Socket()) { - // Try to connect to a private address in the 10.x.y.z range. - // These addresses are usually not routed, so an attempt to - // connect to them will hang the connection attempt, which is - // what we want to simulate in this test. - socket.connect(new InetSocketAddress(host, port), connectTimeout); - // Abort the test if we can connect. - Assume.assumeTrue(false); - } catch (SocketTimeoutException x) { - // Expected timeout during connect, continue the test. - Assume.assumeTrue(true); - } catch (Throwable x) { - // Abort if any other exception happens. - Assume.assumeTrue(false); - } - } - - @Test(timeout = 5000) - public final void testConnectTimeout() throws Exception { - final String host = "10.255.255.1"; - final int port = 80; - int connectTimeout = 1000; - assumeConnectTimeout(host, port, connectTimeout); - - long startTime = Long.MIN_VALUE; - long endTime; - try { - // artificially cause connection timeout - String connUri = buildConnectionUri(host, port, DEFAULT_DATABASE_NAME); - connUri = UriUtil.addParam(connUri, "connectTimeout", "1"); // 1 seconds - startTime = System.currentTimeMillis(); - new JdbcConnection(connUri, new Properties()); - fail("Must be failed"); - } catch (SQLException t) { - endTime = System.currentTimeMillis(); - assertEquals(t.getSQLState(), SQLExceptionUtil.toSQLState(CLIENT_CONNECTION_EXCEPTION)); - // default is 15 seconds. So, if timeout is shorter than 1~2 seconds. - // We can ensure the parameter was effective. - assertTrue(((endTime - startTime) / 1000) < 2); - } - } } diff --git a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbcNegative.java b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbcNegative.java index 8f84226c24..8bf097757b 100644 --- a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbcNegative.java +++ b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbcNegative.java @@ -21,7 +21,10 @@ import org.apache.tajo.IntegrationTest; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.error.Errors.ResultCode; +import org.apache.tajo.exception.SQLExceptionUtil; +import org.apache.tajo.util.UriUtil; import org.junit.AfterClass; +import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -29,9 +32,13 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketTimeoutException; import java.sql.*; +import java.util.Properties; import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; +import static org.apache.tajo.error.Errors.ResultCode.CLIENT_CONNECTION_EXCEPTION; import static org.apache.tajo.exception.SQLExceptionUtil.toSQLState; import static org.apache.tajo.jdbc.TestTajoJdbc.buildConnectionUri; import static org.junit.Assert.*; @@ -53,7 +60,7 @@ public static void tearDown() throws Exception { @Test(expected = SQLException.class) public void testGetConnection() throws SQLException { DriverManager.getConnection("jdbc:taju://" + tajoMasterAddress.getHostName() + ":" + tajoMasterAddress.getPort() - + "/default"); + + "/default"); } @Test @@ -191,4 +198,48 @@ public void testExceptionDuringProcessing() throws Exception { } } } + + private void assumeConnectTimeout(String host, int port, int connectTimeout) throws IOException { + try (Socket socket = new Socket()) { + // Try to connect to a private address in the 10.x.y.z range. + // These addresses are usually not routed, so an attempt to + // connect to them will hang the connection attempt, which is + // what we want to simulate in this test. + socket.connect(new InetSocketAddress(host, port), connectTimeout); + // Abort the test if we can connect. + Assume.assumeTrue(false); + } catch (SocketTimeoutException x) { + // Expected timeout during connect, continue the test. + Assume.assumeTrue(true); + } catch (Throwable x) { + // Abort if any other exception happens. + Assume.assumeTrue(false); + } + } + + @Test(timeout = 5000) + public final void testConnectTimeout() throws Exception { + final String host = "10.255.255.1"; + final int port = 80; + int connectTimeout = 1000; + assumeConnectTimeout(host, port, connectTimeout); + + long startTime = Long.MIN_VALUE; + long endTime; + try { + // artificially cause connection timeout + String connUri = buildConnectionUri(host, port, DEFAULT_DATABASE_NAME); + connUri = UriUtil.addParam(connUri, "connectTimeout", "1"); // 1 seconds + startTime = System.currentTimeMillis(); + new JdbcConnection(connUri, new Properties()); + fail("Must be failed"); + } catch (SQLException t) { + endTime = System.currentTimeMillis(); + assertEquals(t.getSQLState(), SQLExceptionUtil.toSQLState(CLIENT_CONNECTION_EXCEPTION)); + assertEquals("connection timed out: /10.255.255.1:80", t.getMessage()); + // default is 15 seconds. So, if timeout is shorter than 1~2 seconds. + // We can ensure the parameter was effective. + assertTrue(((endTime - startTime) / 1000) < 2); + } + } } diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java index 39340c0033..f29a8f0744 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java @@ -196,6 +196,14 @@ private ChannelFuture doConnect(SocketAddress address) { return this.channelFuture = bootstrap.clone().connect(address); } + private ConnectException makeConnectException(InetSocketAddress address, ChannelFuture future) { + if (future.cause() instanceof UnresolvedAddressException) { + return new ConnectException("Can't resolve host name: " + address.toString()); + } else { + return new ConnectTimeoutException(future.cause().getMessage()); + } + } + public synchronized void connect() throws ConnectException { if (isConnected()) return; @@ -212,7 +220,7 @@ public synchronized void connect() throws ConnectException { if (maxRetryNum > 0) { doReconnect(address, f, ++retries); } else { - throw new ConnectException(ExceptionUtils.getMessage(f.cause())); + throw makeConnectException(address, f); } } } @@ -241,12 +249,7 @@ private void doReconnect(final InetSocketAddress address, ChannelFuture future, } } else { LOG.error("Max retry count has been exceeded. attempts=" + retries + " caused by: " + future.cause()); - - if (future.cause() instanceof UnresolvedAddressException) { - throw new ConnectException("Can't resolve host name: " + address.toString()); - } else { - throw new ConnectTimeoutException(future.cause().getMessage()); - } + throw makeConnectException(address, future); } } } From d0f157b65b3ee02ecc6ea76c7527e4aa50230449 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 21 Sep 2015 04:35:44 -0700 Subject: [PATCH 17/24] Improve TajoCli to take Connection Parameters. --- .../org/apache/tajo/cli/tsql/TajoCli.java | 22 ++++++++++++------- .../tajo/client/v2/LegacyClientDelegate.java | 6 +++-- .../tsql/TestDefaultCliOutputFormatter.java | 2 +- .../org/apache/tajo/cli/tsql/TestTajoCli.java | 16 +++++++++----- .../tajo/cli/tsql/TestTajoCliNegatives.java | 2 +- .../TestExecExternalShellCommand.java | 2 +- .../cli/tsql/commands/TestHdfsCommand.java | 2 +- .../org/apache/tajo/rpc/RpcConstants.java | 2 +- 8 files changed, 33 insertions(+), 21 deletions(-) diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java index 83763e8a2b..8371fa41ac 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java @@ -18,6 +18,7 @@ package org.apache.tajo.cli.tsql; +import com.facebook.presto.hive.shaded.com.google.common.collect.Maps; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.protobuf.ServiceException; @@ -26,6 +27,7 @@ import org.apache.commons.cli.*; import org.apache.tajo.*; import org.apache.tajo.TajoProtos.QueryState; +import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.cli.tsql.ParsedResult.StatementType; import org.apache.tajo.cli.tsql.SimpleParser.ParsingState; @@ -39,16 +41,14 @@ import org.apache.tajo.ipc.ClientProtos; import org.apache.tajo.service.ServiceTrackerFactory; import org.apache.tajo.util.FileUtil; +import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.ShutdownHookManager; import java.io.*; import java.lang.reflect.Constructor; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; +import java.util.*; public class TajoCli { public static final int SHUTDOWN_HOOK_PRIORITY = 50; @@ -176,7 +176,9 @@ public Map getCommands() { } } - public TajoCli(TajoConf c, String [] args, InputStream in, OutputStream out) throws Exception { + public TajoCli(TajoConf c, String [] args, @Nullable Properties connParams, InputStream in, OutputStream out) + throws Exception { + CommandLineParser parser = new PosixParser(); CommandLine cmd = parser.parse(options, args); @@ -234,14 +236,18 @@ public TajoCli(TajoConf c, String [] args, InputStream in, OutputStream out) thr } } + // Get connection parameters + KeyValueSet connectionParameters = + new KeyValueSet(connParams == null ? new HashMap() : Maps.fromProperties(connParams)); + if ((hostName == null) ^ (port == null)) { System.err.println(ERROR_PREFIX + "cannot find valid Tajo server address"); throw new RuntimeException("cannot find valid Tajo server address"); } else if (hostName != null && port != null) { conf.setVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, hostName+":"+port); - client = new TajoClientImpl(ServiceTrackerFactory.get(conf), baseDatabase); + client = new TajoClientImpl(ServiceTrackerFactory.get(conf), baseDatabase, connectionParameters); } else if (hostName == null && port == null) { - client = new TajoClientImpl(ServiceTrackerFactory.get(conf), baseDatabase); + client = new TajoClientImpl(ServiceTrackerFactory.get(conf), baseDatabase, connectionParameters); } try { @@ -694,7 +700,7 @@ public void close() { public static void main(String [] args) throws Exception { TajoConf conf = new TajoConf(); - TajoCli shell = new TajoCli(conf, args, System.in, System.out); + TajoCli shell = new TajoCli(conf, args, new Properties(), System.in, System.out); System.out.println(); System.exit(shell.runShell()); } diff --git a/tajo-client/src/main/java/org/apache/tajo/client/v2/LegacyClientDelegate.java b/tajo-client/src/main/java/org/apache/tajo/client/v2/LegacyClientDelegate.java index 56c0142304..889d974f09 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/v2/LegacyClientDelegate.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/v2/LegacyClientDelegate.java @@ -41,6 +41,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.sql.ResultSet; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -59,12 +60,13 @@ public class LegacyClientDelegate extends SessionConnection implements ClientDel public LegacyClientDelegate(String host, int port, Properties clientParams) { super(new DummyServiceTracker(NetUtils.createSocketAddr(host, port)), null, - new KeyValueSet(Maps.fromProperties(clientParams))); + new KeyValueSet(clientParams == null ? new HashMap() : Maps.fromProperties(clientParams))); queryClient = new QueryClientImpl(this); } public LegacyClientDelegate(ServiceDiscovery discovery, Properties clientParams) { - super(new DelegateServiceTracker(discovery), null, new KeyValueSet(Maps.fromProperties(clientParams))); + super(new DelegateServiceTracker(discovery), null, + new KeyValueSet(clientParams == null ? new HashMap() : Maps.fromProperties(clientParams))); queryClient = new QueryClientImpl(this); } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java index 3b53c60a50..cdcdb6762d 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java @@ -66,7 +66,7 @@ public class TestDefaultCliOutputFormatter { public void setUp() throws Exception { conf = cluster.getConfiguration(); ByteArrayOutputStream out = new ByteArrayOutputStream(); - tajoCli = new TajoCli(conf, new String[]{}, System.in, out); + tajoCli = new TajoCli(conf, new String[]{}, null, System.in, out); cliContext = tajoCli.getContext(); } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index 8ddef0978f..9d13244e63 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -32,6 +32,7 @@ import org.apache.tajo.client.QueryStatus; import org.apache.tajo.client.TajoClient; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.rpc.RpcConstants; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.util.FileUtil; @@ -43,6 +44,7 @@ import java.io.*; import java.net.URL; +import java.util.Properties; import static org.junit.Assert.*; @@ -74,7 +76,9 @@ public TestTajoCli() { @Before public void setUp() throws Exception { out = new ByteArrayOutputStream(); - tajoCli = new TajoCli(cluster.getConfiguration(), new String[]{}, System.in, out); + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, "3"); + tajoCli = new TajoCli(cluster.getConfiguration(), new String[]{}, connParams, System.in, out); } @After @@ -155,7 +159,7 @@ public void testParseConf() throws Exception { assertEquals("tajo.executor.join.inner.in-memory-table-num=256", confValues[1]); TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); - TajoCli testCli = new TajoCli(tajoConf, args, System.in, System.out); + TajoCli testCli = new TajoCli(tajoConf, args, null, System.in, System.out); try { assertEquals("false", testCli.getContext().get(SessionVars.CLI_PAGING_ENABLED)); assertEquals("256", testCli.getContext().getConf().get("tajo.executor.join.inner.in-memory-table-num")); @@ -321,7 +325,7 @@ public void testGetConf() throws Exception { setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); ByteArrayOutputStream out = new ByteArrayOutputStream(); - TajoCli tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out); + TajoCli tajoCli = new TajoCli(tajoConf, new String[]{}, null, System.in, out); try { tajoCli.executeMetaCommand("\\getconf tajo.rootdir"); @@ -338,7 +342,7 @@ public void testShowMasters() throws Exception { setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); ByteArrayOutputStream out = new ByteArrayOutputStream(); - TajoCli tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out); + TajoCli tajoCli = new TajoCli(tajoConf, new String[]{}, null, System.in, out); tajoCli.executeMetaCommand("\\admin -showmasters"); String consoleResult = new String(out.toByteArray()); @@ -372,7 +376,7 @@ public void run() { TajoConf tajoConf = new TajoConf(); setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); - TajoCli tc = new TajoCli(tajoConf, new String[]{}, is, out); + TajoCli tc = new TajoCli(tajoConf, new String[]{}, null, is, out); tc.executeMetaCommand("\\set ON_ERROR_STOP false"); assertSessionVar(tc, SessionVars.ON_ERROR_STOP.keyname(), "false"); @@ -452,7 +456,7 @@ public void testNonForwardQueryPause() throws Exception { assertEquals(0L, tableDesc.getStats().getNumRows().longValue()); InputStream testInput = new ByteArrayInputStream(new byte[]{(byte) DefaultTajoCliOutputFormatter.QUIT_COMMAND}); - cli = new TajoCli(cluster.getConfiguration(), new String[]{}, testInput, out); + cli = new TajoCli(cluster.getConfiguration(), new String[]{}, null, testInput, out); setVar(cli, SessionVars.CLI_PAGE_ROWS, "2"); setVar(cli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCliNegatives.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCliNegatives.java index 689caa750c..11ac14b89a 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCliNegatives.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCliNegatives.java @@ -40,7 +40,7 @@ public class TestTajoCliNegatives extends QueryTestCaseBase { @BeforeClass public static void setUp() throws Exception { out = new ByteArrayOutputStream(); - tajoCli = new TajoCli(testingCluster.getConfiguration(), new String[]{}, System.in, out); + tajoCli = new TajoCli(testingCluster.getConfiguration(), new String[]{}, null, System.in, out); } @AfterClass diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/commands/TestExecExternalShellCommand.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/commands/TestExecExternalShellCommand.java index ed7ee4a7fd..95c3a8bdf8 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/commands/TestExecExternalShellCommand.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/commands/TestExecExternalShellCommand.java @@ -34,7 +34,7 @@ public void testCommand() throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); - TajoCli cli = new TajoCli(tajoConf, new String[]{}, null, out); + TajoCli cli = new TajoCli(tajoConf, new String[]{}, null, null, out); cli.executeMetaCommand("\\! echo \"this is test\""); String consoleResult = new String(out.toByteArray()); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/commands/TestHdfsCommand.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/commands/TestHdfsCommand.java index 496c7e3f78..d239c0ac00 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/commands/TestHdfsCommand.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/commands/TestHdfsCommand.java @@ -37,7 +37,7 @@ public void testHdfCommand() throws Exception { System.setOut(new PrintStream(out)); System.setErr(new PrintStream(out)); - TajoCli cli = new TajoCli(tajoConf, new String[]{}, null, out); + TajoCli cli = new TajoCli(tajoConf, new String[]{}, null, null, out); cli.executeMetaCommand("\\dfs -test"); String consoleResult = new String(out.toByteArray()); diff --git a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java index 601f3d2156..dbbc2e7d73 100644 --- a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java +++ b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java @@ -31,7 +31,7 @@ public class RpcConstants { /** How many times the connect will retry */ public static final String CLIENT_RETRY_NUM = "tajo.rpc.client.retry-num"; - public static final int CLIENT_RETRY_NUM_DEFAULT = 0; + public static final int CLIENT_RETRY_NUM_DEFAULT = 3; /** Client connection timeout (milliseconds) */ public static final String CLIENT_CONNECTION_TIMEOUT = "tajo.rpc.client.connection-timeout-ms"; From 6d11c3babcc94516541e2d109a8d62904d85ba25 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 21 Sep 2015 05:38:46 -0700 Subject: [PATCH 18/24] Fixed unit test failure. --- .../src/test/java/org/apache/tajo/jdbc/TestTajoJdbcNegative.java | 1 + 1 file changed, 1 insertion(+) diff --git a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbcNegative.java b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbcNegative.java index 8bf097757b..1050fc0c61 100644 --- a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbcNegative.java +++ b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbcNegative.java @@ -230,6 +230,7 @@ public final void testConnectTimeout() throws Exception { // artificially cause connection timeout String connUri = buildConnectionUri(host, port, DEFAULT_DATABASE_NAME); connUri = UriUtil.addParam(connUri, "connectTimeout", "1"); // 1 seconds + connUri = UriUtil.addParam(connUri, "retry", "0"); // 1 seconds startTime = System.currentTimeMillis(); new JdbcConnection(connUri, new Properties()); fail("Must be failed"); From 78361bb0e9672a81f17f42530f3b04618cc2d427 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 21 Sep 2015 06:13:11 -0700 Subject: [PATCH 19/24] Changed the default retry num. --- .../src/main/java/org/apache/tajo/rpc/RpcConstants.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java index dbbc2e7d73..601f3d2156 100644 --- a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java +++ b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java @@ -31,7 +31,7 @@ public class RpcConstants { /** How many times the connect will retry */ public static final String CLIENT_RETRY_NUM = "tajo.rpc.client.retry-num"; - public static final int CLIENT_RETRY_NUM_DEFAULT = 3; + public static final int CLIENT_RETRY_NUM_DEFAULT = 0; /** Client connection timeout (milliseconds) */ public static final String CLIENT_CONNECTION_TIMEOUT = "tajo.rpc.client.connection-timeout-ms"; From dde19dea3724bb47734aa6dc8bb47fafe1395b07 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 21 Sep 2015 21:19:54 -0700 Subject: [PATCH 20/24] Fixed some bugs and improved documentation. --- .../apache/tajo/client/ClientParameters.java | 32 +++++++++++++++++++ .../tajo/client/ConnectionPropertyHelper.java | 10 +++--- .../apache/tajo/client/QueryClientImpl.java | 2 +- .../org/apache/tajo/cli/tsql/TestTajoCli.java | 5 ++- .../tajo/util/TestRpcConnectionParamUtil.java | 4 +-- .../apache/tajo/master/QueryInProgress.java | 4 +-- .../querymaster/DefaultTaskScheduler.java | 4 +-- .../apache/tajo/querymaster/QueryMaster.java | 4 +-- .../tajo/querymaster/QueryMasterTask.java | 4 +-- .../org/apache/tajo/querymaster/Stage.java | 4 +-- ...il.java => RpcConnectionParamBuilder.java} | 3 +- .../apache/tajo/worker/NodeStatusUpdater.java | 4 +-- .../org/apache/tajo/worker/TaskManager.java | 4 +-- .../ConnectivityCheckerRuleForTajoWorker.java | 4 +-- 14 files changed, 62 insertions(+), 26 deletions(-) create mode 100644 tajo-client/src/main/java/org/apache/tajo/client/ClientParameters.java rename tajo-core/src/main/java/org/apache/tajo/util/{RpcConnectionParamUtil.java => RpcConnectionParamBuilder.java} (92%) diff --git a/tajo-client/src/main/java/org/apache/tajo/client/ClientParameters.java b/tajo-client/src/main/java/org/apache/tajo/client/ClientParameters.java new file mode 100644 index 0000000000..e3ee019d46 --- /dev/null +++ b/tajo-client/src/main/java/org/apache/tajo/client/ClientParameters.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.client; + +/** + * Client Parameters which can enable or disable some features of TajoClient. + * This class contains the parameter keys. In more detail, + * please refer to http://tajo.apache.org/docs/current/jdbc_driver.html##connection-parameters + */ +public interface ClientParameters { + String USE_COMPRESSION = "useCompression"; + String ROW_FETCH_SIZE = "defaultRowFetchSize"; + String CONNECT_TIMEOUT = "connectTimeout"; + String SOCKET_TIMEOUT = "socketTimeout"; + String RETRY = "retry"; +} diff --git a/tajo-client/src/main/java/org/apache/tajo/client/ConnectionPropertyHelper.java b/tajo-client/src/main/java/org/apache/tajo/client/ConnectionPropertyHelper.java index 467db41c33..fe17373add 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/ConnectionPropertyHelper.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/ConnectionPropertyHelper.java @@ -57,9 +57,9 @@ class ConnectionPropertyHelper { public static Map PARAMETERS = new HashMap<>(); static { - PARAMETERS.put("useCompression", new SimpleSessionAction(COMPRESSED_RESULT_TRANSFER)); - PARAMETERS.put("defaultRowFetchSize", new SimpleSessionAction(FETCH_ROWNUM)); - PARAMETERS.put("connectTimeout", new ConnectionParamAction() { + PARAMETERS.put(ClientParameters.USE_COMPRESSION, new SimpleSessionAction(COMPRESSED_RESULT_TRANSFER)); + PARAMETERS.put(ClientParameters.ROW_FETCH_SIZE, new SimpleSessionAction(FETCH_ROWNUM)); + PARAMETERS.put(ClientParameters.CONNECT_TIMEOUT, new ConnectionParamAction() { @Override Pair doAction(String param) { int seconds = Integer.parseInt(param); @@ -67,14 +67,14 @@ Pair doAction(String param) { return new Pair<>(CLIENT_CONNECTION_TIMEOUT, String.valueOf(TimeUnit.SECONDS.toMillis(seconds))); } }); - PARAMETERS.put("socketTimeout", new ConnectionParamAction() { + PARAMETERS.put(ClientParameters.SOCKET_TIMEOUT, new ConnectionParamAction() { @Override Pair doAction(String param) { int seconds = Integer.parseInt(param); return new Pair<>(CLIENT_SOCKET_TIMEOUT, String.valueOf(TimeUnit.SECONDS.toMillis(seconds))); } }); - PARAMETERS.put("retry", new SimpleConnectionParamAction(RpcConstants.CLIENT_RETRY_NUM)); + PARAMETERS.put(ClientParameters.RETRY, new SimpleConnectionParamAction(RpcConstants.CLIENT_RETRY_NUM)); } enum ActionType { diff --git a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java index c929d159a9..ae99bb0863 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java @@ -353,7 +353,7 @@ public void run() { protected TajoMemoryResultSet fetchNextQueryResult(final QueryId queryId, final int fetchRowNum) throws TajoException { - boolean compress = conn.getProperties().getBool(SessionVars.COMPRESSED_RESULT_TRANSFER); + boolean compress = conn.getProperties().getBool("useCompression"); final BlockingInterface stub = conn.getTMStub(); final GetQueryResultDataRequest.Builder request = GetQueryResultDataRequest.newBuilder(); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index 9d13244e63..da51aedd9b 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -29,6 +29,7 @@ import org.apache.tajo.TpchTestBase; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.client.ClientParameters; import org.apache.tajo.client.QueryStatus; import org.apache.tajo.client.TajoClient; import org.apache.tajo.conf.TajoConf; @@ -376,7 +377,9 @@ public void run() { TajoConf tajoConf = new TajoConf(); setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); - TajoCli tc = new TajoCli(tajoConf, new String[]{}, null, is, out); + Properties connParams = new Properties(); + connParams.setProperty(ClientParameters.RETRY, "3"); + TajoCli tc = new TajoCli(tajoConf, new String[]{}, connParams, is, out); tc.executeMetaCommand("\\set ON_ERROR_STOP false"); assertSessionVar(tc, SessionVars.ON_ERROR_STOP.keyname(), "false"); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcConnectionParamUtil.java b/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcConnectionParamUtil.java index 1a506fdf21..18151931da 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcConnectionParamUtil.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcConnectionParamUtil.java @@ -34,7 +34,7 @@ public class TestRpcConnectionParamUtil { @Test public void testGetDefaults() throws Exception { TajoConf conf = new TajoConf(); - Properties defaultParams = RpcConnectionParamUtil.get(conf); + Properties defaultParams = RpcConnectionParamBuilder.get(conf); assertEquals( ConfVars.RPC_CLIENT_RETRY_NUM.defaultVal, defaultParams.getProperty(CLIENT_RETRY_NUM)); assertEquals( @@ -50,7 +50,7 @@ public void testGet() throws Exception { conf.setLongVar(ConfVars.RPC_CLIENT_CONNECTION_TIMEOUT, (long)(10 * 1000)); conf.setLongVar(ConfVars.RPC_CLIENT_SOCKET_TIMEOUT, (long)60 * 1000); - Properties defaultParams = RpcConnectionParamUtil.get(conf); + Properties defaultParams = RpcConnectionParamBuilder.get(conf); assertEquals("100", defaultParams.getProperty(CLIENT_RETRY_NUM)); assertEquals(10 * 1000 + "", defaultParams.getProperty(CLIENT_CONNECTION_TIMEOUT)); assertEquals(60 * 1000 + "", defaultParams.getProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT)); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java b/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java index a24165a78b..f4b4950d3b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java @@ -39,7 +39,7 @@ import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.session.Session; import org.apache.tajo.util.NetUtils; -import org.apache.tajo.util.RpcConnectionParamUtil; +import org.apache.tajo.util.RpcConnectionParamBuilder; import java.net.ConnectException; import java.net.InetSocketAddress; @@ -91,7 +91,7 @@ public QueryInProgress( queryInfo = new QueryInfo(queryId, queryContext, sql, jsonExpr); queryInfo.setStartTime(System.currentTimeMillis()); - rpcClientParams = RpcConnectionParamUtil.get(masterContext.getConf()); + rpcClientParams = RpcConnectionParamBuilder.get(masterContext.getConf()); ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java index fc9b9554e9..e23a6b4880 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java @@ -48,7 +48,7 @@ import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.NetUtils; -import org.apache.tajo.util.RpcConnectionParamUtil; +import org.apache.tajo.util.RpcConnectionParamBuilder; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.FetchImpl; @@ -97,7 +97,7 @@ public DefaultTaskScheduler(TaskSchedulerContext context, Stage stage) { @Override public void init(Configuration conf) { tajoConf = TUtil.checkTypeAndGet(conf, TajoConf.class); - rpcConnParams = RpcConnectionParamUtil.get(new TajoConf()); + rpcConnParams = RpcConnectionParamBuilder.get(new TajoConf()); scheduledRequests = new ScheduledRequests(); minTaskMemory = tajoConf.getIntVar(TajoConf.ConfVars.TASK_RESOURCE_MINIMUM_MEMORY); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java index 3d2ca75bd6..e6c2aba13b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java @@ -45,7 +45,7 @@ import org.apache.tajo.rpc.*; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.service.ServiceTracker; -import org.apache.tajo.util.RpcConnectionParamUtil; +import org.apache.tajo.util.RpcConnectionParamBuilder; import org.apache.tajo.util.TUtil; import org.apache.tajo.util.history.HistoryWriter.WriterFuture; import org.apache.tajo.util.history.HistoryWriter.WriterHolder; @@ -105,7 +105,7 @@ public void serviceInit(Configuration conf) throws Exception { this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class); this.manager = RpcClientManager.getInstance(); - this.rpcClientParams = RpcConnectionParamUtil.get(this.systemConf); + this.rpcClientParams = RpcConnectionParamBuilder.get(this.systemConf); querySessionTimeout = systemConf.getIntVar(TajoConf.ConfVars.QUERY_SESSION_TIMEOUT); queryMasterContext = new QueryMasterContext(systemConf); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java index dbaef59102..1b38293956 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java @@ -55,7 +55,7 @@ import org.apache.tajo.storage.FormatProperty; import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.TablespaceManager; -import org.apache.tajo.util.RpcConnectionParamUtil; +import org.apache.tajo.util.RpcConnectionParamBuilder; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.event.NodeResourceDeallocateEvent; import org.apache.tajo.worker.event.NodeResourceEvent; @@ -135,7 +135,7 @@ public QueryMasterTask(QueryMaster.QueryMasterContext queryMasterContext, @Override public void serviceInit(Configuration conf) throws Exception { systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class); - rpcConnParams = RpcConnectionParamUtil.get(systemConf); + rpcConnParams = RpcConnectionParamBuilder.get(systemConf); queryTaskContext = new QueryMasterTaskContext(); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index b3c16b3e8a..96bcde8479 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -62,7 +62,7 @@ import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.KeyValueSet; -import org.apache.tajo.util.RpcConnectionParamUtil; +import org.apache.tajo.util.RpcConnectionParamBuilder; import org.apache.tajo.util.TUtil; import org.apache.tajo.util.history.StageHistory; import org.apache.tajo.util.history.TaskHistory; @@ -303,7 +303,7 @@ public Stage(QueryMasterTask.QueryMasterTaskContext context, MasterPlan masterPl this.block = block; this.eventHandler = context.getEventHandler(); - this.rpcClientParams = RpcConnectionParamUtil.get(context.getConf()); + this.rpcClientParams = RpcConnectionParamBuilder.get(context.getConf()); ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); diff --git a/tajo-core/src/main/java/org/apache/tajo/util/RpcConnectionParamUtil.java b/tajo-core/src/main/java/org/apache/tajo/util/RpcConnectionParamBuilder.java similarity index 92% rename from tajo-core/src/main/java/org/apache/tajo/util/RpcConnectionParamUtil.java rename to tajo-core/src/main/java/org/apache/tajo/util/RpcConnectionParamBuilder.java index 3dff61fe55..4232dc30de 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/RpcConnectionParamUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/RpcConnectionParamBuilder.java @@ -28,7 +28,7 @@ /** * Helper class to get RPC Client Connection Parameters */ -public class RpcConnectionParamUtil { +public class RpcConnectionParamBuilder { static final Map PROPERTIES_MAP = new HashMap<>(); @@ -36,6 +36,7 @@ public class RpcConnectionParamUtil { PROPERTIES_MAP.put(RpcConstants.CLIENT_RETRY_NUM, ConfVars.RPC_CLIENT_RETRY_NUM); PROPERTIES_MAP.put(RpcConstants.CLIENT_CONNECTION_TIMEOUT, ConfVars.RPC_CLIENT_CONNECTION_TIMEOUT); PROPERTIES_MAP.put(RpcConstants.CLIENT_SOCKET_TIMEOUT, ConfVars.RPC_CLIENT_SOCKET_TIMEOUT); + PROPERTIES_MAP.put(RpcConstants.CLIENT_HANG_DETECTION, ConfVars.RPC_CLIENT_HANG_DETECTION_ENABLED); } public static Properties get(TajoConf conf) { diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java b/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java index 3dcb6abe7e..48ff8d031c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java @@ -35,7 +35,7 @@ import org.apache.tajo.rpc.RpcConstants; import org.apache.tajo.service.ServiceTracker; import org.apache.tajo.service.ServiceTrackerFactory; -import org.apache.tajo.util.RpcConnectionParamUtil; +import org.apache.tajo.util.RpcConnectionParamBuilder; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.event.NodeStatusEvent; @@ -79,7 +79,7 @@ public NodeStatusUpdater(TajoWorker.WorkerContext workerContext) { public void serviceInit(Configuration conf) throws Exception { this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class); - this.rpcClientParams = RpcConnectionParamUtil.get(this.systemConf); + this.rpcClientParams = RpcConnectionParamBuilder.get(this.systemConf); this.heartBeatRequestQueue = Queues.newLinkedBlockingQueue(); this.serviceTracker = ServiceTrackerFactory.get(systemConf); this.workerContext.getNodeResourceManager().getDispatcher().register(NodeStatusEvent.EventType.class, this); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java index c3b8f0f097..927d0e6320 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java @@ -35,7 +35,7 @@ import org.apache.tajo.rpc.RpcClientManager; import org.apache.tajo.rpc.RpcConstants; import org.apache.tajo.util.NetUtils; -import org.apache.tajo.util.RpcConnectionParamUtil; +import org.apache.tajo.util.RpcConnectionParamBuilder; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.event.*; @@ -71,7 +71,7 @@ public TaskManager(Dispatcher dispatcher, TajoWorker.WorkerContext workerContext this.workerContext = workerContext; this.executionBlockContextMap = Maps.newHashMap(); this.executor = executor; - this.rpcClientParams = RpcConnectionParamUtil.get(this.workerContext.getConf()); + this.rpcClientParams = RpcConnectionParamBuilder.get(this.workerContext.getConf()); } @Override diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java index 56ed8df4f2..e1b2fd6ff9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java @@ -26,7 +26,7 @@ import org.apache.tajo.rule.EvaluationResult.EvaluationResultCode; import org.apache.tajo.service.ServiceTracker; import org.apache.tajo.service.ServiceTrackerFactory; -import org.apache.tajo.util.RpcConnectionParamUtil; +import org.apache.tajo.util.RpcConnectionParamBuilder; import org.apache.tajo.worker.TajoWorker; /** @@ -42,7 +42,7 @@ private void checkTajoMasterConnectivity(TajoConf tajoConf) throws Exception { ServiceTracker serviceTracker = ServiceTrackerFactory.get(tajoConf); NettyClientBase masterClient = manager.getClient(serviceTracker.getUmbilicalAddress(), - QueryCoordinatorProtocol.class, true, RpcConnectionParamUtil.get(tajoConf)); + QueryCoordinatorProtocol.class, true, RpcConnectionParamBuilder.get(tajoConf)); masterClient.getStub(); } From ff22c0e12d7ec31d1beeef93e199d5a2ea00788a Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 21 Sep 2015 22:02:57 -0700 Subject: [PATCH 21/24] Renamed rpc params into connection params. --- .../apache/tajo/catalog/CatalogClient.java | 6 ++--- .../org/apache/tajo/cli/tsql/TajoCli.java | 12 +++++----- ...Helper.java => ClientParameterHelper.java} | 6 ++--- .../apache/tajo/client/SessionConnection.java | 8 +++---- .../tajo/client/v2/ClientDelegateFactory.java | 9 ++++---- ...aramUtil.java => TestRpcParamFactory.java} | 6 ++--- .../apache/tajo/master/QueryInProgress.java | 8 +++---- .../querymaster/DefaultTaskScheduler.java | 12 +++++----- .../apache/tajo/querymaster/QueryMaster.java | 4 ++-- .../tajo/querymaster/QueryMasterTask.java | 10 ++++----- .../org/apache/tajo/querymaster/Stage.java | 8 +++---- ...mBuilder.java => RpcParameterFactory.java} | 2 +- .../apache/tajo/worker/NodeStatusUpdater.java | 8 +++---- .../org/apache/tajo/worker/TaskManager.java | 8 +++---- .../ConnectivityCheckerRuleForTajoWorker.java | 4 ++-- .../org/apache/tajo/rpc/AsyncRpcClient.java | 10 ++++----- .../apache/tajo/rpc/BlockingRpcClient.java | 10 ++++----- .../org/apache/tajo/rpc/NettyClientBase.java | 8 +++---- .../rpc/ProtoClientChannelInitializer.java | 6 ++--- .../org/apache/tajo/rpc/RpcClientManager.java | 22 +++++++++---------- 20 files changed, 83 insertions(+), 84 deletions(-) rename tajo-client/src/main/java/org/apache/tajo/client/{ConnectionPropertyHelper.java => ClientParameterHelper.java} (96%) rename tajo-core-tests/src/test/java/org/apache/tajo/util/{TestRpcConnectionParamUtil.java => TestRpcParamFactory.java} (92%) rename tajo-core/src/main/java/org/apache/tajo/util/{RpcConnectionParamBuilder.java => RpcParameterFactory.java} (97%) diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java index c59058eb5b..2b24a6ba9a 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java @@ -69,12 +69,12 @@ public synchronized NettyClientBase getCatalogConnection() throws ServiceExcepti if (client != null && client.isConnected()) return client; RpcClientManager.cleanup(client); - final Properties connParams = new Properties(); - connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, conf.getVar(ConfVars.RPC_CLIENT_RETRY_NUM)); + final Properties clientParams = new Properties(); + clientParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, conf.getVar(ConfVars.RPC_CLIENT_RETRY_NUM)); // Client do not closed on idle state for support high available this.client = RpcClientManager.getInstance().newClient(getCatalogServerAddr(), CatalogProtocol.class, - false, connParams); + false, clientParams); } catch (Exception e) { throw new ServiceException(e); } diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java index 8371fa41ac..edc7c2bed1 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java @@ -18,7 +18,7 @@ package org.apache.tajo.cli.tsql; -import com.facebook.presto.hive.shaded.com.google.common.collect.Maps; +import com.google.common.collect.Maps; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.protobuf.ServiceException; @@ -176,7 +176,7 @@ public Map getCommands() { } } - public TajoCli(TajoConf c, String [] args, @Nullable Properties connParams, InputStream in, OutputStream out) + public TajoCli(TajoConf c, String [] args, @Nullable Properties clientParams, InputStream in, OutputStream out) throws Exception { CommandLineParser parser = new PosixParser(); @@ -237,17 +237,17 @@ public TajoCli(TajoConf c, String [] args, @Nullable Properties connParams, Inpu } // Get connection parameters - KeyValueSet connectionParameters = - new KeyValueSet(connParams == null ? new HashMap() : Maps.fromProperties(connParams)); + final KeyValueSet actualConnParams = + new KeyValueSet(clientParams == null ? new HashMap() : Maps.fromProperties(clientParams)); if ((hostName == null) ^ (port == null)) { System.err.println(ERROR_PREFIX + "cannot find valid Tajo server address"); throw new RuntimeException("cannot find valid Tajo server address"); } else if (hostName != null && port != null) { conf.setVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, hostName+":"+port); - client = new TajoClientImpl(ServiceTrackerFactory.get(conf), baseDatabase, connectionParameters); + client = new TajoClientImpl(ServiceTrackerFactory.get(conf), baseDatabase, actualConnParams); } else if (hostName == null && port == null) { - client = new TajoClientImpl(ServiceTrackerFactory.get(conf), baseDatabase, connectionParameters); + client = new TajoClientImpl(ServiceTrackerFactory.get(conf), baseDatabase, actualConnParams); } try { diff --git a/tajo-client/src/main/java/org/apache/tajo/client/ConnectionPropertyHelper.java b/tajo-client/src/main/java/org/apache/tajo/client/ClientParameterHelper.java similarity index 96% rename from tajo-client/src/main/java/org/apache/tajo/client/ConnectionPropertyHelper.java rename to tajo-client/src/main/java/org/apache/tajo/client/ClientParameterHelper.java index fe17373add..8549178d2a 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/ConnectionPropertyHelper.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/ClientParameterHelper.java @@ -30,8 +30,8 @@ import static org.apache.tajo.SessionVars.COMPRESSED_RESULT_TRANSFER; import static org.apache.tajo.SessionVars.FETCH_ROWNUM; -import static org.apache.tajo.client.ConnectionPropertyHelper.ActionType.CONNECTION_PARAM; -import static org.apache.tajo.client.ConnectionPropertyHelper.ActionType.SESSION_UPDATE; +import static org.apache.tajo.client.ClientParameterHelper.ActionType.CONNECTION_PARAM; +import static org.apache.tajo.client.ClientParameterHelper.ActionType.SESSION_UPDATE; import static org.apache.tajo.rpc.RpcConstants.CLIENT_CONNECTION_TIMEOUT; import static org.apache.tajo.rpc.RpcConstants.CLIENT_SOCKET_TIMEOUT; @@ -52,7 +52,7 @@ * against some network or connection problems. It determines how many times the connection will retry. * */ -class ConnectionPropertyHelper { +class ClientParameterHelper { public static Map PARAMETERS = new HashMap<>(); diff --git a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java index 1688235e69..f63bb47483 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java @@ -81,7 +81,7 @@ public class SessionConnection implements Closeable { private NettyClientBase client; - private Properties clientConnParams; + private Properties clientParams; private final KeyValueSet properties; @@ -103,7 +103,7 @@ public SessionConnection(@NotNull ServiceTracker tracker, @Nullable String baseD this.manager = RpcClientManager.getInstance(); this.userInfo = UserRoleInfo.getCurrentUser(); // update the connection parameters to RPC client from connection properties - this.clientConnParams = ConnectionPropertyHelper.getConnParams(properties.getAllKeyValus().entrySet()); + this.clientParams = ClientParameterHelper.getConnParams(properties.getAllKeyValus().entrySet()); this.eventLoopGroup = NettyUtils.createEventLoopGroup(getClass().getSimpleName(), 4); try { @@ -114,7 +114,7 @@ public SessionConnection(@NotNull ServiceTracker tracker, @Nullable String baseD } // update the session variables from connection parameters - updateSessionVariables(ConnectionPropertyHelper.getSessionVars(properties.getAllKeyValus().entrySet())); + updateSessionVariables(ClientParameterHelper.getSessionVars(properties.getAllKeyValus().entrySet())); } public Map getClientSideSessionVars() { @@ -132,7 +132,7 @@ public synchronized NettyClientBase getTajoMasterConnection() { // Client do not closed on idle state for support high available this.client = manager.newBlockingClient(getTajoMasterAddr(), TajoMasterClientProtocol.class, - eventLoopGroup, clientConnParams); + eventLoopGroup, clientParams); } catch (Throwable t) { throw new TajoRuntimeException(new ClientConnectionException(t)); } diff --git a/tajo-client/src/main/java/org/apache/tajo/client/v2/ClientDelegateFactory.java b/tajo-client/src/main/java/org/apache/tajo/client/v2/ClientDelegateFactory.java index cf4e87a633..feee37be29 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/v2/ClientDelegateFactory.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/v2/ClientDelegateFactory.java @@ -21,23 +21,22 @@ import org.apache.tajo.annotation.Nullable; import org.apache.tajo.client.v2.exception.ClientUnableToConnectException; -import java.util.Map; import java.util.Properties; public class ClientDelegateFactory { public static ClientDelegate newDefaultDelegate(String host, int port, - @Nullable Properties props) + @Nullable Properties clientParams) throws ClientUnableToConnectException { - return new LegacyClientDelegate(host, port, props); + return new LegacyClientDelegate(host, port, clientParams); } public static ClientDelegate newDefaultDelegate(ServiceDiscovery discovery, - @Nullable Properties props) + @Nullable Properties clientParams) throws ClientUnableToConnectException { - return new LegacyClientDelegate(discovery, props); + return new LegacyClientDelegate(discovery, clientParams); } } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcConnectionParamUtil.java b/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcParamFactory.java similarity index 92% rename from tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcConnectionParamUtil.java rename to tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcParamFactory.java index 18151931da..1d63bba714 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcConnectionParamUtil.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcParamFactory.java @@ -29,12 +29,12 @@ import static org.apache.tajo.rpc.RpcConstants.CLIENT_RETRY_NUM; import static org.junit.Assert.*; -public class TestRpcConnectionParamUtil { +public class TestRpcParamFactory { @Test public void testGetDefaults() throws Exception { TajoConf conf = new TajoConf(); - Properties defaultParams = RpcConnectionParamBuilder.get(conf); + Properties defaultParams = RpcParameterFactory.get(conf); assertEquals( ConfVars.RPC_CLIENT_RETRY_NUM.defaultVal, defaultParams.getProperty(CLIENT_RETRY_NUM)); assertEquals( @@ -50,7 +50,7 @@ public void testGet() throws Exception { conf.setLongVar(ConfVars.RPC_CLIENT_CONNECTION_TIMEOUT, (long)(10 * 1000)); conf.setLongVar(ConfVars.RPC_CLIENT_SOCKET_TIMEOUT, (long)60 * 1000); - Properties defaultParams = RpcConnectionParamBuilder.get(conf); + Properties defaultParams = RpcParameterFactory.get(conf); assertEquals("100", defaultParams.getProperty(CLIENT_RETRY_NUM)); assertEquals(10 * 1000 + "", defaultParams.getProperty(CLIENT_CONNECTION_TIMEOUT)); assertEquals(60 * 1000 + "", defaultParams.getProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT)); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java b/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java index f4b4950d3b..8e999c335b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java @@ -39,7 +39,7 @@ import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.session.Session; import org.apache.tajo.util.NetUtils; -import org.apache.tajo.util.RpcConnectionParamBuilder; +import org.apache.tajo.util.RpcParameterFactory; import java.net.ConnectException; import java.net.InetSocketAddress; @@ -72,7 +72,7 @@ public class QueryInProgress { private AllocationResourceProto allocationResource; - private final Properties rpcClientParams; + private final Properties rpcParams; private final Lock readLock; private final Lock writeLock; @@ -91,7 +91,7 @@ public QueryInProgress( queryInfo = new QueryInfo(queryId, queryContext, sql, jsonExpr); queryInfo.setStartTime(System.currentTimeMillis()); - rpcClientParams = RpcConnectionParamBuilder.get(masterContext.getConf()); + rpcParams = RpcParameterFactory.get(masterContext.getConf()); ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); @@ -188,7 +188,7 @@ private void connectQueryMaster(WorkerConnectionInfo connectionInfo) InetSocketAddress addr = NetUtils.createSocketAddr(connectionInfo.getHost(), connectionInfo.getQueryMasterPort()); LOG.info("Try to connect to QueryMaster:" + addr); - queryMasterRpc = RpcClientManager.getInstance().newClient(addr, QueryMasterProtocol.class, true, rpcClientParams); + queryMasterRpc = RpcClientManager.getInstance().newClient(addr, QueryMasterProtocol.class, true, rpcParams); queryMasterRpcClient = queryMasterRpc.getStub(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java index e23a6b4880..25695a599a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java @@ -48,7 +48,7 @@ import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.NetUtils; -import org.apache.tajo.util.RpcConnectionParamBuilder; +import org.apache.tajo.util.RpcParameterFactory; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.FetchImpl; @@ -70,7 +70,7 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler { private final TaskSchedulerContext context; private Stage stage; private TajoConf tajoConf; - private Properties rpcConnParams; + private Properties rpcParams; private Thread schedulingThread; private volatile boolean isStopped; @@ -97,7 +97,7 @@ public DefaultTaskScheduler(TaskSchedulerContext context, Stage stage) { @Override public void init(Configuration conf) { tajoConf = TUtil.checkTypeAndGet(conf, TajoConf.class); - rpcConnParams = RpcConnectionParamBuilder.get(new TajoConf()); + rpcParams = RpcParameterFactory.get(new TajoConf()); scheduledRequests = new ScheduledRequests(); minTaskMemory = tajoConf.getIntVar(TajoConf.ConfVars.TASK_RESOURCE_MINIMUM_MEMORY); @@ -298,7 +298,7 @@ protected LinkedList createTaskRequest(final int incompleteTas ServiceTracker serviceTracker = context.getMasterContext().getQueryMasterContext().getWorkerContext().getServiceTracker(); NettyClientBase tmClient = RpcClientManager.getInstance(). - getClient(serviceTracker.getUmbilicalAddress(), QueryCoordinatorProtocol.class, true, rpcConnParams); + getClient(serviceTracker.getUmbilicalAddress(), QueryCoordinatorProtocol.class, true, rpcParams); QueryCoordinatorProtocolService masterClientService = tmClient.getStub(); CallFuture callBack = new CallFuture(); @@ -891,7 +891,7 @@ public void assignToLeafTasks(LinkedList taskRequests) { totalAttempts++; try { tajoWorkerRpc = RpcClientManager.getInstance().getClient(addr, TajoWorkerProtocol.class, true, - rpcConnParams); + rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); tajoWorkerRpcClient.allocateTasks(callFuture.getController(), requestProto.build(), callFuture); @@ -1011,7 +1011,7 @@ public void assignToNonLeafTasks(LinkedList taskRequests) { AsyncRpcClient tajoWorkerRpc; try { tajoWorkerRpc = RpcClientManager.getInstance().getClient(addr, TajoWorkerProtocol.class, true, - rpcConnParams); + rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); tajoWorkerRpcClient.allocateTasks(callFuture.getController(), requestProto.build(), callFuture); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java index e6c2aba13b..1b900809ae 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java @@ -45,7 +45,7 @@ import org.apache.tajo.rpc.*; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.service.ServiceTracker; -import org.apache.tajo.util.RpcConnectionParamBuilder; +import org.apache.tajo.util.RpcParameterFactory; import org.apache.tajo.util.TUtil; import org.apache.tajo.util.history.HistoryWriter.WriterFuture; import org.apache.tajo.util.history.HistoryWriter.WriterHolder; @@ -105,7 +105,7 @@ public void serviceInit(Configuration conf) throws Exception { this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class); this.manager = RpcClientManager.getInstance(); - this.rpcClientParams = RpcConnectionParamBuilder.get(this.systemConf); + this.rpcClientParams = RpcParameterFactory.get(this.systemConf); querySessionTimeout = systemConf.getIntVar(TajoConf.ConfVars.QUERY_SESSION_TIMEOUT); queryMasterContext = new QueryMasterContext(systemConf); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java index 1b38293956..3fa98ef4b5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java @@ -55,7 +55,7 @@ import org.apache.tajo.storage.FormatProperty; import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.TablespaceManager; -import org.apache.tajo.util.RpcConnectionParamBuilder; +import org.apache.tajo.util.RpcParameterFactory; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.event.NodeResourceDeallocateEvent; import org.apache.tajo.worker.event.NodeResourceEvent; @@ -96,7 +96,7 @@ public class QueryMasterTask extends CompositeService { private TajoConf systemConf; - private Properties rpcConnParams; + private Properties rpcParams; private AtomicLong lastClientHeartbeat = new AtomicLong(-1); @@ -135,7 +135,7 @@ public QueryMasterTask(QueryMaster.QueryMasterContext queryMasterContext, @Override public void serviceInit(Configuration conf) throws Exception { systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class); - rpcConnParams = RpcConnectionParamBuilder.get(systemConf); + rpcParams = RpcParameterFactory.get(systemConf); queryTaskContext = new QueryMasterTaskContext(); @@ -259,7 +259,7 @@ protected void killTaskAttempt(int workerId, TaskAttemptId taskAttemptId) { try { tajoWorkerRpc = RpcClientManager.getInstance().getClient(workerAddress, TajoWorkerProtocol.class, true, - rpcConnParams); + rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); CallFuture callFuture = new CallFuture(); tajoWorkerRpcClient.killTaskAttempt(null, taskAttemptId.getProto(), callFuture); @@ -477,7 +477,7 @@ private void cleanupQuery(final QueryId queryId) { public void run() { try { AsyncRpcClient rpc = RpcClientManager.getInstance().getClient(worker, TajoWorkerProtocol.class, true, - rpcConnParams); + rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerProtocolService = rpc.getStub(); tajoWorkerProtocolService.stopQuery(null, queryId.getProto(), NullCallback.get()); } catch (Throwable e) { diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index 96bcde8479..068e1a5648 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -62,7 +62,7 @@ import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.KeyValueSet; -import org.apache.tajo.util.RpcConnectionParamBuilder; +import org.apache.tajo.util.RpcParameterFactory; import org.apache.tajo.util.TUtil; import org.apache.tajo.util.history.StageHistory; import org.apache.tajo.util.history.TaskHistory; @@ -89,7 +89,7 @@ public class Stage implements EventHandler { private static final Log LOG = LogFactory.getLog(Stage.class); - private final Properties rpcClientParams; + private final Properties rpcParams; private MasterPlan masterPlan; private ExecutionBlock block; @@ -303,7 +303,7 @@ public Stage(QueryMasterTask.QueryMasterTaskContext context, MasterPlan masterPl this.block = block; this.eventHandler = context.getEventHandler(); - this.rpcClientParams = RpcConnectionParamBuilder.get(context.getConf()); + this.rpcParams = RpcParameterFactory.get(context.getConf()); ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); @@ -725,7 +725,7 @@ private void sendStopExecutionBlockEvent(final StopExecutionBlockRequest request public void run() { try { AsyncRpcClient tajoWorkerRpc = - RpcClientManager.getInstance().getClient(worker, TajoWorkerProtocol.class, true, rpcClientParams); + RpcClientManager.getInstance().getClient(worker, TajoWorkerProtocol.class, true, rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); tajoWorkerRpcClient.stopExecutionBlock(null, requestProto, NullCallback.get(PrimitiveProtos.BoolProto.class)); diff --git a/tajo-core/src/main/java/org/apache/tajo/util/RpcConnectionParamBuilder.java b/tajo-core/src/main/java/org/apache/tajo/util/RpcParameterFactory.java similarity index 97% rename from tajo-core/src/main/java/org/apache/tajo/util/RpcConnectionParamBuilder.java rename to tajo-core/src/main/java/org/apache/tajo/util/RpcParameterFactory.java index 4232dc30de..6da4dac459 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/RpcConnectionParamBuilder.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/RpcParameterFactory.java @@ -28,7 +28,7 @@ /** * Helper class to get RPC Client Connection Parameters */ -public class RpcConnectionParamBuilder { +public class RpcParameterFactory { static final Map PROPERTIES_MAP = new HashMap<>(); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java b/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java index 48ff8d031c..a3b71e1cc0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java @@ -35,7 +35,7 @@ import org.apache.tajo.rpc.RpcConstants; import org.apache.tajo.service.ServiceTracker; import org.apache.tajo.service.ServiceTrackerFactory; -import org.apache.tajo.util.RpcConnectionParamBuilder; +import org.apache.tajo.util.RpcParameterFactory; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.event.NodeStatusEvent; @@ -65,7 +65,7 @@ public class NodeStatusUpdater extends AbstractService implements EventHandler heartBeatRequestQueue; private final TajoWorker.WorkerContext workerContext; private AsyncRpcClient rmClient; - private Properties rpcClientParams; + private Properties rpcParams; private ServiceTracker serviceTracker; private TajoResourceTrackerProtocol.TajoResourceTrackerProtocolService.Interface resourceTracker; private int queueingThreshold; @@ -79,7 +79,7 @@ public NodeStatusUpdater(TajoWorker.WorkerContext workerContext) { public void serviceInit(Configuration conf) throws Exception { this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class); - this.rpcClientParams = RpcConnectionParamBuilder.get(this.systemConf); + this.rpcParams = RpcParameterFactory.get(this.systemConf); this.heartBeatRequestQueue = Queues.newLinkedBlockingQueue(); this.serviceTracker = ServiceTrackerFactory.get(systemConf); this.workerContext.getNodeResourceManager().getDispatcher().register(NodeStatusEvent.EventType.class, this); @@ -153,7 +153,7 @@ protected TajoResourceTrackerProtocol.TajoResourceTrackerProtocolService.Interfa RpcClientManager rpcManager = RpcClientManager.getInstance(); rmClient = rpcManager.newClient(serviceTracker.getResourceTrackerAddress(), - TajoResourceTrackerProtocol.class, true, rpcClientParams); + TajoResourceTrackerProtocol.class, true, rpcParams); return rmClient.getStub(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java index 927d0e6320..a0b3f97f17 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java @@ -35,7 +35,7 @@ import org.apache.tajo.rpc.RpcClientManager; import org.apache.tajo.rpc.RpcConstants; import org.apache.tajo.util.NetUtils; -import org.apache.tajo.util.RpcConnectionParamBuilder; +import org.apache.tajo.util.RpcParameterFactory; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.event.*; @@ -58,7 +58,7 @@ public class TaskManager extends AbstractService implements EventHandler executionBlockContextMap; private final Dispatcher dispatcher; private TaskExecutor executor; - private final Properties rpcClientParams; + private final Properties rpcParams; public TaskManager(Dispatcher dispatcher, TajoWorker.WorkerContext workerContext){ this(dispatcher, workerContext, null); @@ -71,7 +71,7 @@ public TaskManager(Dispatcher dispatcher, TajoWorker.WorkerContext workerContext this.workerContext = workerContext; this.executionBlockContextMap = Maps.newHashMap(); this.executor = executor; - this.rpcClientParams = RpcConnectionParamBuilder.get(this.workerContext.getConf()); + this.rpcParams = RpcParameterFactory.get(this.workerContext.getConf()); } @Override @@ -120,7 +120,7 @@ protected ExecutionBlockContext createExecutionBlock(ExecutionBlockId executionB request.setExecutionBlockId(executionBlockId.getProto()) .setWorker(getWorkerContext().getConnectionInfo().getProto()); - client = RpcClientManager.getInstance().newClient(address, QueryMasterProtocol.class, true, rpcClientParams); + client = RpcClientManager.getInstance().newClient(address, QueryMasterProtocol.class, true, rpcParams); QueryMasterProtocol.QueryMasterProtocolService.Interface stub = client.getStub(); CallFuture callback = new CallFuture(); stub.getExecutionBlockContext(callback.getController(), request.build(), callback); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java index e1b2fd6ff9..ee428cd6f5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java @@ -26,7 +26,7 @@ import org.apache.tajo.rule.EvaluationResult.EvaluationResultCode; import org.apache.tajo.service.ServiceTracker; import org.apache.tajo.service.ServiceTrackerFactory; -import org.apache.tajo.util.RpcConnectionParamBuilder; +import org.apache.tajo.util.RpcParameterFactory; import org.apache.tajo.worker.TajoWorker; /** @@ -42,7 +42,7 @@ private void checkTajoMasterConnectivity(TajoConf tajoConf) throws Exception { ServiceTracker serviceTracker = ServiceTrackerFactory.get(tajoConf); NettyClientBase masterClient = manager.getClient(serviceTracker.getUmbilicalAddress(), - QueryCoordinatorProtocol.class, true, RpcConnectionParamBuilder.get(tajoConf)); + QueryCoordinatorProtocol.class, true, RpcParameterFactory.get(tajoConf)); masterClient.getStub(); } diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java index 4b33e7e935..c613bace99 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java @@ -43,27 +43,27 @@ public class AsyncRpcClient extends NettyClientBase implements ProtoDeclaration, Closeable * Constructor of NettyClientBase * * @param rpcConnectionKey RpcConnectionKey - * @param connectionParameters Connection Parameters (see RpcConstants) + * @param rpcParams Rpc connection parameters (see RpcConstants) * * @throws ClassNotFoundException * @throws NoSuchMethodException * @see RpcConstants */ - public NettyClientBase(RpcConnectionKey rpcConnectionKey, Properties connectionParameters) + public NettyClientBase(RpcConnectionKey rpcConnectionKey, Properties rpcParams) throws ClassNotFoundException, NoSuchMethodException { this.key = rpcConnectionKey; this.maxRetryNum = Integer.parseInt( - connectionParameters.getProperty(CLIENT_RETRY_NUM, String.valueOf(CLIENT_RETRY_NUM_DEFAULT))); + rpcParams.getProperty(CLIENT_RETRY_NUM, String.valueOf(CLIENT_RETRY_NUM_DEFAULT))); this.connTimeoutMillis = Integer.parseInt( - connectionParameters.getProperty(CLIENT_CONNECTION_TIMEOUT, String.valueOf(CLIENT_CONNECTION_TIMEOUT_DEFAULT))); + rpcParams.getProperty(CLIENT_CONNECTION_TIMEOUT, String.valueOf(CLIENT_CONNECTION_TIMEOUT_DEFAULT))); // Netty only takes integer value range and this is to avoid integer overflow. Preconditions.checkArgument(this.connTimeoutMillis <= Integer.MAX_VALUE, "Too long connection timeout"); diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java index 396be36b4c..5d544cb9ef 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java @@ -35,7 +35,7 @@ class ProtoClientChannelInitializer extends ChannelInitializer { private final MessageLite defaultInstance; private final ChannelHandler handler; private final long idleTimeout; - private final boolean usePing; + private final boolean hangDetection; /** * Channel Pipe Initializer @@ -49,7 +49,7 @@ public ProtoClientChannelInitializer(ChannelHandler handler, MessageLite default this.handler = handler; this.defaultInstance = defaultInstance; this.idleTimeout = idleTimeout; - this.usePing = hangDetection; + this.hangDetection = hangDetection; } @Override @@ -58,7 +58,7 @@ protected void initChannel(Channel channel) throws Exception { pipeline.addLast("idleStateHandler", new IdleStateHandler(idleTimeout, idleTimeout / 2, 0, TimeUnit.MILLISECONDS)); - if (usePing) { + if (hangDetection) { pipeline.addLast("MonitorClientHandler", new MonitorClientHandler()); } pipeline.addLast("frameDecoder", new ProtobufVarint32FrameDecoder()); diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java index aa9e3c7360..032cf355e8 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java @@ -58,23 +58,23 @@ public static RpcClientManager getInstance() { } private T makeClient(RpcConnectionKey rpcConnectionKey, - Properties connectionParameters) + Properties rpcParams) throws NoSuchMethodException, ConnectException, ClassNotFoundException { - return makeClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connectionParameters); + return makeClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, rpcParams); } private T makeClient(EventLoopGroup eventLoopGroup, RpcConnectionKey rpcConnectionKey, - Properties connectionParameters) + Properties rpcParams) throws NoSuchMethodException, ClassNotFoundException, ConnectException { NettyClientBase client; if (rpcConnectionKey.asyncMode) { - client = new AsyncRpcClient(eventLoopGroup, rpcConnectionKey, connectionParameters); + client = new AsyncRpcClient(eventLoopGroup, rpcConnectionKey, rpcParams); } else { - client = new BlockingRpcClient(eventLoopGroup, rpcConnectionKey, connectionParameters); + client = new BlockingRpcClient(eventLoopGroup, rpcConnectionKey, rpcParams); } return (T) client; } @@ -86,7 +86,7 @@ private T makeClient(EventLoopGroup eventLoopGroup, public T getClient(InetSocketAddress addr, Class protocolClass, boolean asyncMode, - Properties connectionParameters) + Properties rpcParams) throws NoSuchMethodException, ClassNotFoundException, ConnectException { RpcConnectionKey key = new RpcConnectionKey(addr, protocolClass, asyncMode); @@ -94,7 +94,7 @@ public T getClient(InetSocketAddress addr, synchronized (clients) { client = clients.get(key); if (client == null) { - clients.put(key, client = makeClient(key, connectionParameters)); + clients.put(key, client = makeClient(key, rpcParams)); } } @@ -128,10 +128,10 @@ public void channelUnregistered(ChannelHandlerContext ctx) { public T newClient(InetSocketAddress addr, Class protocolClass, boolean asyncMode, - Properties connectionParameters) + Properties rpcParams) throws NoSuchMethodException, ClassNotFoundException, ConnectException { - return newClient(new RpcConnectionKey(addr, protocolClass, asyncMode), connectionParameters); + return newClient(new RpcConnectionKey(addr, protocolClass, asyncMode), rpcParams); } /** @@ -157,10 +157,10 @@ public synchronized T newClient(RpcConnectionKey key public synchronized T newBlockingClient(InetSocketAddress addr, Class protocolClass, EventLoopGroup eventLoopGroup, - Properties connectionParameters) + Properties rpcParams) throws NoSuchMethodException, ClassNotFoundException, ConnectException { - T client = makeClient(eventLoopGroup, new RpcConnectionKey(addr, protocolClass, false), connectionParameters); + T client = makeClient(eventLoopGroup, new RpcConnectionKey(addr, protocolClass, false), rpcParams); client.connect(); assert client.isConnected(); return client; From 12df2c1440bc5d70e8ed1378f4c72326cb27426b Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 21 Sep 2015 22:48:13 -0700 Subject: [PATCH 22/24] Changed the literal to constant. --- .../src/main/java/org/apache/tajo/client/QueryClientImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java index ae99bb0863..2e542e0111 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java @@ -353,7 +353,7 @@ public void run() { protected TajoMemoryResultSet fetchNextQueryResult(final QueryId queryId, final int fetchRowNum) throws TajoException { - boolean compress = conn.getProperties().getBool("useCompression"); + boolean compress = conn.getProperties().getBool(ClientParameters.USE_COMPRESSION); final BlockingInterface stub = conn.getTMStub(); final GetQueryResultDataRequest.Builder request = GetQueryResultDataRequest.newBuilder(); From 64105b09bdf215eefdb3029a02b2454628041ce2 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 22 Sep 2015 02:37:24 -0700 Subject: [PATCH 23/24] Add default connection params to TajoCli. --- .../tajo/cli/tsql/CliClientParamsFactory.java | 50 +++++++++++++++++++ .../org/apache/tajo/cli/tsql/TajoCli.java | 4 +- .../apache/tajo/client/QueryClientImpl.java | 2 +- 3 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java new file mode 100644 index 0000000000..56cc13daf6 --- /dev/null +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.cli.tsql; + +import org.apache.tajo.annotation.Nullable; +import org.apache.tajo.client.ClientParameters; + +import javax.validation.constraints.Null; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +class CliClientParamsFactory { + static Map DEFAULT_PARAMS = new HashMap<>(); + + static { + // Keep lexicographic order of parameter names. + DEFAULT_PARAMS.put(ClientParameters.CONNECT_TIMEOUT, "0"); + DEFAULT_PARAMS.put(ClientParameters.SOCKET_TIMEOUT, "0"); + DEFAULT_PARAMS.put(ClientParameters.RETRY, "3"); + DEFAULT_PARAMS.put(ClientParameters.ROW_FETCH_SIZE, "200"); + DEFAULT_PARAMS.put(ClientParameters.USE_COMPRESSION, "false"); + } + + public static Properties get(@Nullable Properties connParam) { + Properties copy = (Properties) connParam.clone(); + for (Map.Entry entry : DEFAULT_PARAMS.entrySet()) { + if (!copy.contains(entry.getKey())) { + copy.setProperty(entry.getKey(), entry.getValue()); + } + } + return copy; + } +} diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java index edc7c2bed1..8ae7075771 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java @@ -237,8 +237,8 @@ public TajoCli(TajoConf c, String [] args, @Nullable Properties clientParams, In } // Get connection parameters - final KeyValueSet actualConnParams = - new KeyValueSet(clientParams == null ? new HashMap() : Maps.fromProperties(clientParams)); + Properties defaultConnParams = CliClientParamsFactory.get(clientParams); + final KeyValueSet actualConnParams = new KeyValueSet(Maps.fromProperties(defaultConnParams)); if ((hostName == null) ^ (port == null)) { System.err.println(ERROR_PREFIX + "cannot find valid Tajo server address"); diff --git a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java index 2e542e0111..364292e21c 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java @@ -353,7 +353,7 @@ public void run() { protected TajoMemoryResultSet fetchNextQueryResult(final QueryId queryId, final int fetchRowNum) throws TajoException { - boolean compress = conn.getProperties().getBool(ClientParameters.USE_COMPRESSION); + final boolean compress = conn.getProperties().getBool(ClientParameters.USE_COMPRESSION); final BlockingInterface stub = conn.getTMStub(); final GetQueryResultDataRequest.Builder request = GetQueryResultDataRequest.newBuilder(); From 036d5dfc9d60fa3e2e6d0dee22e2a2592ddfb85e Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 22 Sep 2015 02:41:19 -0700 Subject: [PATCH 24/24] Fixed NPE. --- .../java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java index 56cc13daf6..5fabae8505 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java @@ -39,7 +39,7 @@ class CliClientParamsFactory { } public static Properties get(@Nullable Properties connParam) { - Properties copy = (Properties) connParam.clone(); + Properties copy = connParam == null ? new Properties() : (Properties) connParam.clone(); for (Map.Entry entry : DEFAULT_PARAMS.entrySet()) { if (!copy.contains(entry.getKey())) { copy.setProperty(entry.getKey(), entry.getValue());