From 1e131d494dac1d1a66703ec4b2a5fc256aa9ce6f Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 7 Apr 2015 18:53:30 +0900 Subject: [PATCH 01/11] TAJO-1526 --- .../engine/planner/physical/TestExternalSortExec.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java index 946e0f30bf..0ef8294f25 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java @@ -61,7 +61,7 @@ public class TestExternalSortExec { private LogicalPlanner planner; private Path testDir; - private final int numTuple = 3000000; + private final int numTuple = 100000; private Random rnd = new Random(System.currentTimeMillis()); private TableDesc employee; @@ -136,13 +136,17 @@ public final void testNext() throws IOException, PlanningException { ProjectionExec proj = (ProjectionExec) exec; // TODO - should be planed with user's optimization hint + ExternalSortExec extSort; if (!(proj.getChild() instanceof ExternalSortExec)) { UnaryPhysicalExec sortExec = proj.getChild(); SeqScanExec scan = sortExec.getChild(); - ExternalSortExec extSort = new ExternalSortExec(ctx, ((MemSortExec)sortExec).getPlan(), scan); + extSort = new ExternalSortExec(ctx, ((MemSortExec)sortExec).getPlan(), scan); proj.setChild(extSort); + } else { + extSort = proj.getChild(); } + extSort.setSortBufferBytesNum(1024*1024); Tuple tuple; Tuple preVal = null; From f37fe3f0eb969dd6b0bbeb7c64458e97ec1ea168 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 7 Apr 2015 20:25:12 +0900 Subject: [PATCH 02/11] TAJO-1526 --- tajo-core/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml index 61a156b668..4f36a41b9b 100644 --- a/tajo-core/pom.xml +++ b/tajo-core/pom.xml @@ -91,7 +91,7 @@ TRUE - -Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 + -Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 @@ -781,7 +781,7 @@ ${maven.fork.count} true - -Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 + -Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 true true From 2aa692885c9199883d67b45e8cb709bccc81fc8c Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 7 Apr 2015 20:57:28 +0900 Subject: [PATCH 03/11] Trigger Travis again --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index d59a205390..dd1463b21a 100644 --- a/CHANGES +++ b/CHANGES @@ -74,6 +74,7 @@ Release 0.11.0 - unreleased BUG FIXES + TAJO-1510: Change a function name from getFileCunks to getFileChunks. (Contributed by Soonwoong Lee, Committed by jaehwa) From 5df477634e74bb0ccfb7de6dd7f892915216ec2b Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 7 Apr 2015 23:43:21 +0900 Subject: [PATCH 04/11] Trigger Travis --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index 523fb45edb..f6c764df06 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.11.0 - unreleased NEW FEATURES From 076e5a5dd9ac62c80267a4d0374004a077142f34 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 7 Apr 2015 23:46:25 +0900 Subject: [PATCH 05/11] investigate jvm permSize after remove jersey warning --- .../rs/netty/NettyRestHandlerContainer.java | 49 +++++++------------ .../ws/rs/netty/NettyRestServerListener.java | 6 +-- 2 files changed, 20 insertions(+), 35 deletions(-) diff --git a/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestHandlerContainer.java b/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestHandlerContainer.java index 81d1eeb759..236d3a8b70 100644 --- a/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestHandlerContainer.java +++ b/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestHandlerContainer.java @@ -18,53 +18,37 @@ package org.apache.tajo.ws.rs.netty; -import java.io.OutputStream; -import java.net.InetSocketAddress; -import java.net.URI; -import java.net.URISyntaxException; -import java.security.Principal; -import java.util.List; -import java.util.Map.Entry; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - import io.netty.buffer.*; -import io.netty.channel.ChannelDuplexHandler; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; +import io.netty.channel.*; import io.netty.channel.ChannelHandler.Sharable; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelPromise; -import io.netty.handler.codec.http.DefaultFullHttpResponse; -import io.netty.handler.codec.http.FullHttpRequest; -import io.netty.handler.codec.http.FullHttpResponse; -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpResponseStatus; -import io.netty.handler.codec.http.HttpVersion; -import io.netty.handler.codec.http.LastHttpContent; +import io.netty.handler.codec.http.*; import io.netty.handler.ssl.SslHandler; import io.netty.util.CharsetUtil; import io.netty.util.ReferenceCountUtil; import io.netty.util.concurrent.GenericFutureListener; - -import javax.ws.rs.core.Application; -import javax.ws.rs.core.MultivaluedMap; -import javax.ws.rs.core.SecurityContext; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.glassfish.hk2.api.ServiceLocator; import org.glassfish.jersey.internal.MapPropertiesDelegate; -import org.glassfish.jersey.server.ApplicationHandler; -import org.glassfish.jersey.server.ContainerException; -import org.glassfish.jersey.server.ContainerRequest; -import org.glassfish.jersey.server.ContainerResponse; -import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.server.*; import org.glassfish.jersey.server.internal.ConfigHelper; import org.glassfish.jersey.server.spi.Container; import org.glassfish.jersey.server.spi.ContainerLifecycleListener; import org.glassfish.jersey.server.spi.ContainerResponseWriter; +import javax.ws.rs.core.Application; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.SecurityContext; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.Principal; +import java.util.List; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + /** * Jersy Container implementation on Netty */ @@ -89,6 +73,7 @@ public class NettyRestHandlerContainer extends ChannelDuplexHandler implements C NettyRestHandlerContainer(ApplicationHandler appHandler) { applicationHandler = appHandler; lifecycleListener = ConfigHelper.getContainerLifecycleListener(applicationHandler); + lifecycleListener.onStartup(this); } @Override diff --git a/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestServerListener.java b/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestServerListener.java index ecd5bb0f92..2bd62d22ce 100644 --- a/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestServerListener.java +++ b/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestServerListener.java @@ -49,9 +49,9 @@ public void onAfterShutdown(Object obj) { @Override public void onAfterStart(Object obj) { - ApplicationHandler applicationHandler = new ApplicationHandler(container.getConfiguration()); - ContainerLifecycleListener lifecycleListener = ConfigHelper.getContainerLifecycleListener(applicationHandler); - lifecycleListener.onStartup(container); +// ApplicationHandler applicationHandler = new ApplicationHandler(container.getConfiguration()); +// ContainerLifecycleListener lifecycleListener = ConfigHelper.getContainerLifecycleListener(container.); +// lifecycleListener.onStartup(container); } @Override From 701457cf0fe96cb416ef9d62da4ed0563f3441a8 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 8 Apr 2015 10:13:56 +0900 Subject: [PATCH 06/11] Trigger Travis CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index 523fb45edb..f6c764df06 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.11.0 - unreleased NEW FEATURES From 7368d0fb2f9050c8794c5fb73ea71f8bc427d5d0 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 8 Apr 2015 13:10:28 +0900 Subject: [PATCH 07/11] Merged with Jinho's work --- .../org/apache/tajo/ws/rs/netty/NettyRestServerListener.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestServerListener.java b/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestServerListener.java index 2bd62d22ce..90e18cf782 100644 --- a/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestServerListener.java +++ b/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestServerListener.java @@ -49,9 +49,6 @@ public void onAfterShutdown(Object obj) { @Override public void onAfterStart(Object obj) { -// ApplicationHandler applicationHandler = new ApplicationHandler(container.getConfiguration()); -// ContainerLifecycleListener lifecycleListener = ConfigHelper.getContainerLifecycleListener(container.); -// lifecycleListener.onStartup(container); } @Override From 9e34ad0df1860fa4e7939fb6d85702f2d2507d3a Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 8 Apr 2015 13:43:12 +0900 Subject: [PATCH 08/11] Without Jinho's patch --- .../org/apache/tajo/ws/rs/netty/NettyRestHandlerContainer.java | 1 - .../org/apache/tajo/ws/rs/netty/NettyRestServerListener.java | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestHandlerContainer.java b/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestHandlerContainer.java index 236d3a8b70..76b81c39f8 100644 --- a/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestHandlerContainer.java +++ b/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestHandlerContainer.java @@ -73,7 +73,6 @@ public class NettyRestHandlerContainer extends ChannelDuplexHandler implements C NettyRestHandlerContainer(ApplicationHandler appHandler) { applicationHandler = appHandler; lifecycleListener = ConfigHelper.getContainerLifecycleListener(applicationHandler); - lifecycleListener.onStartup(this); } @Override diff --git a/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestServerListener.java b/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestServerListener.java index 90e18cf782..ecd5bb0f92 100644 --- a/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestServerListener.java +++ b/tajo-rpc/tajo-ws-rs/src/main/java/org/apache/tajo/ws/rs/netty/NettyRestServerListener.java @@ -49,6 +49,9 @@ public void onAfterShutdown(Object obj) { @Override public void onAfterStart(Object obj) { + ApplicationHandler applicationHandler = new ApplicationHandler(container.getConfiguration()); + ContainerLifecycleListener lifecycleListener = ConfigHelper.getContainerLifecycleListener(applicationHandler); + lifecycleListener.onStartup(container); } @Override From 0be68eddcd674849575ead2a47c5f7af4b82d145 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 8 Apr 2015 14:55:00 +0900 Subject: [PATCH 09/11] Trigger Travis --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index ddcb7f1249..fba55e9b43 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.11.0 - unreleased NEW FEATURES From aee09d05f3ca3c5e26df51841aaf3fc53a51ad9f Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 8 Apr 2015 15:47:15 +0900 Subject: [PATCH 10/11] Trigger Travis --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index fba55e9b43..ddcb7f1249 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.11.0 - unreleased NEW FEATURES From 2e9e4beb409203d83166b570a114672f268f67cb Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 8 Apr 2015 16:43:08 +0900 Subject: [PATCH 11/11] Trigger Travis --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index ddcb7f1249..fba55e9b43 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.11.0 - unreleased NEW FEATURES