From 525f97f28582133a24e5c2f67722f23d95ac05d2 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 11:25:51 +0900 Subject: [PATCH 01/83] decrease Xms size for python process fork --- tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml | 2 +- tajo-catalog/tajo-catalog-server/pom.xml | 2 +- tajo-cli/pom.xml | 2 +- tajo-client-example/pom.xml | 2 +- tajo-client/pom.xml | 2 +- tajo-cluster-tests/pom.xml | 2 +- tajo-core-tests/pom.xml | 4 ++-- tajo-jdbc/pom.xml | 2 +- tajo-storage/tajo-storage-common/pom.xml | 2 +- tajo-storage/tajo-storage-hbase/pom.xml | 2 +- tajo-storage/tajo-storage-hdfs/pom.xml | 2 +- tajo-storage/tajo-storage-jdbc/pom.xml | 2 +- tajo-storage/tajo-storage-pgsql/pom.xml | 2 +- 13 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml b/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml index b2995a8a34..22d14bffa9 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml @@ -82,7 +82,7 @@ true - -Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null diff --git a/tajo-catalog/tajo-catalog-server/pom.xml b/tajo-catalog/tajo-catalog-server/pom.xml index cc421a8491..bb4d9abb78 100644 --- a/tajo-catalog/tajo-catalog-server/pom.xml +++ b/tajo-catalog/tajo-catalog-server/pom.xml @@ -120,7 +120,7 @@ true - -Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null diff --git a/tajo-cli/pom.xml b/tajo-cli/pom.xml index a7a3b91068..e670529f63 100644 --- a/tajo-cli/pom.xml +++ b/tajo-cli/pom.xml @@ -71,7 +71,7 @@ true - -Xms512m -Xmx1024m -Dfile.encoding=UTF-8 + -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 diff --git a/tajo-client-example/pom.xml b/tajo-client-example/pom.xml index d531f570d8..daee2f9892 100644 --- a/tajo-client-example/pom.xml +++ b/tajo-client-example/pom.xml @@ -74,7 +74,7 @@ true - -Xms512m -Xmx1024m -Dfile.encoding=UTF-8 + -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 diff --git a/tajo-client/pom.xml b/tajo-client/pom.xml index 015cbc76fb..90fcec6a30 100644 --- a/tajo-client/pom.xml +++ b/tajo-client/pom.xml @@ -75,7 +75,7 @@ true - -Xms512m -Xmx1024m -Dfile.encoding=UTF-8 + -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 diff --git a/tajo-cluster-tests/pom.xml b/tajo-cluster-tests/pom.xml index 0aafa27f81..4bf506bc66 100644 --- a/tajo-cluster-tests/pom.xml +++ b/tajo-cluster-tests/pom.xml @@ -67,7 +67,7 @@ true - -Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null diff --git a/tajo-core-tests/pom.xml b/tajo-core-tests/pom.xml index 6d3d75c76e..10e7de9278 100644 --- a/tajo-core-tests/pom.xml +++ b/tajo-core-tests/pom.xml @@ -73,7 +73,7 @@ true - -Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null @@ -375,7 +375,7 @@ ${maven.fork.count} true false - -Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null true true diff --git a/tajo-jdbc/pom.xml b/tajo-jdbc/pom.xml index 1b852d18d3..57e850d3c7 100644 --- a/tajo-jdbc/pom.xml +++ b/tajo-jdbc/pom.xml @@ -99,7 +99,7 @@ true - -Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null diff --git a/tajo-storage/tajo-storage-common/pom.xml b/tajo-storage/tajo-storage-common/pom.xml index 106d9f3d61..9d4f9d8384 100644 --- a/tajo-storage/tajo-storage-common/pom.xml +++ b/tajo-storage/tajo-storage-common/pom.xml @@ -75,7 +75,7 @@ limitations under the License. TRUE - -Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null diff --git a/tajo-storage/tajo-storage-hbase/pom.xml b/tajo-storage/tajo-storage-hbase/pom.xml index 1adfa10027..3ebd1ea59f 100644 --- a/tajo-storage/tajo-storage-hbase/pom.xml +++ b/tajo-storage/tajo-storage-hbase/pom.xml @@ -78,7 +78,7 @@ TRUE - -Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null diff --git a/tajo-storage/tajo-storage-hdfs/pom.xml b/tajo-storage/tajo-storage-hdfs/pom.xml index 3d69492521..24e54fd92e 100644 --- a/tajo-storage/tajo-storage-hdfs/pom.xml +++ b/tajo-storage/tajo-storage-hdfs/pom.xml @@ -81,7 +81,7 @@ TRUE - -Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null diff --git a/tajo-storage/tajo-storage-jdbc/pom.xml b/tajo-storage/tajo-storage-jdbc/pom.xml index f225c72638..cac02e3ba5 100644 --- a/tajo-storage/tajo-storage-jdbc/pom.xml +++ b/tajo-storage/tajo-storage-jdbc/pom.xml @@ -77,7 +77,7 @@ TRUE - -Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 + -Xms128m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 diff --git a/tajo-storage/tajo-storage-pgsql/pom.xml b/tajo-storage/tajo-storage-pgsql/pom.xml index 6f04420c90..0cb5d2bd3c 100644 --- a/tajo-storage/tajo-storage-pgsql/pom.xml +++ b/tajo-storage/tajo-storage-pgsql/pom.xml @@ -234,7 +234,7 @@ TRUE - -Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 + -Xms128m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 From b8ac09d5f8175d28f457b79a56d09454b2a0c55a Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 11:30:47 +0900 Subject: [PATCH 02/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index f3dfd6808a..9a0cabbd5a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From 165a0c820f5caee054d7029949f9c1d7f3ca4276 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 11:31:17 +0900 Subject: [PATCH 03/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index 9a0cabbd5a..f3dfd6808a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From bfd4b7866c5432a8261c66acc9180e3ed2a617c9 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 11:31:44 +0900 Subject: [PATCH 04/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index f3dfd6808a..9a0cabbd5a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From 701f1348998be99696b2d12848d07d6f73b2ca17 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 11:32:07 +0900 Subject: [PATCH 05/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index 9a0cabbd5a..f3dfd6808a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From f2b8b20bac2a622a1fe0db5cd654f088d41559db Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 12:37:30 +0900 Subject: [PATCH 06/83] disable python tests --- .../java/org/apache/tajo/engine/query/TestGroupByQuery.java | 4 ++-- .../java/org/apache/tajo/engine/query/TestSelectQuery.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java index cc9028bfbb..1052a5cc75 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java @@ -773,7 +773,7 @@ public final void testGroupbyWithLimit3() throws Exception { cleanupQuery(res); } - @Test + /*@Test @Option(sort = true) @SimpleTest public final void testGroupbyWithPythonFunc() throws Exception { @@ -821,7 +821,7 @@ public final void testPythonUdafWithNullData() throws Exception { ResultSet res = executeQuery(); assertResultSet(res); cleanupQuery(res); - } + }*/ // TODO: this test cannot be executed due to the bug of logical planner (TAJO-1588) // @Test diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index e55acf1fc0..0adb4ebcb7 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -750,7 +750,7 @@ public void testMultiBytesDelimiter4() throws Exception { } } - @Test + /*@Test public void testSelectPythonFuncs() throws Exception { ResultSet res = executeQuery(); assertResultSet(res); @@ -769,7 +769,7 @@ public void testNestedPythonFunction() throws Exception { ResultSet res = executeQuery(); assertResultSet(res); cleanupQuery(res); - } + }*/ @Test public void testSelectWithParentheses1() throws Exception { From d85e1a45d61a25f3b7574528a5324eb45ad7d133 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 12:38:36 +0900 Subject: [PATCH 07/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index f3dfd6808a..9a0cabbd5a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From 0e0c92c5298e6bc797c1f8c806c0b5d725d9ed38 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 12:38:55 +0900 Subject: [PATCH 08/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index 9a0cabbd5a..f3dfd6808a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From 0b2bafac46594f32cc719c614d217a9237b9e35c Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 12:39:15 +0900 Subject: [PATCH 09/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index f3dfd6808a..9a0cabbd5a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From 1b43233a0b6568fce147a36eb06d789bfe5818a2 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 12:39:35 +0900 Subject: [PATCH 10/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index 9a0cabbd5a..f3dfd6808a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From b00d5e9e3fbf302fdc0eb60f31e4519324e96a04 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 16:33:35 +0900 Subject: [PATCH 11/83] decrease hbase handler and memory cache --- .../org/apache/tajo/TajoTestingCluster.java | 17 +++++++++++++---- .../tajo/engine/query/TestGroupByQuery.java | 4 ++-- .../tajo/engine/query/TestSelectQuery.java | 4 ++-- .../apache/tajo/storage/TablespaceManager.java | 11 +++++++++-- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index a4fff574e2..76565d2f7b 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -160,6 +160,18 @@ void initPropertiesAndConfigs() { // Query output file conf.setVar(ConfVars.QUERY_OUTPUT_DEFAULT_FILE_FORMAT, BuiltinStorages.DRAW); + /** decrease Hbase thread and memory cache for testing */ + //server handler + conf.setInt("hbase.regionserver.handler.count", 5); + //client handler + conf.setInt("hbase.hconnection.threads.core", 5); + conf.setInt("hbase.hconnection.threads.max", 10); + conf.setInt("hbase.hconnection.meta.lookup.threads.core", 5); + conf.setInt("hbase.hconnection.meta.lookup.threads.max", 10); + //memory cache + conf.setFloat("hfile.block.cache.size", 0.1f); + conf.setInt("hbase.bucketcache.size", 10); + /* Since Travis CI limits the size of standard output log up to 4MB */ if (!StringUtils.isEmpty(LOG_LEVEL)) { Level defaultLevel = Logger.getRootLogger().getLevel(); @@ -489,7 +501,7 @@ public void startMiniCluster(final int numSlaves) public void startMiniCluster(final int numSlaves, final String [] dataNodeHosts) throws Exception { int numDataNodes = numSlaves; - if(dataNodeHosts != null && dataNodeHosts.length != 0) { + if (dataNodeHosts != null && dataNodeHosts.length != 0) { numDataNodes = dataNodeHosts.length; } @@ -504,9 +516,6 @@ public void startMiniCluster(final int numSlaves, final String [] dataNodeHosts) startMiniDFSCluster(numDataNodes, clusterTestBuildDir, dataNodeHosts); this.dfsCluster.waitClusterUp(); - - conf.setInt("hbase.hconnection.threads.core", 5); - conf.setInt("hbase.hconnection.threads.max", 50); hbaseUtil = new HBaseTestClusterUtil(conf, clusterTestBuildDir); startMiniTajoCluster(this.clusterTestBuildDir, numSlaves, false); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java index 1052a5cc75..cc9028bfbb 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java @@ -773,7 +773,7 @@ public final void testGroupbyWithLimit3() throws Exception { cleanupQuery(res); } - /*@Test + @Test @Option(sort = true) @SimpleTest public final void testGroupbyWithPythonFunc() throws Exception { @@ -821,7 +821,7 @@ public final void testPythonUdafWithNullData() throws Exception { ResultSet res = executeQuery(); assertResultSet(res); cleanupQuery(res); - }*/ + } // TODO: this test cannot be executed due to the bug of logical planner (TAJO-1588) // @Test diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index 0adb4ebcb7..e55acf1fc0 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -750,7 +750,7 @@ public void testMultiBytesDelimiter4() throws Exception { } } - /*@Test + @Test public void testSelectPythonFuncs() throws Exception { ResultSet res = executeQuery(); assertResultSet(res); @@ -769,7 +769,7 @@ public void testNestedPythonFunction() throws Exception { ResultSet res = executeQuery(); assertResultSet(res); cleanupQuery(res); - }*/ + } @Test public void testSelectWithParentheses1() throws Exception { diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java index 569f1a2989..da487e0d00 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java @@ -210,7 +210,7 @@ public static void AddTableSpace(String spaceName, URI uri, boolean isDefault, J registerTableSpace(spaceName, uri, configs, true, override); } - private static void registerTableSpace(String spaceName, URI uri, JSONObject spaceDesc, + private static Tablespace registerTableSpace(String spaceName, URI uri, JSONObject spaceDesc, boolean visible, boolean override) { Tablespace tableSpace = initializeTableSpace(spaceName, uri, spaceDesc); tableSpace.setVisible(visible); @@ -232,6 +232,7 @@ private static void registerTableSpace(String spaceName, URI uri, JSONObject spa registerTableSpace(tmpName, rootUri, spaceDesc, false, override); } } + return tableSpace; } private static void putTablespace(Tablespace space, boolean override) { @@ -313,7 +314,13 @@ public static Optional addTableSpaceForTest(Tablespace space) { existing = TABLE_SPACES.remove(space.getUri()); // Add anotherone for test - registerTableSpace(space.name, space.uri, space.getConfig(), true, true); + Tablespace tablespace = registerTableSpace(space.name, space.uri, space.getConfig(), true, true); + try { + if (space.conf != null) + tablespace.init(space.conf); + } catch (IOException e) { + throw new RuntimeException(e); + } } // if there is an existing one, return it. return Optional.ofNullable(existing); From cd207b2247f85947ca5981d4c0df8a18259a268d Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 16:34:36 +0900 Subject: [PATCH 12/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index 426a44adfb..cea19b0737 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From d9d6d362dea8abd6f585a267f7e94fb40315bead Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 16:34:54 +0900 Subject: [PATCH 13/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index cea19b0737..426a44adfb 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From c72553381630df9b2da3b6d4aafaf341c00fae6b Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 16:35:14 +0900 Subject: [PATCH 14/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index 426a44adfb..cea19b0737 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From 3497a4c259b3c8c2658016889b78fadfd6013121 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 16:35:33 +0900 Subject: [PATCH 15/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index cea19b0737..426a44adfb 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From 1655c02b7b9d8586aae10e87fc949d83daaaac77 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 17:40:23 +0900 Subject: [PATCH 16/83] change hbase cache engine to heap --- .../src/test/java/org/apache/tajo/TajoTestingCluster.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index 76565d2f7b..b6158f0ba0 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -168,9 +168,12 @@ void initPropertiesAndConfigs() { conf.setInt("hbase.hconnection.threads.max", 10); conf.setInt("hbase.hconnection.meta.lookup.threads.core", 5); conf.setInt("hbase.hconnection.meta.lookup.threads.max", 10); + //memory cache conf.setFloat("hfile.block.cache.size", 0.1f); - conf.setInt("hbase.bucketcache.size", 10); + conf.setBoolean("hbase.bucketcache.combinedcache.enabled", false); + conf.setInt("hbase.bucketcache.size", 100); //100 MB + conf.set("hbase.bucketcache.ioengine", "heap"); /* Since Travis CI limits the size of standard output log up to 4MB */ if (!StringUtils.isEmpty(LOG_LEVEL)) { From 6daaec00e53ef88f1a4e45bbc79f4a1b6620df81 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 17:41:41 +0900 Subject: [PATCH 17/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index 426a44adfb..cea19b0737 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From dd2ee728b1f49c7d28440b90eeac851008ed3377 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 17:42:01 +0900 Subject: [PATCH 18/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index cea19b0737..426a44adfb 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From 0d569eac8b826a836dae8b8936cd201ee74c2e50 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 17:42:30 +0900 Subject: [PATCH 19/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index 426a44adfb..cea19b0737 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From 031e0f406ccadcbb630d8a4218b2fd4501972176 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 17:42:53 +0900 Subject: [PATCH 20/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index cea19b0737..426a44adfb 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From 1670588f8b8ce8c2ffce2253c2817fd2e0397ebd Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 18:11:20 +0900 Subject: [PATCH 21/83] disable block cache --- .../src/test/java/org/apache/tajo/TajoTestingCluster.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index b6158f0ba0..9b5cf92e0e 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -170,10 +170,8 @@ void initPropertiesAndConfigs() { conf.setInt("hbase.hconnection.meta.lookup.threads.max", 10); //memory cache - conf.setFloat("hfile.block.cache.size", 0.1f); + conf.setFloat("hfile.block.cache.size", 0.0f); //disable cache conf.setBoolean("hbase.bucketcache.combinedcache.enabled", false); - conf.setInt("hbase.bucketcache.size", 100); //100 MB - conf.set("hbase.bucketcache.ioengine", "heap"); /* Since Travis CI limits the size of standard output log up to 4MB */ if (!StringUtils.isEmpty(LOG_LEVEL)) { From 23fe83ad774f5cd864ffdae259a67933aeaf68e7 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 18:12:37 +0900 Subject: [PATCH 22/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index 426a44adfb..cea19b0737 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From bc740d2dd11c91418dd6b1a24964cf16318100e6 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 18:13:06 +0900 Subject: [PATCH 23/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index cea19b0737..426a44adfb 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From 075f6a7e3de4cfc6f916a9a1b7c42c39378f3e12 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 18:13:42 +0900 Subject: [PATCH 24/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index 426a44adfb..cea19b0737 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From d677f1b2eb24c3a991feb612ba26c7b93fc96a48 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 18:13:58 +0900 Subject: [PATCH 25/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index cea19b0737..426a44adfb 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From 293eaf0761abd03d0699778431bfde9595ad906f Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 21:42:44 +0900 Subject: [PATCH 26/83] add direct memory usage --- .../java/org/apache/tajo/QueryTestCaseBase.java | 16 ++++++++++++---- .../java/org/apache/tajo/storage/BufferPool.java | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java index ffa2dab007..826ea53861 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -38,7 +38,6 @@ import org.apache.tajo.cli.tsql.SimpleParser; import org.apache.tajo.client.TajoClient; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.InsufficientPrivilegeException; import org.apache.tajo.exception.TajoException; @@ -46,12 +45,14 @@ import org.apache.tajo.jdbc.FetchResultSet; import org.apache.tajo.jdbc.TajoMemoryResultSet; import org.apache.tajo.master.GlobalEngine; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.plan.LogicalOptimizer; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.plan.verifier.LogicalPlanVerifier; import org.apache.tajo.plan.verifier.PreLogicalPlanVerifier; import org.apache.tajo.plan.verifier.VerificationState; +import org.apache.tajo.storage.BufferPool; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.util.FileUtil; import org.junit.AfterClass; @@ -67,6 +68,7 @@ import java.io.IOException; import java.io.InputStream; import java.lang.annotation.*; +import java.lang.management.BufferPoolMXBean; import java.lang.reflect.Method; import java.net.URL; import java.sql.ResultSet; @@ -248,9 +250,15 @@ public static void tearDownClass() throws Exception { @Before public void printTestName() { /* protect a travis stalled build */ - System.out.println("Run: " + name.getMethodName() + - " Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) - / (1024 * 1024)) + " MBytes, Active Threads:" + Thread.activeCount()); + BufferPoolMXBean pool = BufferPool.getDirectBufferPool(); + System.out.println(String.format("Used heap: %s/%s, Used direct:%s/%s, Active Threads: %d, Run: %s.%s", + FileUtil.humanReadableByteCount(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(), false), + FileUtil.humanReadableByteCount(Runtime.getRuntime().maxMemory(), false), + FileUtil.humanReadableByteCount(pool.getMemoryUsed(), false), + FileUtil.humanReadableByteCount(pool.getTotalCapacity(), false), + Thread.activeCount(), + getClass().getSimpleName(), + name.getMethodName())); } public QueryTestCaseBase() { diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java b/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java index 403d789e37..c403e61a0c 100644 --- a/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java +++ b/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java @@ -25,9 +25,12 @@ import org.apache.tajo.TajoConstants; import org.apache.tajo.conf.TajoConf; +import java.lang.management.BufferPoolMXBean; +import java.lang.management.ManagementFactory; import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.List; /* this class is PooledBuffer holder */ public class BufferPool { @@ -139,4 +142,17 @@ public static ByteBuf ensureWritable(ByteBuf buf, int minWritableBytes) { public static void free(ByteBuffer byteBuffer) { PlatformDependent.freeDirectBuffer(byteBuffer); } + + public static BufferPoolMXBean getDirectBufferPool() { + for (BufferPoolMXBean pool : getBufferPools()) { + if (pool.getName().equals("direct")) { + return pool; + } + } + return null; + } + + private static List getBufferPools() { + return ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class); + } } From ea5f6eaac5f5394c8cdb9c24d8636f76f956eb00 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 22:14:12 +0900 Subject: [PATCH 27/83] print mapped memory --- .../test/java/org/apache/tajo/QueryTestCaseBase.java | 11 +++++++---- .../main/java/org/apache/tajo/storage/BufferPool.java | 9 +++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 826ea53861..0639a76fb2 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -250,12 +250,15 @@ public static void tearDownClass() throws Exception { @Before public void printTestName() { /* protect a travis stalled build */ - BufferPoolMXBean pool = BufferPool.getDirectBufferPool(); - System.out.println(String.format("Used heap: %s/%s, Used direct:%s/%s, Active Threads: %d, Run: %s.%s", + BufferPoolMXBean direct = BufferPool.getDirectBufferPool(); + BufferPoolMXBean mapped = BufferPool.getMappedBufferPool(); + System.out.println(String.format("Used heap: %s/%s, direct:%s/%s, mapped:%s/%s, Active Threads: %d, Run: %s.%s", FileUtil.humanReadableByteCount(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(), false), FileUtil.humanReadableByteCount(Runtime.getRuntime().maxMemory(), false), - FileUtil.humanReadableByteCount(pool.getMemoryUsed(), false), - FileUtil.humanReadableByteCount(pool.getTotalCapacity(), false), + FileUtil.humanReadableByteCount(direct.getMemoryUsed(), false), + FileUtil.humanReadableByteCount(direct.getTotalCapacity(), false), + FileUtil.humanReadableByteCount(mapped.getMemoryUsed(), false), + FileUtil.humanReadableByteCount(mapped.getTotalCapacity(), false), Thread.activeCount(), getClass().getSimpleName(), name.getMethodName())); diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java b/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java index c403e61a0c..3bc1075f5a 100644 --- a/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java +++ b/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java @@ -152,6 +152,15 @@ public static BufferPoolMXBean getDirectBufferPool() { return null; } + public static BufferPoolMXBean getMappedBufferPool() { + for (BufferPoolMXBean pool : getBufferPools()) { + if (pool.getName().equals("mapped")) { + return pool; + } + } + return null; + } + private static List getBufferPools() { return ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class); } From fca53a1ec4ac3c22ea02e47604a7c6bd93178d80 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 22:15:12 +0900 Subject: [PATCH 28/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index 426a44adfb..cea19b0737 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From cdef33d7d3c9174a39f23584a905981e0f92a28a Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 3 Nov 2015 22:15:40 +0900 Subject: [PATCH 29/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index cea19b0737..426a44adfb 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From 08a3d1887748a29dcd3b6a711c85d7c59512d1b5 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 10:34:01 +0900 Subject: [PATCH 30/83] disable python tests for testing --- .../java/org/apache/tajo/engine/query/TestGroupByQuery.java | 4 ++-- .../java/org/apache/tajo/engine/query/TestSelectQuery.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java index cc9028bfbb..bc4103b3a2 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java @@ -773,7 +773,7 @@ public final void testGroupbyWithLimit3() throws Exception { cleanupQuery(res); } - @Test + /*@Test @Option(sort = true) @SimpleTest public final void testGroupbyWithPythonFunc() throws Exception { @@ -837,5 +837,5 @@ public final void testDistinctPythonUdafWithUnion1() throws Exception { ResultSet res = executeQuery(); assertResultSet(res); cleanupQuery(res); - } + }*/ } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index e55acf1fc0..0adb4ebcb7 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -750,7 +750,7 @@ public void testMultiBytesDelimiter4() throws Exception { } } - @Test + /*@Test public void testSelectPythonFuncs() throws Exception { ResultSet res = executeQuery(); assertResultSet(res); @@ -769,7 +769,7 @@ public void testNestedPythonFunction() throws Exception { ResultSet res = executeQuery(); assertResultSet(res); cleanupQuery(res); - } + }*/ @Test public void testSelectWithParentheses1() throws Exception { From 6a258818ce93113a836962f654d02039b35ad927 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 10:34:53 +0900 Subject: [PATCH 31/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index 426a44adfb..cea19b0737 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From 1a3c643d792ce5ee8d0aad12a465a25ffe0f24a2 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 10:35:12 +0900 Subject: [PATCH 32/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index cea19b0737..426a44adfb 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From 410c6944c1a55f4187acadc3670dacc130a02d47 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 10:39:37 +0900 Subject: [PATCH 33/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index ed96bb0cfa..3b5d46bc16 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From 5267a5085bdff7a1b39edbb3712be459e6661355 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 10:39:53 +0900 Subject: [PATCH 34/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index 3b5d46bc16..ed96bb0cfa 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From 00657038aa50d7a3683da54caa5d5a09328a93ce Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 15:26:25 +0900 Subject: [PATCH 35/83] change to unpooled netty buffer and add MaxMetaspaceSize in tajo-core-tests --- .../tajo-catalog-drivers/tajo-hive/pom.xml | 2 +- tajo-catalog/tajo-catalog-server/pom.xml | 2 +- tajo-cli/pom.xml | 2 +- tajo-client-example/pom.xml | 2 +- tajo-client/pom.xml | 2 +- tajo-cluster-tests/pom.xml | 2 +- tajo-core-tests/pom.xml | 4 ++-- .../tajo/engine/query/TestGroupByQuery.java | 4 ++-- .../tajo/engine/query/TestSelectQuery.java | 4 ++-- .../org/apache/tajo/util/JvmPauseMonitor.java | 1 + .../java/org/apache/tajo/worker/Fetcher.java | 3 +-- tajo-jdbc/pom.xml | 2 +- tajo-project/pom.xml | 2 +- .../pullserver/TajoPullServerService.java | 3 +-- .../org/apache/tajo/rpc/NettyServerBase.java | 8 +++---- .../java/org/apache/tajo/rpc/NettyUtils.java | 21 +++++++++++++++++-- .../org/apache/tajo/rpc/RpcConstants.java | 3 +++ .../org/apache/tajo/rpc/NettyClientBase.java | 3 +-- tajo-storage/tajo-storage-common/pom.xml | 2 +- tajo-storage/tajo-storage-hbase/pom.xml | 2 +- tajo-storage/tajo-storage-hdfs/pom.xml | 2 +- tajo-storage/tajo-storage-jdbc/pom.xml | 2 +- tajo-storage/tajo-storage-pgsql/pom.xml | 2 +- 23 files changed, 48 insertions(+), 32 deletions(-) diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml b/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml index 1a8a188b79..1cd91eb273 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml @@ -82,7 +82,7 @@ true - -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null diff --git a/tajo-catalog/tajo-catalog-server/pom.xml b/tajo-catalog/tajo-catalog-server/pom.xml index 4c585f95e9..b5a3b5a724 100644 --- a/tajo-catalog/tajo-catalog-server/pom.xml +++ b/tajo-catalog/tajo-catalog-server/pom.xml @@ -120,7 +120,7 @@ true - -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null diff --git a/tajo-cli/pom.xml b/tajo-cli/pom.xml index e670529f63..ec6a5b078b 100644 --- a/tajo-cli/pom.xml +++ b/tajo-cli/pom.xml @@ -71,7 +71,7 @@ true - -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 + -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 diff --git a/tajo-client-example/pom.xml b/tajo-client-example/pom.xml index daee2f9892..04005f0700 100644 --- a/tajo-client-example/pom.xml +++ b/tajo-client-example/pom.xml @@ -74,7 +74,7 @@ true - -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 + -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 diff --git a/tajo-client/pom.xml b/tajo-client/pom.xml index 90fcec6a30..e9511723c4 100644 --- a/tajo-client/pom.xml +++ b/tajo-client/pom.xml @@ -75,7 +75,7 @@ true - -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 + -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 diff --git a/tajo-cluster-tests/pom.xml b/tajo-cluster-tests/pom.xml index 835da55365..cb264c6cf7 100644 --- a/tajo-cluster-tests/pom.xml +++ b/tajo-cluster-tests/pom.xml @@ -67,7 +67,7 @@ true - -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1000m -XX:MaxMetaspaceSize=160m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null diff --git a/tajo-core-tests/pom.xml b/tajo-core-tests/pom.xml index 8b1a0b7be9..8963144099 100644 --- a/tajo-core-tests/pom.xml +++ b/tajo-core-tests/pom.xml @@ -73,7 +73,7 @@ true - -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1000m -XX:MaxMetaspaceSize=160m -XX:+CMSClassUnloadingEnabled -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null @@ -375,7 +375,7 @@ ${maven.fork.count} true false - -Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1000m -XX:MaxMetaspaceSize=160m -XX:+CMSClassUnloadingEnabled -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null true true diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java index bc4103b3a2..cc9028bfbb 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java @@ -773,7 +773,7 @@ public final void testGroupbyWithLimit3() throws Exception { cleanupQuery(res); } - /*@Test + @Test @Option(sort = true) @SimpleTest public final void testGroupbyWithPythonFunc() throws Exception { @@ -837,5 +837,5 @@ public final void testDistinctPythonUdafWithUnion1() throws Exception { ResultSet res = executeQuery(); assertResultSet(res); cleanupQuery(res); - }*/ + } } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index 0adb4ebcb7..e55acf1fc0 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -750,7 +750,7 @@ public void testMultiBytesDelimiter4() throws Exception { } } - /*@Test + @Test public void testSelectPythonFuncs() throws Exception { ResultSet res = executeQuery(); assertResultSet(res); @@ -769,7 +769,7 @@ public void testNestedPythonFunction() throws Exception { ResultSet res = executeQuery(); assertResultSet(res); cleanupQuery(res); - }*/ + } @Test public void testSelectWithParentheses1() throws Exception { diff --git a/tajo-core/src/main/java/org/apache/tajo/util/JvmPauseMonitor.java b/tajo-core/src/main/java/org/apache/tajo/util/JvmPauseMonitor.java index 3ec6c4022e..8939bda057 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/JvmPauseMonitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/JvmPauseMonitor.java @@ -211,6 +211,7 @@ public void run() { * log messages about the GC pauses. */ public static void main(String []args) throws Exception { + Thread.setDefaultUncaughtExceptionHandler(new TajoUncaughtExceptionHandler()); new JvmPauseMonitor(new Configuration()).start(); List list = Lists.newArrayList(); int i = 0; diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Fetcher.java b/tajo-core/src/main/java/org/apache/tajo/worker/Fetcher.java index 71d30cd502..2e1639ff37 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/Fetcher.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Fetcher.java @@ -20,7 +20,6 @@ import io.netty.bootstrap.Bootstrap; import io.netty.buffer.ByteBuf; -import io.netty.buffer.PooledByteBufAllocator; import io.netty.channel.*; import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.handler.codec.http.*; @@ -92,7 +91,7 @@ public Fetcher(TajoConf conf, URI uri, FileChunk chunk) { NettyUtils.getSharedEventLoopGroup(NettyUtils.GROUP.FETCHER, conf.getIntVar(TajoConf.ConfVars.SHUFFLE_RPC_CLIENT_WORKER_THREAD_NUM))) .channel(NioSocketChannel.class) - .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) + .option(ChannelOption.ALLOCATOR, NettyUtils.ALLOCATOR) .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, conf.getIntVar(TajoConf.ConfVars.SHUFFLE_FETCHER_CONNECT_TIMEOUT) * 1000) .option(ChannelOption.SO_RCVBUF, 1048576) // set 1M diff --git a/tajo-jdbc/pom.xml b/tajo-jdbc/pom.xml index b42d90de3f..bc77a83fed 100644 --- a/tajo-jdbc/pom.xml +++ b/tajo-jdbc/pom.xml @@ -99,7 +99,7 @@ true - -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml index ab7370c6ee..72e99d118e 100644 --- a/tajo-project/pom.xml +++ b/tajo-project/pom.xml @@ -38,7 +38,7 @@ 0.12.0-SNAPSHOT 1.1.1 1.1.0 - 4.0.29.Final + 4.0.32.Final 2.6 6.1.14 ${project.parent.relativePath}/.. diff --git a/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java b/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java index 4a4ae534b6..b3ab85d1ed 100644 --- a/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java +++ b/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java @@ -20,7 +20,6 @@ import com.google.common.collect.Lists; import io.netty.bootstrap.ServerBootstrap; -import io.netty.buffer.PooledByteBufAllocator; import io.netty.buffer.Unpooled; import io.netty.channel.*; import io.netty.channel.group.ChannelGroup; @@ -189,7 +188,7 @@ public void init(Configuration conf) { selector = NettyUtils.createServerBootstrap("TajoPullServerService", workerNum) .option(ChannelOption.TCP_NODELAY, true) - .childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) + .childOption(ChannelOption.ALLOCATOR, NettyUtils.ALLOCATOR) .childOption(ChannelOption.TCP_NODELAY, true); localFS = new LocalFileSystem(); diff --git a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/NettyServerBase.java b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/NettyServerBase.java index 2c154bf518..258afe8768 100644 --- a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/NettyServerBase.java +++ b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/NettyServerBase.java @@ -18,11 +18,7 @@ package org.apache.tajo.rpc; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - import io.netty.bootstrap.ServerBootstrap; -import io.netty.buffer.PooledByteBufAllocator; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelInitializer; @@ -31,6 +27,8 @@ import io.netty.channel.group.DefaultChannelGroup; import io.netty.channel.socket.nio.NioServerSocketChannel; import io.netty.util.concurrent.GlobalEventExecutor; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import java.io.IOException; import java.net.DatagramSocket; @@ -87,7 +85,7 @@ public void init(ChannelInitializer initializer, int workerNum) { .childHandler(initializer) .option(ChannelOption.SO_REUSEADDR, true) .option(ChannelOption.TCP_NODELAY, true) - .childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) + .childOption(ChannelOption.ALLOCATOR, NettyUtils.ALLOCATOR) .childOption(ChannelOption.TCP_NODELAY, true) .childOption(ChannelOption.CONNECT_TIMEOUT_MILLIS, 10000) .childOption(ChannelOption.SO_RCVBUF, 1048576 * 10); diff --git a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/NettyUtils.java b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/NettyUtils.java index 90b337a42f..4734b79e45 100644 --- a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/NettyUtils.java +++ b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/NettyUtils.java @@ -20,8 +20,12 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.netty.bootstrap.ServerBootstrap; +import io.netty.buffer.ByteBufAllocator; +import io.netty.buffer.PooledByteBufAllocator; +import io.netty.buffer.UnpooledByteBufAllocator; import io.netty.channel.EventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.util.ResourceLeakDetector; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -34,9 +38,10 @@ public final class NettyUtils { private static final Log LOG = LogFactory.getLog(NettyUtils.class); private static final int DEFAULT_THREAD_NUM = Runtime.getRuntime().availableProcessors() * 2; - private static final Object lockObjectForLoopGroup = new Object(); - private static AtomicInteger serverCount = new AtomicInteger(0); + private static final AtomicInteger serverCount = new AtomicInteger(0); + + public static final ByteBufAllocator ALLOCATOR; public enum GROUP { DEFAULT, @@ -46,6 +51,18 @@ public enum GROUP { private static final Map eventLoopGroupMap = new ConcurrentHashMap<>(); + static { + if (RpcConstants.IS_TEST_MODE) { + /* Disable pooling buffers for memory usage */ + ALLOCATOR = UnpooledByteBufAllocator.DEFAULT; + + /* if you are finding memory leak, please enable this line */ + ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.ADVANCED); + } else { + ALLOCATOR = PooledByteBufAllocator.DEFAULT; + } + } + private NettyUtils(){ } diff --git a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java index 601f3d2156..95e5ae4783 100644 --- a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java +++ b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java @@ -47,4 +47,7 @@ public class RpcConstants { public static final String CLIENT_HANG_DETECTION = "tajo.rpc.client.hang-detection"; public final static boolean CLIENT_HANG_DETECTION_DEFAULT = false; + + public static final String TEST_KEY = "tajo.test.enabled"; + public static final boolean IS_TEST_MODE = Boolean.parseBoolean(System.getProperty(TEST_KEY, "false")); } diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java index 6008c4c709..74ed3a0b99 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java @@ -23,7 +23,6 @@ import com.google.protobuf.Message; import com.google.protobuf.ServiceException; import io.netty.bootstrap.Bootstrap; -import io.netty.buffer.PooledByteBufAllocator; import io.netty.channel.*; import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.handler.timeout.IdleState; @@ -94,7 +93,7 @@ protected void init(ChannelInitializer initializer, EventLoopGroup even .group(eventLoopGroup) .channel(NioSocketChannel.class) .handler(initializer) - .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) + .option(ChannelOption.ALLOCATOR, NettyUtils.ALLOCATOR) .option(ChannelOption.SO_REUSEADDR, true) .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, (int) connTimeoutMillis) .option(ChannelOption.SO_RCVBUF, 1048576 * 10) diff --git a/tajo-storage/tajo-storage-common/pom.xml b/tajo-storage/tajo-storage-common/pom.xml index 446fca93df..e48fb7c910 100644 --- a/tajo-storage/tajo-storage-common/pom.xml +++ b/tajo-storage/tajo-storage-common/pom.xml @@ -75,7 +75,7 @@ limitations under the License. TRUE - -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null diff --git a/tajo-storage/tajo-storage-hbase/pom.xml b/tajo-storage/tajo-storage-hbase/pom.xml index 9a52d1f97a..0878c169b9 100644 --- a/tajo-storage/tajo-storage-hbase/pom.xml +++ b/tajo-storage/tajo-storage-hbase/pom.xml @@ -78,7 +78,7 @@ TRUE - -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null diff --git a/tajo-storage/tajo-storage-hdfs/pom.xml b/tajo-storage/tajo-storage-hdfs/pom.xml index 7c1e3e8f17..f121be530e 100644 --- a/tajo-storage/tajo-storage-hdfs/pom.xml +++ b/tajo-storage/tajo-storage-hdfs/pom.xml @@ -81,7 +81,7 @@ TRUE - -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null diff --git a/tajo-storage/tajo-storage-jdbc/pom.xml b/tajo-storage/tajo-storage-jdbc/pom.xml index f331493fd8..9b2d4b71d6 100644 --- a/tajo-storage/tajo-storage-jdbc/pom.xml +++ b/tajo-storage/tajo-storage-jdbc/pom.xml @@ -77,7 +77,7 @@ TRUE - -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 + -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 diff --git a/tajo-storage/tajo-storage-pgsql/pom.xml b/tajo-storage/tajo-storage-pgsql/pom.xml index ad97ea80b3..af71d8e5c6 100644 --- a/tajo-storage/tajo-storage-pgsql/pom.xml +++ b/tajo-storage/tajo-storage-pgsql/pom.xml @@ -234,7 +234,7 @@ TRUE - -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 + -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 From 35c5ce6344eaad48884821f27e5b2902fb2407af Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 17:27:10 +0900 Subject: [PATCH 36/83] seperate hbase tests --- .travis.yml | 3 ++- tajo-core-tests/pom.xml | 4 ++-- tajo-project/pom.xml | 7 ++++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f8a6772808..e5db6942e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,4 +37,5 @@ before_install: ulimit -t 514029 -u 2048 -n 3000 install: ./dev-support/travis-install-dependencies.sh script: - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql + mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest=!%regex[.*.TestHBaseTable.*] + mvn test -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dtest=TestHBaseTable -pl tajo-core-tests diff --git a/tajo-core-tests/pom.xml b/tajo-core-tests/pom.xml index 8963144099..a1307e9544 100644 --- a/tajo-core-tests/pom.xml +++ b/tajo-core-tests/pom.xml @@ -73,7 +73,7 @@ true - -Xms128m -Xmx1000m -XX:MaxMetaspaceSize=160m -XX:+CMSClassUnloadingEnabled -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1000m -XX:MaxMetaspaceSize=200m -XX:+CMSClassUnloadingEnabled -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null @@ -375,7 +375,7 @@ ${maven.fork.count} true false - -Xms128m -Xmx1000m -XX:MaxMetaspaceSize=160m -XX:+CMSClassUnloadingEnabled -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1000m -XX:MaxMetaspaceSize=200m -XX:+CMSClassUnloadingEnabled -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null true true diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml index 72e99d118e..b8c640b6dd 100644 --- a/tajo-project/pom.xml +++ b/tajo-project/pom.xml @@ -479,7 +479,12 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 + 2.19 + + + org.apache.maven.plugins + maven-surefire-plugin + 2.19 org.codehaus.mojo From 6fa8122cd366b157dfaea1246f5862b50b12fd91 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 17:32:06 +0900 Subject: [PATCH 37/83] escape exclamation mark --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e5db6942e9..cd48c9848b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more + Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 @@ -37,5 +37,5 @@ before_install: ulimit -t 514029 -u 2048 -n 3000 install: ./dev-support/travis-install-dependencies.sh script: - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest=!%regex[.*.TestHBaseTable.*] + mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest='!'"%regex[.*.TestHBaseTable.*]" mvn test -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dtest=TestHBaseTable -pl tajo-core-tests From fa45a2e4c9b09a25fc7f079dcbe1345894babf4d Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 17:36:34 +0900 Subject: [PATCH 38/83] fix yaml syntax --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cd48c9848b..ac49babb9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,5 +37,5 @@ before_install: ulimit -t 514029 -u 2048 -n 3000 install: ./dev-support/travis-install-dependencies.sh script: - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest='!'"%regex[.*.TestHBaseTable.*]" + mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest='!'%regex[.*.TestHBaseTable.*] mvn test -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dtest=TestHBaseTable -pl tajo-core-tests From 83562100583f5e7822de9e05047498ba43874780 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 17:40:28 +0900 Subject: [PATCH 39/83] fix escape --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ac49babb9b..f6d6b0f2eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,5 +37,5 @@ before_install: ulimit -t 514029 -u 2048 -n 3000 install: ./dev-support/travis-install-dependencies.sh script: - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest='!'%regex[.*.TestHBaseTable.*] + mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest=\!%regex[.*.TestHBaseTable.*] mvn test -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dtest=TestHBaseTable -pl tajo-core-tests From b3c630e9164169afe173a4a107335d61f4c2e9e6 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 17:47:05 +0900 Subject: [PATCH 40/83] test yaml syntax --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f6d6b0f2eb..e1f482b940 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,5 +37,5 @@ before_install: ulimit -t 514029 -u 2048 -n 3000 install: ./dev-support/travis-install-dependencies.sh script: - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest=\!%regex[.*.TestHBaseTable.*] + mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql "-Dtest=!%regex[.*.TestHBaseTable.*]" mvn test -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dtest=TestHBaseTable -pl tajo-core-tests From d517b6d1bf1e5be5f8c43d0457d96c303c602cbe Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 17:52:11 +0900 Subject: [PATCH 41/83] fix license syntax --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e1f482b940..377e4b1f25 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ - Licensed to the Apache Software Foundation (ASF) under one or more +# Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 @@ -37,5 +37,5 @@ before_install: ulimit -t 514029 -u 2048 -n 3000 install: ./dev-support/travis-install-dependencies.sh script: - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql "-Dtest=!%regex[.*.TestHBaseTable.*]" + mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest=\!%regex[.*.TestHBaseTable.*] mvn test -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dtest=TestHBaseTable -pl tajo-core-tests From 88a2e567752ac484dc45717ef3c1a8a951b35326 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 17:58:19 +0900 Subject: [PATCH 42/83] test travis --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 377e4b1f25..c7345f04a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,5 +37,5 @@ before_install: ulimit -t 514029 -u 2048 -n 3000 install: ./dev-support/travis-install-dependencies.sh script: - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest=\!%regex[.*.TestHBaseTable.*] - mvn test -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dtest=TestHBaseTable -pl tajo-core-tests + - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql "-Dtest=!%regex[.*.TestHBaseTable.*]" + - mvn test -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dtest=TestHBaseTable -pl tajo-core-tests From 58db7d266120330ba40a09f3048575b342c8bf4a Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 18:09:25 +0900 Subject: [PATCH 43/83] test travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c7345f04a9..a0f4628e17 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,5 +37,5 @@ before_install: ulimit -t 514029 -u 2048 -n 3000 install: ./dev-support/travis-install-dependencies.sh script: - - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql "-Dtest=!%regex[.*.TestHBaseTable.*]" + - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest="!%regex[.*.TestHBaseTable.*]" - mvn test -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dtest=TestHBaseTable -pl tajo-core-tests From ed790a243bad009e93f88bcb3b11a83eaee5cc73 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 18:23:36 +0900 Subject: [PATCH 44/83] test travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a0f4628e17..284754ab94 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,5 +37,5 @@ before_install: ulimit -t 514029 -u 2048 -n 3000 install: ./dev-support/travis-install-dependencies.sh script: - - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest="!%regex[.*.TestHBaseTable.*]" + - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest=!%regex[.*.TestHBaseTable.*] - mvn test -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dtest=TestHBaseTable -pl tajo-core-tests From 6344ce928dc93629c571e7e71f60bdede1d6aea5 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 18:40:39 +0900 Subject: [PATCH 45/83] test travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 284754ab94..70cb9b6b46 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,5 +37,5 @@ before_install: ulimit -t 514029 -u 2048 -n 3000 install: ./dev-support/travis-install-dependencies.sh script: - - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest=!%regex[.*.TestHBaseTable.*] + - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest=!TestHBaseTable -DfailIfNoTests=false - mvn test -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dtest=TestHBaseTable -pl tajo-core-tests From 73a1d95e4a2aa95e59960071ccad2a7cc5c9e975 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 19:09:10 +0900 Subject: [PATCH 46/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index ed96bb0cfa..3b5d46bc16 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From 0695352e1470dabb0e166d68c527bbf141c60c6e Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 19:09:46 +0900 Subject: [PATCH 47/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index 3b5d46bc16..ed96bb0cfa 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From 7ce25418f3c4174945122287af89474477b1599d Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 19:10:15 +0900 Subject: [PATCH 48/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index ed96bb0cfa..3b5d46bc16 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From a752cfe8b73c92f7d896a3a3c34fd139fff06078 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 19:10:35 +0900 Subject: [PATCH 49/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index 3b5d46bc16..ed96bb0cfa 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From 5d0b73de3bdf8686b3c649043c08176e4e2c6483 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 19:10:53 +0900 Subject: [PATCH 50/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index ed96bb0cfa..3b5d46bc16 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From 7a458a632643fd78d0b0ac612411d7b5ed3b04d9 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 19:11:12 +0900 Subject: [PATCH 51/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index 3b5d46bc16..ed96bb0cfa 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From 70a8df0a161a1c0751a19a409a8e2b10ee653472 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 20:01:52 +0900 Subject: [PATCH 52/83] Trigger CI --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 70cb9b6b46..5ff10735d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,12 @@ notifications: - issues@tajo.apache.org irc: "chat.freenode.net#tajo" -before_install: ulimit -t 514029 -u 2048 -n 3000 +before_install: + - ulimit -a + - ulimit -t 514029 -u 2048 -n 10000 + - ulimit -a + - free -m + - df -h install: ./dev-support/travis-install-dependencies.sh script: From 1d4ea8dc07517b5b19db8932ff6590a298403ea0 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 20:27:43 +0900 Subject: [PATCH 53/83] Trigger CI --- .travis.yml | 2 -- tajo-core-tests/pom.xml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5ff10735d3..ef23b83f5c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,8 +34,6 @@ notifications: irc: "chat.freenode.net#tajo" before_install: - - ulimit -a - - ulimit -t 514029 -u 2048 -n 10000 - ulimit -a - free -m - df -h diff --git a/tajo-core-tests/pom.xml b/tajo-core-tests/pom.xml index a1307e9544..0d6e4628a0 100644 --- a/tajo-core-tests/pom.xml +++ b/tajo-core-tests/pom.xml @@ -375,7 +375,7 @@ ${maven.fork.count} true false - -Xms128m -Xmx1000m -XX:MaxMetaspaceSize=200m -XX:+CMSClassUnloadingEnabled -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx800m -XX:MaxMetaspaceSize=170m -XX:+CMSClassUnloadingEnabled -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null true true From 36846bbf6aa6f1aca6332e431d3058c10c1bcab7 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 20:32:58 +0900 Subject: [PATCH 54/83] Trigger CI --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ef23b83f5c..e442dd3337 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,7 @@ notifications: irc: "chat.freenode.net#tajo" before_install: + - ulimit -t unlimited -u 10000 -n 64000 - ulimit -a - free -m - df -h From ed03ee747a9abf4ec77f9be8ecb5de37c0dcb374 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 20:57:40 +0900 Subject: [PATCH 55/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index ed96bb0cfa..3b5d46bc16 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From b2f47f2451730d4443d42a11fab393a75014bdae Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 20:58:08 +0900 Subject: [PATCH 56/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index 3b5d46bc16..ed96bb0cfa 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From a33ddd4a77669e13c904e54848a0f940a9e55bc1 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 20:58:33 +0900 Subject: [PATCH 57/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index ed96bb0cfa..3b5d46bc16 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From e92b758489bf54831f4c1046ee2dd14b4e705910 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 20:59:02 +0900 Subject: [PATCH 58/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index 3b5d46bc16..ed96bb0cfa 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From 00013d2a687346ff49e7a7d3b0e9e756854a792f Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 22:36:50 +0900 Subject: [PATCH 59/83] cleanup memory store of derby --- .../apache/tajo/catalog/store/DerbyStore.java | 2 +- .../org/apache/tajo/CatalogTestingUtil.java | 24 +++++++++++++++++++ .../org/apache/tajo/TajoTestingCluster.java | 7 ++++++ tajo-core-tests/pom.xml | 2 +- 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java index 5e3ff49bbf..a83b24996a 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java @@ -36,7 +36,7 @@ public class DerbyStore extends AbstractDBStore { private static final Log LOG = LogFactory.getLog(DerbyStore.class); - private static final String CATALOG_DRIVER="org.apache.derby.jdbc.EmbeddedDriver"; + public static final String CATALOG_DRIVER="org.apache.derby.jdbc.EmbeddedDriver"; protected String getCatalogDriverName(){ return CATALOG_DRIVER; diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/CatalogTestingUtil.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/CatalogTestingUtil.java index 3f45edeb0f..7fed708445 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/CatalogTestingUtil.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/CatalogTestingUtil.java @@ -21,11 +21,16 @@ import org.apache.tajo.annotation.NotNull; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.CatalogConstants; +import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.store.*; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.exception.UnsupportedCatalogStore; +import java.net.URI; +import java.sql.Connection; +import java.sql.DriverManager; + public class CatalogTestingUtil { public static TajoConf configureCatalog(TajoConf conf, String testDirPath) throws UnsupportedCatalogStore { @@ -39,6 +44,25 @@ public static TajoConf configureCatalog(TajoConf conf, String testDirPath) throw return conf; } + public static void shutdownCatalogStore(TajoConf conf) throws Exception { + String catalogUri = conf.get(CatalogConstants.CATALOG_URI); + URI uri = new URI(catalogUri); + String[] schemeSpecificPart = uri.getSchemeSpecificPart().split(":"); + + if("derby".equals(schemeSpecificPart[0]) && "memory".equals(schemeSpecificPart[1])) { + Connection conn = null; + try { + // Removing an in-memory database. + String removingUri = catalogUri.split(";")[0] + ";drop=true"; + Class.forName(DerbyStore.CATALOG_DRIVER).newInstance(); + System.out.println(removingUri); + conn = DriverManager.getConnection(removingUri); + } finally { + CatalogUtil.closeQuietly(conn); + } + } + } + static boolean requireAuth(Class clazz) { return clazz.equals(MySQLStore.class) || clazz.equals(MariaDBStore.class) || diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index 9b5cf92e0e..35ac49722d 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -319,6 +319,13 @@ public CatalogServer startCatalogCluster() throws Exception { } public void shutdownCatalogCluster() { + + try { + CatalogTestingUtil.shutdownCatalogStore(conf); + } catch (Exception e) { + //ignore + } + if (catalogServer != null) { this.catalogServer.stop(); } diff --git a/tajo-core-tests/pom.xml b/tajo-core-tests/pom.xml index 0d6e4628a0..6fd05a5fd5 100644 --- a/tajo-core-tests/pom.xml +++ b/tajo-core-tests/pom.xml @@ -375,7 +375,7 @@ ${maven.fork.count} true false - -Xms128m -Xmx800m -XX:MaxMetaspaceSize=170m -XX:+CMSClassUnloadingEnabled -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx800m -XX:MaxMetaspaceSize=200m -XX:+CMSClassUnloadingEnabled -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null true true From 8357371bdf267e7541d1f5b1caa55dff1af06dab Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 22:38:46 +0900 Subject: [PATCH 60/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index ed96bb0cfa..3b5d46bc16 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From b4741af47c116db67e78047a8ae6915f82ff8a6e Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 4 Nov 2015 22:39:12 +0900 Subject: [PATCH 61/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index 3b5d46bc16..ed96bb0cfa 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From 3fd61dfd28fe7e37b91daae9ed39fa68bb608706 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 00:56:01 +0900 Subject: [PATCH 62/83] decrease cpu-core for travis guide (2-4) --- .../src/test/java/org/apache/tajo/TajoTestingCluster.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index 35ac49722d..beec825981 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -127,10 +127,10 @@ void initPropertiesAndConfigs() { conf.setClassVar(ConfVars.LOGICAL_PLAN_REWRITE_RULE_PROVIDER_CLASS, LogicalPlanTestRuleProvider.class); conf.setClassVar(ConfVars.GLOBAL_PLAN_REWRITE_RULE_PROVIDER_CLASS, GlobalPlanTestRuleProvider.class); - conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES.varname, 4); + conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES.varname, 3); conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_MEMORY_MB.varname, 2000); conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_DISK_PARALLEL_NUM.varname, 3); - conf.setInt(ConfVars.SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM.varname, 2); + conf.setInt(ConfVars.SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM.varname, 1); // Client API RPC conf.setIntVar(ConfVars.RPC_CLIENT_WORKER_THREAD_NUM, 2); From c312986b2f3e00d653b3c7c400f59bbda5d0447f Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 10:05:43 +0900 Subject: [PATCH 63/83] Trigger CI --- .../src/test/java/org/apache/tajo/TajoTestingCluster.java | 2 +- tajo-core-tests/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index beec825981..44cdab4460 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -127,7 +127,7 @@ void initPropertiesAndConfigs() { conf.setClassVar(ConfVars.LOGICAL_PLAN_REWRITE_RULE_PROVIDER_CLASS, LogicalPlanTestRuleProvider.class); conf.setClassVar(ConfVars.GLOBAL_PLAN_REWRITE_RULE_PROVIDER_CLASS, GlobalPlanTestRuleProvider.class); - conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES.varname, 3); + conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES.varname, 2); conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_MEMORY_MB.varname, 2000); conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_DISK_PARALLEL_NUM.varname, 3); conf.setInt(ConfVars.SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM.varname, 1); diff --git a/tajo-core-tests/pom.xml b/tajo-core-tests/pom.xml index 6fd05a5fd5..a1307e9544 100644 --- a/tajo-core-tests/pom.xml +++ b/tajo-core-tests/pom.xml @@ -375,7 +375,7 @@ ${maven.fork.count} true false - -Xms128m -Xmx800m -XX:MaxMetaspaceSize=200m -XX:+CMSClassUnloadingEnabled -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1000m -XX:MaxMetaspaceSize=200m -XX:+CMSClassUnloadingEnabled -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null true true From 5178ce68b2e957f3b8cc3a7d5dd5549e6bceeb11 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 10:06:44 +0900 Subject: [PATCH 64/83] Trigger CI --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e442dd3337..54c538c751 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,5 +41,4 @@ before_install: install: ./dev-support/travis-install-dependencies.sh script: - - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest=!TestHBaseTable -DfailIfNoTests=false - - mvn test -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dtest=TestHBaseTable -pl tajo-core-tests + - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql From 85d5e9bfc63a66e60c4e485625f6a8c685aa4911 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 11:46:41 +0900 Subject: [PATCH 65/83] Trigger CI --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 54c538c751..5a3635aa09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,3 +42,4 @@ install: ./dev-support/travis-install-dependencies.sh script: - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql + - dmesg | grep -i kill From cd11d95269a859ad4f4a3da59b725820277da5ee Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 13:46:05 +0900 Subject: [PATCH 66/83] stop unused service (mysql,postgresql) for more memory --- .travis.yml | 8 +++++++- .../src/test/java/org/apache/tajo/TajoTestingCluster.java | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5a3635aa09..4cf6ab83ce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,8 +38,14 @@ before_install: - ulimit -a - free -m - df -h + # Disable services enabled by default + # http://docs.travis-ci.com/user/database-setup/#MySQL + - sudo /etc/init.d/mysql stop + - sudo /etc/init.d/postgresql stop + install: ./dev-support/travis-install-dependencies.sh script: - - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql + - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest=!TestHBaseTable -DfailIfNoTests=false + - mvn test -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dtest=TestHBaseTable -pl tajo-core-tests - dmesg | grep -i kill diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index 44cdab4460..046a224695 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -127,7 +127,7 @@ void initPropertiesAndConfigs() { conf.setClassVar(ConfVars.LOGICAL_PLAN_REWRITE_RULE_PROVIDER_CLASS, LogicalPlanTestRuleProvider.class); conf.setClassVar(ConfVars.GLOBAL_PLAN_REWRITE_RULE_PROVIDER_CLASS, GlobalPlanTestRuleProvider.class); - conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES.varname, 2); + conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES.varname, 4); conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_MEMORY_MB.varname, 2000); conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_DISK_PARALLEL_NUM.varname, 3); conf.setInt(ConfVars.SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM.varname, 1); From 0345043f8391ad61ccc9339c8437e73b70a708fd Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 13:50:06 +0900 Subject: [PATCH 67/83] Trigger CI --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4cf6ab83ce..a8f94d678d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,14 +34,14 @@ notifications: irc: "chat.freenode.net#tajo" before_install: - - ulimit -t unlimited -u 10000 -n 64000 - - ulimit -a - - free -m - - df -h # Disable services enabled by default # http://docs.travis-ci.com/user/database-setup/#MySQL - sudo /etc/init.d/mysql stop - sudo /etc/init.d/postgresql stop + - ulimit -t unlimited -u 10000 -n 64000 + - ulimit -a + - free -m + - df -h install: ./dev-support/travis-install-dependencies.sh From 1ea0606ddd9d6a1624c5aaa5a107f2317af37ef3 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 14:13:51 +0900 Subject: [PATCH 68/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index ed96bb0cfa..3b5d46bc16 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From 55fc15a32da4d9c2683de464422f56e7861eb3f2 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 14:14:17 +0900 Subject: [PATCH 69/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index 3b5d46bc16..ed96bb0cfa 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From 80842c15e84bd1695ae580986e740717a7a7ac68 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 14:51:50 +0900 Subject: [PATCH 70/83] fix exit code --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a8f94d678d..36021bb937 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,10 +42,11 @@ before_install: - ulimit -a - free -m - df -h + - sh -c "dmesg | grep -i OOM || exit 0" install: ./dev-support/travis-install-dependencies.sh script: - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest=!TestHBaseTable -DfailIfNoTests=false - mvn test -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dtest=TestHBaseTable -pl tajo-core-tests - - dmesg | grep -i kill + - sh -c "dmesg | grep -i OOM || exit 0" From e1b179de4035f4ab9ed9edb93372dc4189e51a48 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 15:24:48 +0900 Subject: [PATCH 71/83] decrease max heap for parallel test --- .travis.yml | 2 +- tajo-core-tests/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 36021bb937..1e0acc2f03 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,6 @@ before_install: - ulimit -a - free -m - df -h - - sh -c "dmesg | grep -i OOM || exit 0" install: ./dev-support/travis-install-dependencies.sh @@ -50,3 +49,4 @@ script: - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest=!TestHBaseTable -DfailIfNoTests=false - mvn test -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dtest=TestHBaseTable -pl tajo-core-tests - sh -c "dmesg | grep -i OOM || exit 0" + - free -m diff --git a/tajo-core-tests/pom.xml b/tajo-core-tests/pom.xml index a1307e9544..6fd05a5fd5 100644 --- a/tajo-core-tests/pom.xml +++ b/tajo-core-tests/pom.xml @@ -375,7 +375,7 @@ ${maven.fork.count} true false - -Xms128m -Xmx1000m -XX:MaxMetaspaceSize=200m -XX:+CMSClassUnloadingEnabled -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx800m -XX:MaxMetaspaceSize=200m -XX:+CMSClassUnloadingEnabled -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null true true From a45d29cd872e92a9050e7379a694c52fb6dd00e5 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 15:25:50 +0900 Subject: [PATCH 72/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index ed96bb0cfa..3b5d46bc16 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From c191c89700fadc4149c4417afd95a4bcb025d2fc Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 15:26:06 +0900 Subject: [PATCH 73/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index 3b5d46bc16..ed96bb0cfa 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From d7eb88591019f8d239ceee5aec7c192561312a7a Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 15:26:26 +0900 Subject: [PATCH 74/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index ed96bb0cfa..3b5d46bc16 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From e376c2352d6f4ebba8f7c0282e0d2a5d74154f82 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 15:26:43 +0900 Subject: [PATCH 75/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index 3b5d46bc16..ed96bb0cfa 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From 04f52f3098f8d40632fa54ca1e89262324432870 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 15:54:39 +0900 Subject: [PATCH 76/83] cleanup unnecessary options --- pom.xml | 2 +- tajo-algebra/pom.xml | 1 - tajo-catalog/tajo-catalog-client/pom.xml | 1 - tajo-catalog/tajo-catalog-common/pom.xml | 1 - tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml | 2 +- tajo-catalog/tajo-catalog-server/pom.xml | 2 +- tajo-cli/pom.xml | 2 +- tajo-client-example/pom.xml | 2 +- tajo-client/pom.xml | 2 +- tajo-cluster-tests/pom.xml | 2 +- tajo-common/pom.xml | 1 - tajo-core/pom.xml | 2 -- tajo-jdbc/pom.xml | 2 +- tajo-metrics/pom.xml | 1 - tajo-plan/pom.xml | 1 - tajo-project/pom.xml | 2 +- tajo-rpc/tajo-rpc-common/pom.xml | 2 -- tajo-rpc/tajo-rpc-protobuf/pom.xml | 2 -- tajo-rpc/tajo-ws-rs/pom.xml | 2 -- tajo-storage/tajo-storage-common/pom.xml | 3 +-- tajo-storage/tajo-storage-hbase/pom.xml | 3 +-- tajo-storage/tajo-storage-hdfs/pom.xml | 3 +-- tajo-storage/tajo-storage-jdbc/pom.xml | 3 +-- tajo-storage/tajo-storage-pgsql/pom.xml | 3 +-- 24 files changed, 14 insertions(+), 33 deletions(-) diff --git a/pom.xml b/pom.xml index 3e17e281c5..1ded49f4bd 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,7 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 + 2.19 true diff --git a/tajo-algebra/pom.xml b/tajo-algebra/pom.xml index 0e942bf0f8..c963f9f3e4 100644 --- a/tajo-algebra/pom.xml +++ b/tajo-algebra/pom.xml @@ -202,7 +202,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 diff --git a/tajo-catalog/tajo-catalog-client/pom.xml b/tajo-catalog/tajo-catalog-client/pom.xml index dfeb426830..933ec82142 100644 --- a/tajo-catalog/tajo-catalog-client/pom.xml +++ b/tajo-catalog/tajo-catalog-client/pom.xml @@ -178,7 +178,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 diff --git a/tajo-catalog/tajo-catalog-common/pom.xml b/tajo-catalog/tajo-catalog-common/pom.xml index d2756004fd..27e87b8d1c 100644 --- a/tajo-catalog/tajo-catalog-common/pom.xml +++ b/tajo-catalog/tajo-catalog-common/pom.xml @@ -199,7 +199,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml b/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml index 1cd91eb273..1a8a188b79 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml @@ -82,7 +82,7 @@ true - -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null diff --git a/tajo-catalog/tajo-catalog-server/pom.xml b/tajo-catalog/tajo-catalog-server/pom.xml index b5a3b5a724..4c585f95e9 100644 --- a/tajo-catalog/tajo-catalog-server/pom.xml +++ b/tajo-catalog/tajo-catalog-server/pom.xml @@ -120,7 +120,7 @@ true - -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null diff --git a/tajo-cli/pom.xml b/tajo-cli/pom.xml index ec6a5b078b..e670529f63 100644 --- a/tajo-cli/pom.xml +++ b/tajo-cli/pom.xml @@ -71,7 +71,7 @@ true - -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 + -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 diff --git a/tajo-client-example/pom.xml b/tajo-client-example/pom.xml index 04005f0700..daee2f9892 100644 --- a/tajo-client-example/pom.xml +++ b/tajo-client-example/pom.xml @@ -74,7 +74,7 @@ true - -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 + -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 diff --git a/tajo-client/pom.xml b/tajo-client/pom.xml index e9511723c4..90fcec6a30 100644 --- a/tajo-client/pom.xml +++ b/tajo-client/pom.xml @@ -75,7 +75,7 @@ true - -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 + -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 diff --git a/tajo-cluster-tests/pom.xml b/tajo-cluster-tests/pom.xml index cb264c6cf7..365f2e0b4d 100644 --- a/tajo-cluster-tests/pom.xml +++ b/tajo-cluster-tests/pom.xml @@ -67,7 +67,7 @@ true - -Xms128m -Xmx1000m -XX:MaxMetaspaceSize=160m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1000m -XX:MaxMetaspaceSize=200m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null diff --git a/tajo-common/pom.xml b/tajo-common/pom.xml index 70fb08b62d..06703b333d 100644 --- a/tajo-common/pom.xml +++ b/tajo-common/pom.xml @@ -183,7 +183,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml index a2a373d846..ed006a7191 100644 --- a/tajo-core/pom.xml +++ b/tajo-core/pom.xml @@ -335,7 +335,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 true @@ -417,7 +416,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 diff --git a/tajo-jdbc/pom.xml b/tajo-jdbc/pom.xml index bc77a83fed..b42d90de3f 100644 --- a/tajo-jdbc/pom.xml +++ b/tajo-jdbc/pom.xml @@ -99,7 +99,7 @@ true - -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null diff --git a/tajo-metrics/pom.xml b/tajo-metrics/pom.xml index 5425d89321..1361694f66 100644 --- a/tajo-metrics/pom.xml +++ b/tajo-metrics/pom.xml @@ -258,7 +258,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 diff --git a/tajo-plan/pom.xml b/tajo-plan/pom.xml index a9a3f31694..32a6df07b0 100644 --- a/tajo-plan/pom.xml +++ b/tajo-plan/pom.xml @@ -277,7 +277,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml index b8c640b6dd..32a141e63b 100644 --- a/tajo-project/pom.xml +++ b/tajo-project/pom.xml @@ -1305,7 +1305,7 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 + 2.19 true diff --git a/tajo-rpc/tajo-rpc-common/pom.xml b/tajo-rpc/tajo-rpc-common/pom.xml index 187d2ff5a1..80fb9acf42 100644 --- a/tajo-rpc/tajo-rpc-common/pom.xml +++ b/tajo-rpc/tajo-rpc-common/pom.xml @@ -67,7 +67,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 @@ -207,7 +206,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 diff --git a/tajo-rpc/tajo-rpc-protobuf/pom.xml b/tajo-rpc/tajo-rpc-protobuf/pom.xml index 4335d6a524..4d9a62bf74 100644 --- a/tajo-rpc/tajo-rpc-protobuf/pom.xml +++ b/tajo-rpc/tajo-rpc-protobuf/pom.xml @@ -130,7 +130,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 @@ -266,7 +265,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 diff --git a/tajo-rpc/tajo-ws-rs/pom.xml b/tajo-rpc/tajo-ws-rs/pom.xml index 8868155a2e..3064d622b2 100644 --- a/tajo-rpc/tajo-ws-rs/pom.xml +++ b/tajo-rpc/tajo-ws-rs/pom.xml @@ -63,7 +63,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 @@ -209,7 +208,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 diff --git a/tajo-storage/tajo-storage-common/pom.xml b/tajo-storage/tajo-storage-common/pom.xml index e48fb7c910..c552ab5f02 100644 --- a/tajo-storage/tajo-storage-common/pom.xml +++ b/tajo-storage/tajo-storage-common/pom.xml @@ -75,7 +75,7 @@ limitations under the License. TRUE - -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null @@ -343,7 +343,6 @@ limitations under the License. org.apache.maven.plugins maven-surefire-report-plugin - 2.15 diff --git a/tajo-storage/tajo-storage-hbase/pom.xml b/tajo-storage/tajo-storage-hbase/pom.xml index 0878c169b9..ddb53ca8d5 100644 --- a/tajo-storage/tajo-storage-hbase/pom.xml +++ b/tajo-storage/tajo-storage-hbase/pom.xml @@ -78,7 +78,7 @@ TRUE - -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null @@ -366,7 +366,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 diff --git a/tajo-storage/tajo-storage-hdfs/pom.xml b/tajo-storage/tajo-storage-hdfs/pom.xml index f121be530e..3b89e1c145 100644 --- a/tajo-storage/tajo-storage-hdfs/pom.xml +++ b/tajo-storage/tajo-storage-hdfs/pom.xml @@ -81,7 +81,7 @@ TRUE - -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null @@ -397,7 +397,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 diff --git a/tajo-storage/tajo-storage-jdbc/pom.xml b/tajo-storage/tajo-storage-jdbc/pom.xml index 9b2d4b71d6..61481c74a9 100644 --- a/tajo-storage/tajo-storage-jdbc/pom.xml +++ b/tajo-storage/tajo-storage-jdbc/pom.xml @@ -77,7 +77,7 @@ TRUE - -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 + -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 @@ -231,7 +231,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 diff --git a/tajo-storage/tajo-storage-pgsql/pom.xml b/tajo-storage/tajo-storage-pgsql/pom.xml index af71d8e5c6..6f37ab6d2a 100644 --- a/tajo-storage/tajo-storage-pgsql/pom.xml +++ b/tajo-storage/tajo-storage-pgsql/pom.xml @@ -234,7 +234,7 @@ TRUE - -Xms128m -Xmx1024m -XX:MaxMetaspaceSize=128m -Dfile.encoding=UTF-8 + -Xms128m -Xmx1024m -Dfile.encoding=UTF-8 @@ -274,7 +274,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - 2.15 From 3ce90af393d98b5cc9c7cfc11cfac2f889bfb960 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 16:15:12 +0900 Subject: [PATCH 77/83] separate rest api tests --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1e0acc2f03..6c8798473f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,7 +46,7 @@ before_install: install: ./dev-support/travis-install-dependencies.sh script: - - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest=!TestHBaseTable -DfailIfNoTests=false - - mvn test -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dtest=TestHBaseTable -pl tajo-core-tests + - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest=!TestHBaseTable,!**/ws/rs/** -DfailIfNoTests=false + - mvn test -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dtest=TestHBaseTable,**/ws/rs/** -pl tajo-core-tests - sh -c "dmesg | grep -i OOM || exit 0" - free -m From 54c10617b46f015e09bc0fd3132e1a2fce4919d4 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 16:21:10 +0900 Subject: [PATCH 78/83] separate eval tests for metaspace --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6c8798473f..29b5e2555b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,7 +46,7 @@ before_install: install: ./dev-support/travis-install-dependencies.sh script: - - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest=!TestHBaseTable,!**/ws/rs/** -DfailIfNoTests=false - - mvn test -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dtest=TestHBaseTable,**/ws/rs/** -pl tajo-core-tests + - mvn clean install -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 -Ptest-storage-pgsql -Dtest=!TestHBaseTable,!**/ws/rs/**,!**/engine/eval/** -DfailIfNoTests=false + - mvn test -q -ff -Dsurefire.useFile=false -Pparallel-test -DLOG_LEVEL=WARN -Dtest=TestHBaseTable,**/ws/rs/**,**/engine/eval/** -pl tajo-core-tests - sh -c "dmesg | grep -i OOM || exit 0" - free -m From af8dfda031b85010ed6e40c704178f15baeb0dba Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 16:55:25 +0900 Subject: [PATCH 79/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index ed96bb0cfa..3b5d46bc16 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From ab3a175d67beca8bc98906b610c799f041ff101e Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 16:55:47 +0900 Subject: [PATCH 80/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index 3b5d46bc16..ed96bb0cfa 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From 68dd4142728248a98a6bfdf56507a286644f81da Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 16:56:05 +0900 Subject: [PATCH 81/83] Trigger CI --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index ed96bb0cfa..3b5d46bc16 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ Tajo Change Log + Release 0.12.0 - unreleased NEW FEATURES From d772bd593e7aabf8874dae9dde0cfa70658fff00 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 16:56:23 +0900 Subject: [PATCH 82/83] Trigger CI --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index 3b5d46bc16..ed96bb0cfa 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.12.0 - unreleased NEW FEATURES From ad742a521aa607623f9f5cb08f9945c0f74e46f3 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 5 Nov 2015 17:27:57 +0900 Subject: [PATCH 83/83] remove MaxMetaspaceSize --- tajo-cluster-tests/pom.xml | 2 +- tajo-core-tests/pom.xml | 4 ++-- tajo-project/pom.xml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tajo-cluster-tests/pom.xml b/tajo-cluster-tests/pom.xml index 365f2e0b4d..7b537ff0dd 100644 --- a/tajo-cluster-tests/pom.xml +++ b/tajo-cluster-tests/pom.xml @@ -67,7 +67,7 @@ true - -Xms128m -Xmx1000m -XX:MaxMetaspaceSize=200m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1000m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null diff --git a/tajo-core-tests/pom.xml b/tajo-core-tests/pom.xml index 6fd05a5fd5..2037f0be30 100644 --- a/tajo-core-tests/pom.xml +++ b/tajo-core-tests/pom.xml @@ -73,7 +73,7 @@ true - -Xms128m -Xmx1000m -XX:MaxMetaspaceSize=200m -XX:+CMSClassUnloadingEnabled -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx1000m -XX:+CMSClassUnloadingEnabled -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null @@ -375,7 +375,7 @@ ${maven.fork.count} true false - -Xms128m -Xmx800m -XX:MaxMetaspaceSize=200m -XX:+CMSClassUnloadingEnabled -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null + -Xms128m -Xmx800m -XX:+CMSClassUnloadingEnabled -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null true true diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml index 32a141e63b..f8a82d056f 100644 --- a/tajo-project/pom.xml +++ b/tajo-project/pom.xml @@ -38,7 +38,7 @@ 0.12.0-SNAPSHOT 1.1.1 1.1.0 - 4.0.32.Final + 4.0.33.Final 2.6 6.1.14 ${project.parent.relativePath}/.. @@ -1013,7 +1013,7 @@ junit junit - 4.11 + 4.12 jar compile