From 6b2544de0c1f92f70260764b1789f63c5f08d977 Mon Sep 17 00:00:00 2001 From: andrew Date: Wed, 26 Aug 2015 15:11:17 -0700 Subject: [PATCH 1/5] First pass at test re-factoring Unifying test class hierarchy and adding randomization functionality. --- common/pom.xml | 105 +++- .../org/apache/drill/test/DrillTestBase.java | 254 +++++++++ .../apache/drill/test/DrillUnitTestBase.java | 27 + .../drill/test/ReproducibleListener.java | 62 +++ contrib/pom.xml | 89 +++- .../drill/exec/DrillIntegrationTestBase.java | 486 ++++++++++++++++++ 6 files changed, 1003 insertions(+), 20 deletions(-) create mode 100644 common/src/test/java/org/apache/drill/test/DrillTestBase.java create mode 100644 common/src/test/java/org/apache/drill/test/DrillUnitTestBase.java create mode 100644 common/src/test/java/org/apache/drill/test/ReproducibleListener.java create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/DrillIntegrationTestBase.java diff --git a/common/pom.xml b/common/pom.xml index 85b487efb39..8dbfecdcfda 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -15,9 +15,10 @@ See the License for the specific language governing permissions and limitations under the License. --> - + 4.0.0 - + drill-root org.apache.drill @@ -28,49 +29,50 @@ jar Common (Logical Plan, Base expressions) + + 2.1.16 + .local-${project.version}-execution-hints.log + + org.apache.drill drill-protocol ${project.version} - - + junit junit ${dep.junit.version} - org.apache.calcite calcite-core - com.typesafe config 1.0.0 - org.apache.commons commons-lang3 3.1 - org.msgpack msgpack 0.6.6 - org.reflections reflections 0.9.8 - com.fasterxml.jackson.core jackson-annotations @@ -81,7 +83,6 @@ jackson-databind 2.4.3 - org.hibernate hibernate-validator @@ -98,16 +99,34 @@ antlr-runtime 3.4 - - joda-time - joda-time - 2.3 + joda-time + joda-time + 2.3 + + + com.carrotsearch.randomizedtesting + randomizedtesting-runner + ${carrotsearch.testframework} + test + + + com.carrotsearch.randomizedtesting + junit4-maven-plugin + ${carrotsearch.testframework} + test + + + + commons-logging + commons-logging-api + + + - @@ -115,7 +134,57 @@ 2.15 false + true + + + + com.carrotsearch.randomizedtesting + junit4-maven-plugin + ${carrotsearch.testframework} + + true + + + + + + + + + + tests + test + + junit4 + + + + + + + + + + + + + **/*Tests.class + **/*Test.class + + + + maven-jar-plugin @@ -132,9 +201,7 @@ org.antlr antlr3-maven-plugin 3.4 - - - + diff --git a/common/src/test/java/org/apache/drill/test/DrillTestBase.java b/common/src/test/java/org/apache/drill/test/DrillTestBase.java new file mode 100644 index 00000000000..46719a6907b --- /dev/null +++ b/common/src/test/java/org/apache/drill/test/DrillTestBase.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.test; + +import org.junit.Before; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.AfterClass; +import org.junit.Rule; +import org.junit.rules.TestName; +import org.junit.rules.TestRule; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; +import org.junit.runner.RunWith; + +import com.carrotsearch.randomizedtesting.RandomizedTest; +import com.carrotsearch.randomizedtesting.RandomizedRunner; +import com.carrotsearch.randomizedtesting.RandomizedContext; +import com.carrotsearch.randomizedtesting.generators.RandomPicks; +import com.carrotsearch.randomizedtesting.annotations.Listeners; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; + +import org.apache.drill.common.util.TestTools; +import org.apache.drill.common.util.DrillStringUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.management.BufferPoolMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.util.Collection; +import java.util.List; +import java.util.Random; + +/** + * Abstract base class for all Drill tests. + * + * In addition to setting up and tearing down basic test infrastructure, this class also + * provides randomization utilities for those classes which choose to avail themselves of it. + * + * @see "http://labs.carrotsearch.com/randomizedtesting.html" + */ +@Listeners(ReproducibleListener.class) +@RunWith(RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public abstract class DrillTestBase { + + private static final Logger logger = LoggerFactory.getLogger("org.apache.drill.TestReporter"); + private static final TestLogReporter LOG_OUTCOME = new TestLogReporter(); + + static { + System.setProperty("line.separator", "\n"); + } + + protected static final SystemManager manager = new SystemManager(); + protected static MemWatcher memWatcher; + protected static String className; + + @Rule public final TestRule TIMEOUT = TestTools.getTimeoutRule(50000); + @Rule public final TestLogReporter logOutcome = LOG_OUTCOME; + @Rule public final TestRule REPEAT_RULE = TestTools.getRepeatRule(false); + @Rule public TestName TEST_NAME = new TestName(); + + @BeforeClass + public static void initDrillTest() throws Exception { + memWatcher = new MemWatcher(); + } + + @AfterClass + public static void finishDrillTest() throws InterruptedException{ + logger.info(String.format("Test Class done (%s): %s.", memWatcher.getMemString(true), className)); + LOG_OUTCOME.sleepIfFailure(); + } + + @Before + public void beforeDrillTestBase() throws Exception { + System.out.printf("Running %s#%s\n", getClass().getName(), TEST_NAME.getMethodName()); + } + + @After + public void afterDrillTestBase() { + // Optionally add anything here that needs to be cleared after each test. + } + + /* *** Randomization Utilities *** */ + + public static Random random() { + return RandomizedContext.current().getRandom(); + } + + public static boolean randomBoolean() { + return random().nextBoolean(); + } + + public static byte randomByte() { + return (byte) random().nextInt(); + } + + public static short randomShort() { + return (short) random().nextInt(); + } + + public static int randomInt() { + return random().nextInt(); + } + + public static float randomFloat() { + return random().nextFloat(); + } + + public static double randomDouble() { + return random().nextDouble(); + } + + public static long randomLong() { + return random().nextLong(); + } + + public static int randomInt(int max) { + return RandomizedTest.randomInt(max); + } + + public static String randomAscii() { + return RandomizedTest.randomAsciiOfLength(32); + } + + public static String randomAsciiOfLength(int codeUnits) { + return RandomizedTest.randomAsciiOfLength(codeUnits); + } + + public static T randomFrom(List list) { + return RandomPicks.randomFrom(random(), list); + } + + public static T randomFrom(T[] array) { + return RandomPicks.randomFrom(random(), array); + } + + public static T randomFrom(Collection collection) { + return RandomPicks.randomFrom(random(), collection); + } + + /* *** Utility Classes *** */ + + protected static class MemWatcher { + private long startDirect; + private long startHeap; + private long startNonHeap; + + public MemWatcher() { + startDirect = manager.getMemDirect(); + startHeap = manager.getMemHeap(); + startNonHeap = manager.getMemNonHeap(); + } + + public Object getMemString() { + return getMemString(false); + } + + public String getMemString(boolean runGC) { + if (runGC) { + Runtime.getRuntime().gc(); + } + long endDirect = manager.getMemDirect(); + long endHeap = manager.getMemHeap(); + long endNonHeap = manager.getMemNonHeap(); + return String.format("d: %s(%s), h: %s(%s), nh: %s(%s)", // + DrillStringUtils.readable(endDirect - startDirect), DrillStringUtils.readable(endDirect), // + DrillStringUtils.readable(endHeap - startHeap), DrillStringUtils.readable(endHeap), // + DrillStringUtils.readable(endNonHeap - startNonHeap), DrillStringUtils.readable(endNonHeap) // + ); + } + } + + private static class TestLogReporter extends TestWatcher { + + private DrillTest.MemWatcher memWatcher; + private int failureCount = 0; + + @Override + protected void starting(Description description) { + super.starting(description); + className = description.getClassName(); + memWatcher = new DrillTest.MemWatcher(); + } + + @Override + protected void failed(Throwable e, Description description) { + logger.error(String.format("Test Failed (%s): %s", memWatcher.getMemString(), description.getDisplayName()), e); + failureCount++; + } + + @Override + public void succeeded(Description description) { + logger.info(String.format("Test Succeeded (%s): %s", memWatcher.getMemString(), description.getDisplayName())); + } + + public void sleepIfFailure() throws InterruptedException { + if(failureCount > 0){ + Thread.sleep(2000); + failureCount = 0; + } else { + // pause to get logger to catch up. + Thread.sleep(250); + } + } + } + + private static class SystemManager { + + final BufferPoolMXBean directBean; + final MemoryMXBean memoryBean; + + public SystemManager() { + memoryBean = ManagementFactory.getMemoryMXBean(); + BufferPoolMXBean localBean = null; + List pools = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class); + for(BufferPoolMXBean b : pools) { + if (b.getName().equals("direct")) { + localBean = b; + } + } + directBean = localBean; + } + + public long getMemDirect() { + return directBean.getMemoryUsed(); + } + + public long getMemHeap() { + return memoryBean.getHeapMemoryUsage().getUsed(); + } + + public long getMemNonHeap() { + return memoryBean.getNonHeapMemoryUsage().getUsed(); + } + } +} diff --git a/common/src/test/java/org/apache/drill/test/DrillUnitTestBase.java b/common/src/test/java/org/apache/drill/test/DrillUnitTestBase.java new file mode 100644 index 00000000000..c1b2d8f8071 --- /dev/null +++ b/common/src/test/java/org/apache/drill/test/DrillUnitTestBase.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.test; + +/** + * Abstract base class for all Drill unit tests. + * + * Tests which do not require a drillbit to be running should extend this class. + */ +public abstract class DrillUnitTestBase extends DrillTestBase { + +} diff --git a/common/src/test/java/org/apache/drill/test/ReproducibleListener.java b/common/src/test/java/org/apache/drill/test/ReproducibleListener.java new file mode 100644 index 00000000000..65358b63a76 --- /dev/null +++ b/common/src/test/java/org/apache/drill/test/ReproducibleListener.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.test; + +import org.junit.runner.Description; +import org.junit.runner.notification.Failure; +import org.junit.runner.notification.RunListener; + +import com.carrotsearch.randomizedtesting.ReproduceErrorMessageBuilder; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * When tests fail, this class prints a helpful message with the exact maven + * command to re-run the test with the same random seed. + */ +public class ReproducibleListener extends RunListener { + + private static final Logger logger = LoggerFactory.getLogger(ReproducibleListener.class); + + @Override + public void testStarted(Description description) throws Exception { + // If you find it useful, feel free to add something here + } + + @Override + public void testFinished(Description description) throws Exception { + // If you find it useful, feel free to add something here + } + + @Override + public void testFailure(Failure failure) throws Exception { + + StringBuilder sb = new StringBuilder(); + + sb.append("\n==============================================================\n"); + sb.append("Command line to reproduce test failure with the same random seed:\n\tmvn test"); + ReproduceErrorMessageBuilder mavenMessageBuilder = new ReproduceErrorMessageBuilder(sb); + mavenMessageBuilder.appendAllOpts(failure.getDescription()); + sb.append("\n==============================================================\n"); + + String message = sb.toString(); + System.err.println(message); + logger.error(message); + } +} diff --git a/contrib/pom.xml b/contrib/pom.xml index 8c00e762d48..1cb1d729ec6 100644 --- a/contrib/pom.xml +++ b/contrib/pom.xml @@ -15,7 +15,8 @@ See the License for the specific language governing permissions and limitations under the License. --> - + 4.0.0 drill-root @@ -28,9 +29,95 @@ contrib/Parent Pom pom + + 2.1.16 + .local-${project.version}-execution-hints.log + + + + + com.carrotsearch.randomizedtesting + randomizedtesting-runner + ${carrotsearch.testframework} + test + + + com.carrotsearch.randomizedtesting + junit4-maven-plugin + ${carrotsearch.testframework} + test + + + + commons-logging + commons-logging-api + + + + + + + maven-surefire-plugin + 2.15 + + false + true + + + + com.carrotsearch.randomizedtesting + junit4-maven-plugin + ${carrotsearch.testframework} + + true + + + + + + + + + + + tests + test + + junit4 + + + + + + + + + + + + + **/*Tests.class + **/*Test.class + + + + + + + + storage-hbase storage-hive diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/DrillIntegrationTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/exec/DrillIntegrationTestBase.java new file mode 100644 index 00000000000..a197b966db3 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/DrillIntegrationTestBase.java @@ -0,0 +1,486 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec; + +import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; +import com.google.common.io.Files; +import com.google.common.io.Resources; + +import org.apache.drill.QueryTestUtil; +import org.apache.drill.TestBuilder; +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.exceptions.UserException; +import org.apache.drill.exec.client.DrillClient; +import org.apache.drill.exec.exception.SchemaChangeException; +import org.apache.drill.exec.memory.BufferAllocator; +import org.apache.drill.exec.memory.TopLevelAllocator; +import org.apache.drill.exec.proto.UserBitShared; +import org.apache.drill.exec.record.RecordBatchLoader; +import org.apache.drill.exec.rpc.user.ConnectionThrottle; +import org.apache.drill.exec.rpc.user.QueryDataBatch; +import org.apache.drill.exec.rpc.user.UserResultsListener; +import org.apache.drill.exec.server.Drillbit; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.server.RemoteServiceSet; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.util.TestUtilities; +import org.apache.drill.exec.util.VectorUtil; +import org.apache.drill.test.DrillTestBase; + +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.net.URL; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.core.StringContains.containsString; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * Abstract base class for all Drill integration tests. + */ +public abstract class DrillIntegrationTestBase extends DrillTestBase { + + private static final Logger logger = LoggerFactory.getLogger(DrillIntegrationTestBase.class); + + protected static final String TEMP_SCHEMA = "dfs_test.tmp"; + + private static final String ENABLE_FULL_CACHE = "drill.exec.test.use-full-cache"; + private static final int MAX_WIDTH_PER_NODE = 2; + + protected static DrillClient client; + protected static Drillbit[] bits; + protected static RemoteServiceSet serviceSet; + protected static DrillConfig config; + protected static BufferAllocator allocator; + + @SuppressWarnings("serial") + private static final Properties TEST_CONFIGURATIONS = new Properties() { + { + put(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, "false"); + put(ExecConstants.HTTP_ENABLE, "false"); + } + }; + + /** + * Number of Drillbits in test cluster. Default is 1. + * + * Tests can update the cluster size through {@link #updateTestCluster(int, DrillConfig)} + */ + private static int drillbitCount = 1; + + /** + * Location of the dfs_test.tmp schema on local filesystem. + */ + private static String dfsTestTmpSchemaLocation; + + private int[] columnWidths = new int[] { 8 }; + + @BeforeClass + public static void setupDefaultTestCluster() throws Exception { + config = DrillConfig.create(TEST_CONFIGURATIONS); + openClient(); + } + + protected static void updateTestCluster(int newDrillbitCount, DrillConfig newConfig) { + Preconditions.checkArgument(newDrillbitCount > 0, "Number of Drillbits must be at least one"); + if (drillbitCount != newDrillbitCount || config != null) { + // TODO: Currently we have to shutdown the existing Drillbit cluster before starting a new one with the given + // Drillbit count. Revisit later to avoid stopping the cluster. + try { + closeClient(); + drillbitCount = newDrillbitCount; + if (newConfig != null) { + // For next test class, updated DrillConfig will be replaced by default DrillConfig in BaseTestQuery as part + // of the @BeforeClass method of test class. + config = newConfig; + } + openClient(); + } catch(Exception e) { + throw new RuntimeException("Failure while updating the test Drillbit cluster.", e); + } + } + } + + /** + * Useful for tests that require a DrillbitContext to get/add storage plugins, options etc. + * + * @return DrillbitContext of first Drillbit in the cluster. + */ + protected static DrillbitContext getDrillbitContext() { + Preconditions.checkState(bits != null && bits[0] != null, "Drillbits are not setup."); + return bits[0].getContext(); + } + + protected static Properties cloneDefaultTestConfigProperties() { + final Properties props = new Properties(); + for(String propName : TEST_CONFIGURATIONS.stringPropertyNames()) { + props.put(propName, TEST_CONFIGURATIONS.getProperty(propName)); + } + + return props; + } + + protected static String getDfsTestTmpSchemaLocation() { + return dfsTestTmpSchemaLocation; + } + + private static void resetClientAndBit() throws Exception{ + closeClient(); + openClient(); + } + + private static void openClient() throws Exception { + allocator = new TopLevelAllocator(config); + if (config.hasPath(ENABLE_FULL_CACHE) && config.getBoolean(ENABLE_FULL_CACHE)) { + serviceSet = RemoteServiceSet.getServiceSetWithFullCache(config, allocator); + } else { + serviceSet = RemoteServiceSet.getLocalServiceSet(); + } + + dfsTestTmpSchemaLocation = TestUtilities.createTempDir(); + + bits = new Drillbit[drillbitCount]; + for(int i = 0; i < drillbitCount; i++) { + bits[i] = new Drillbit(config, serviceSet); + bits[i].run(); + + final StoragePluginRegistry pluginRegistry = bits[i].getContext().getStorage(); + TestUtilities.updateDfsTestTmpSchemaLocation(pluginRegistry, dfsTestTmpSchemaLocation); + TestUtilities.makeDfsTmpSchemaImmutable(pluginRegistry); + } + + client = QueryTestUtil.createClient(config, serviceSet, MAX_WIDTH_PER_NODE, null); + } + + /** + * Close the current client and open a new client using the given properties. All tests executed + * after this method call use the new client. + */ + public static void updateClient(Properties properties) throws Exception { + Preconditions.checkState(bits != null && bits[0] != null, "Drillbits are not setup."); + if (client != null) { + client.close(); + client = null; + } + + client = QueryTestUtil.createClient(config, serviceSet, MAX_WIDTH_PER_NODE, properties); + } + + /* + * Close the current client and open a new client for the given user. All tests executed + * after this method call use the new client. + * @param user + */ + public static void updateClient(String user) throws Exception { + final Properties props = new Properties(); + props.setProperty("user", user); + updateClient(props); + } + + protected static BufferAllocator getAllocator() { + return allocator; + } + + public static TestBuilder newTest() { + return testBuilder(); + } + + public static TestBuilder testBuilder() { + return new TestBuilder(allocator); + } + + @AfterClass + public static void closeClient() throws IOException, InterruptedException { + if (client != null) { + client.close(); + } + + if (bits != null) { + for(Drillbit bit : bits) { + if (bit != null) { + bit.close(); + } + } + } + + if(serviceSet != null) { + serviceSet.close(); + } + if (allocator != null) { + allocator.close(); + } + } + + @AfterClass + public static void resetDrillbitCount() { + // some test classes assume this value to be 1 and will fail if run along other tests that increase it + drillbitCount = 1; + } + + protected static void runSQL(String sql) throws Exception { + SilentListener listener = new SilentListener(); + testWithListener(UserBitShared.QueryType.SQL, sql, listener); + listener.waitForCompletion(); + } + + protected static List testSqlWithResults(String sql) throws Exception{ + return testRunAndReturn(UserBitShared.QueryType.SQL, sql); + } + + protected static List testLogicalWithResults(String logical) throws Exception{ + return testRunAndReturn(UserBitShared.QueryType.LOGICAL, logical); + } + + protected static List testPhysicalWithResults(String physical) throws Exception{ + return testRunAndReturn(UserBitShared.QueryType.PHYSICAL, physical); + } + + public static List testRunAndReturn(UserBitShared.QueryType type, String query) throws Exception{ + query = QueryTestUtil.normalizeQuery(query); + return client.runQuery(type, query); + } + + public static int testRunAndPrint(final UserBitShared.QueryType type, final String query) throws Exception { + return QueryTestUtil.testRunAndPrint(client, type, query); + } + + protected static void testWithListener(UserBitShared.QueryType type, String query, UserResultsListener resultListener) { + QueryTestUtil.testWithListener(client, type, query, resultListener); + } + + protected static void testNoResult(String query, Object... args) throws Exception { + testNoResult(1, query, args); + } + + protected static void testNoResult(int interation, String query, Object... args) throws Exception { + query = String.format(query, args); + logger.debug("Running query:\n--------------\n" + query); + for (int i = 0; i < interation; i++) { + List results = client.runQuery(UserBitShared.QueryType.SQL, query); + for (QueryDataBatch queryDataBatch : results) { + queryDataBatch.release(); + } + } + } + + public static void test(String query, Object... args) throws Exception { + QueryTestUtil.test(client, String.format(query, args)); + } + + public static void test(final String query) throws Exception { + QueryTestUtil.test(client, query); + } + + protected static int testLogical(String query) throws Exception{ + return testRunAndPrint(UserBitShared.QueryType.LOGICAL, query); + } + + protected static int testPhysical(String query) throws Exception{ + return testRunAndPrint(UserBitShared.QueryType.PHYSICAL, query); + } + + protected static int testSql(String query) throws Exception{ + return testRunAndPrint(UserBitShared.QueryType.SQL, query); + } + + protected static void testPhysicalFromFile(String file) throws Exception{ + testPhysical(getFile(file)); + } + + protected static List testPhysicalFromFileWithResults(String file) throws Exception { + return testRunAndReturn(UserBitShared.QueryType.PHYSICAL, getFile(file)); + } + + protected static void testLogicalFromFile(String file) throws Exception{ + testLogical(getFile(file)); + } + + protected static void testSqlFromFile(String file) throws Exception{ + test(getFile(file)); + } + + /** + * Utility method which tests given query produces a {@link UserException} and the exception message contains + * the given message. + * @param testSqlQuery Test query + * @param expectedErrorMsg Expected error message. + */ + protected static void errorMsgTestHelper(final String testSqlQuery, final String expectedErrorMsg) throws Exception { + UserException expException = null; + try { + test(testSqlQuery); + } catch (final UserException ex) { + expException = ex; + } + + assertNotNull("Expected a UserException", expException); + assertThat(expException.getMessage(), containsString(expectedErrorMsg)); + } + + /** + * Utility method which tests given query produces a {@link UserException} + * with {@link org.apache.drill.exec.proto.UserBitShared.DrillPBError.ErrorType} being DrillPBError.ErrorType.PARSE + * the given message. + * @param testSqlQuery Test query + */ + protected static void parseErrorHelper(final String testSqlQuery) throws Exception { + errorMsgTestHelper(testSqlQuery, UserBitShared.DrillPBError.ErrorType.PARSE.name()); + } + + public static String getFile(String resource) throws IOException{ + URL url = Resources.getResource(resource); + if (url == null) { + throw new IOException(String.format("Unable to find path %s.", resource)); + } + return Resources.toString(url, Charsets.UTF_8); + } + + /** + * Copy the resource (ex. file on classpath) to a physical file on FileSystem. + * @throws IOException + */ + public static String getPhysicalFileFromResource(final String resource) throws IOException { + final File file = File.createTempFile("tempfile", ".txt"); + file.deleteOnExit(); + PrintWriter printWriter = new PrintWriter(file); + printWriter.write(DrillIntegrationTestBase.getFile(resource)); + printWriter.close(); + + return file.getPath(); + } + + /** + * Create a temp directory to store the given dirName + * @return Full path including temp parent directory and given directory name. + */ + public static String getTempDir(final String dirName) { + File dir = Files.createTempDir(); + dir.deleteOnExit(); + + return dir.getAbsolutePath() + File.separator + dirName; + } + + + protected static void setSessionOption(final String option, final String value) { + try { + runSQL(String.format("alter session set `%s` = %s", option, value)); + } catch(final Exception e) { + fail(String.format("Failed to set session option `%s` = %s, Error: %s", option, value, e.toString())); + } + } + + private static class SilentListener implements UserResultsListener { + private volatile UserException exception; + private AtomicInteger count = new AtomicInteger(); + private CountDownLatch latch = new CountDownLatch(1); + + @Override + public void submissionFailed(UserException ex) { + exception = ex; + System.out.println("Query failed: " + ex.getMessage()); + latch.countDown(); + } + + @Override + public void queryCompleted(UserBitShared.QueryResult.QueryState state) { + System.out.println("Query completed successfully with row count: " + count.get()); + latch.countDown(); + } + + @Override + public void dataArrived(QueryDataBatch result, ConnectionThrottle throttle) { + int rows = result.getHeader().getRowCount(); + if (result.getData() != null) { + count.addAndGet(rows); + } + result.release(); + } + + @Override + public void queryIdArrived(UserBitShared.QueryId queryId) {} + + public int waitForCompletion() throws Exception { + latch.await(); + if (exception != null) { + throw exception; + } + return count.get(); + } + } + + protected void setColumnWidth(int columnWidth) { + this.columnWidths = new int[] { columnWidth }; + } + + protected void setColumnWidths(int[] columnWidths) { + this.columnWidths = columnWidths; + } + + protected int printResult(List results) throws SchemaChangeException { + int rowCount = 0; + RecordBatchLoader loader = new RecordBatchLoader(getAllocator()); + for(QueryDataBatch result : results) { + rowCount += result.getHeader().getRowCount(); + loader.load(result.getHeader().getDef(), result.getData()); + // TODO: Clean: DRILL-2933: That load(...) no longer throws + // SchemaChangeException, so check/clean throw clause above. + if (loader.getRecordCount() <= 0) { + continue; + } + VectorUtil.showVectorAccessibleContent(loader, columnWidths); + loader.clear(); + result.release(); + } + System.out.println("Total record count: " + rowCount); + return rowCount; + } + + protected static String getResultString(List results, String delimiter) + throws SchemaChangeException { + StringBuilder formattedResults = new StringBuilder(); + boolean includeHeader = true; + RecordBatchLoader loader = new RecordBatchLoader(getAllocator()); + for(QueryDataBatch result : results) { + loader.load(result.getHeader().getDef(), result.getData()); + if (loader.getRecordCount() <= 0) { + continue; + } + VectorUtil.appendVectorAccessibleContent(loader, formattedResults, delimiter, includeHeader); + if (!includeHeader) { + includeHeader = false; + } + loader.clear(); + result.release(); + } + + return formattedResults.toString(); + } +} From d894859c872b82738e218e06774351cf76d2debc Mon Sep 17 00:00:00 2001 From: andrew Date: Thu, 27 Aug 2015 22:08:33 -0700 Subject: [PATCH 2/5] Cluster scope annotations. --- .../drill/exec/DrillIntegrationTestBase.java | 87 ++++++++++++++++++- 1 file changed, 86 insertions(+), 1 deletion(-) diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/DrillIntegrationTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/exec/DrillIntegrationTestBase.java index a197b966db3..e836debf7b2 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/DrillIntegrationTestBase.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/DrillIntegrationTestBase.java @@ -52,6 +52,11 @@ import java.io.File; import java.io.IOException; import java.io.PrintWriter; +import java.lang.annotation.Annotation; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; import java.net.URL; import java.util.List; import java.util.Properties; @@ -64,7 +69,7 @@ import static org.junit.Assert.fail; /** - * Abstract base class for all Drill integration tests. + * Abstract base class for integration tests. */ public abstract class DrillIntegrationTestBase extends DrillTestBase { @@ -89,6 +94,86 @@ public abstract class DrillIntegrationTestBase extends DrillTestBase { } }; + /* *** Cluster Scope Settings *** */ + + private static final int DEFAULT_MIN_NUM_DRILLBITS = 1; + private static final int DEFAULT_MAX_NUM_DRILLBITS = 3; + + @Retention(RetentionPolicy.RUNTIME) + @Target({ElementType.TYPE}) + public @interface ClusterScope { + + /** + * Returns the scope. {@link Scope#SUITE} is default. + */ + Scope scope() default Scope.SUITE; + + /** + * Returns the number of drillbits in the cluster. Default is -1 which means + * a random number of drillbits is used, where the minimum and maximum number of nodes + * are either the specified ones or the default ones if not specified. + */ + int numDrillbits() default -1; + + /** + * Returns the minimum number of drillbits in the cluster. Default is -1. + * Ignored when {@link ClusterScope#numDrillbits()} is set. + */ + int minNumDrillbits() default -1; + + /** + * Returns the maximum number of drillbits in the cluster. Default is -1. + * Ignored when {@link ClusterScope#numDrillbits()} is set. + */ + int maxNumDrillbits() default -1; + } + + /** + * The scope of a test cluster used together with + * {@link ClusterScope} annotations on {@link DrillIntegrationTestBase} subclasses. + */ + public enum Scope { + /** A cluster shared across all methods in a single test suite */ + SUITE, + /** A cluster that is exclusive to an individual test */ + TEST + } + + private Scope getCurrentClusterScope() { + return getCurrentClusterScope(this.getClass()); + } + + private static Scope getCurrentClusterScope(Class clazz) { + ClusterScope annotation = getAnnotation(clazz, ClusterScope.class); + return annotation == null ? Scope.SUITE : annotation.scope(); + } + + private int getNumDrillbits() { + ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class); + return annotation == null ? -1 : annotation.numDrillbits(); + } + + private int getMinNumDrillbits() { + ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class); + return annotation == null || annotation.minNumDrillbits() == -1 ? DEFAULT_MIN_NUM_DRILLBITS : annotation.minNumDrillbits(); + } + + private int getMaxNumDrillbits() { + ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class); + return annotation == null || annotation.maxNumDrillbits() == -1 ? DEFAULT_MAX_NUM_DRILLBITS : annotation.maxNumDrillbits(); + } + + private static A getAnnotation(Class clazz, Class annotationClass) { + if (clazz == Object.class || clazz == DrillIntegrationTestBase.class) { + return null; + } + A annotation = clazz.getAnnotation(annotationClass); + if (annotation != null) { + return annotation; + } + return getAnnotation(clazz.getSuperclass(), annotationClass); + } + /** * Number of Drillbits in test cluster. Default is 1. * From ba0a66de8bf3ca9c6a6cef87966994f2cb41d8f3 Mon Sep 17 00:00:00 2001 From: andrew Date: Sat, 29 Aug 2015 08:53:32 -0700 Subject: [PATCH 3/5] Checkpoint --- common/pom.xml | 13 + .../org/apache/drill/test/DrillTestBase.java | 14 + .../hbase/BaseHBaseTest_NewTestFramework.java | 105 ++++++ .../apache/drill/hbase/IntegrationTest_A.java | 44 +++ .../apache/drill/hbase/IntegrationTest_B.java | 43 +++ ...HBaseProjectPushDown_NewTestFramework.java | 90 +++++ .../src/test/resources/logback.xml | 5 + .../java/org/apache/drill/BaseTestQuery.java | 6 +- .../drill/exec/DrillIntegrationTestBase.java | 327 ++++++++---------- 9 files changed, 459 insertions(+), 188 deletions(-) create mode 100644 contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest_NewTestFramework.java create mode 100644 contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_A.java create mode 100644 contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_B.java create mode 100644 contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown_NewTestFramework.java diff --git a/common/pom.xml b/common/pom.xml index 8dbfecdcfda..8f09a14af4a 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -30,6 +30,7 @@ Common (Logical Plan, Base expressions) + 5.2.1 2.1.16 .local-${project.version}-execution-hints.log @@ -125,6 +126,18 @@ + + org.apache.lucene + lucene-test-framework + ${lucene.version} + + + + com.carrotsearch.randomizedtesting + junit4-ant + + + diff --git a/common/src/test/java/org/apache/drill/test/DrillTestBase.java b/common/src/test/java/org/apache/drill/test/DrillTestBase.java index 46719a6907b..86f3d56fc2a 100644 --- a/common/src/test/java/org/apache/drill/test/DrillTestBase.java +++ b/common/src/test/java/org/apache/drill/test/DrillTestBase.java @@ -21,6 +21,7 @@ import org.junit.After; import org.junit.BeforeClass; import org.junit.AfterClass; +import org.junit.ClassRule; import org.junit.Rule; import org.junit.rules.TestName; import org.junit.rules.TestRule; @@ -28,6 +29,8 @@ import org.junit.runner.Description; import org.junit.runner.RunWith; +import org.apache.lucene.util.TestRuleStoreClassName; + import com.carrotsearch.randomizedtesting.RandomizedTest; import com.carrotsearch.randomizedtesting.RandomizedRunner; import com.carrotsearch.randomizedtesting.RandomizedContext; @@ -77,6 +80,9 @@ public abstract class DrillTestBase { @Rule public final TestRule REPEAT_RULE = TestTools.getRepeatRule(false); @Rule public TestName TEST_NAME = new TestName(); + @ClassRule + public static final TestRuleStoreClassName classNameRule = new TestRuleStoreClassName(); + @BeforeClass public static void initDrillTest() throws Exception { memWatcher = new MemWatcher(); @@ -98,6 +104,10 @@ public void afterDrillTestBase() { // Optionally add anything here that needs to be cleared after each test. } + public static Class getTestClass() { + return classNameRule.getTestClass(); + } + /* *** Randomization Utilities *** */ public static Random random() { @@ -136,6 +146,10 @@ public static int randomInt(int max) { return RandomizedTest.randomInt(max); } + public static int randomIntBetween(int min, int max) { + return RandomizedTest.randomIntBetween(min, max); + } + public static String randomAscii() { return RandomizedTest.randomAsciiOfLength(32); } diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest_NewTestFramework.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest_NewTestFramework.java new file mode 100644 index 00000000000..10e5593fc6c --- /dev/null +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest_NewTestFramework.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.hbase; + +import com.google.common.base.Charsets; +import com.google.common.io.Files; +import org.apache.drill.BaseTestQuery; +import org.apache.drill.common.util.FileUtils; +import org.apache.drill.exec.DrillIntegrationTestBase; +import org.apache.drill.exec.exception.SchemaChangeException; +import org.apache.drill.exec.rpc.user.QueryDataBatch; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.store.hbase.HBaseStoragePlugin; +import org.apache.drill.exec.store.hbase.HBaseStoragePluginConfig; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.List; + +public class BaseHBaseTest_NewTestFramework extends DrillIntegrationTestBase { + + private static final String HBASE_STORAGE_PLUGIN_NAME = "hbase"; + + protected static Configuration conf = HBaseConfiguration.create(); + + protected static HBaseStoragePlugin storagePlugin; + + protected static HBaseStoragePluginConfig storagePluginConfig; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + /* + * Change the following to HBaseTestsSuite.configure(false, true) + * if you want to test against an externally running HBase cluster. + */ + HBaseTestsSuite.configure(true, true); + HBaseTestsSuite.initCluster(); + + final StoragePluginRegistry pluginRegistry = getCluster().randomDrillBitContext().getStorage(); + storagePlugin = (HBaseStoragePlugin) pluginRegistry.getPlugin(HBASE_STORAGE_PLUGIN_NAME); + storagePluginConfig = storagePlugin.getConfig(); + storagePluginConfig.setEnabled(true); + storagePluginConfig.setZookeeperPort(HBaseTestsSuite.getZookeeperPort()); + pluginRegistry.createOrUpdate(HBASE_STORAGE_PLUGIN_NAME, storagePluginConfig, true); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + HBaseTestsSuite.tearDownCluster(); + } + + protected String getPlanText(String planFile, String tableName) throws IOException { + return Files.toString(FileUtils.getResourceAsFile(planFile), Charsets.UTF_8) + .replaceFirst("\"hbase\\.zookeeper\\.property\\.clientPort\".*:.*\\d+", "\"hbase.zookeeper.property.clientPort\" : " + HBaseTestsSuite.getZookeeperPort()) + .replace("[TABLE_NAME]", tableName); + } + + protected void runHBasePhysicalVerifyCount(String planFile, String tableName, int expectedRowCount) throws Exception{ + String physicalPlan = getPlanText(planFile, tableName); + List results = testPhysicalWithResults(physicalPlan); + printResultAndVerifyRowCount(results, expectedRowCount); + } + + protected List runHBaseSQLlWithResults(String sql) throws Exception { + sql = canonizeHBaseSQL(sql); + System.out.println("Running query:\n" + sql); + return testSqlWithResults(sql); + } + + protected void runHBaseSQLVerifyCount(String sql, int expectedRowCount) throws Exception{ + List results = runHBaseSQLlWithResults(sql); + printResultAndVerifyRowCount(results, expectedRowCount); + } + + private void printResultAndVerifyRowCount(List results, int expectedRowCount) throws SchemaChangeException { + int rowCount = printResult(results); + if (expectedRowCount != -1) { + Assert.assertEquals(expectedRowCount, rowCount); + } + } + + protected String canonizeHBaseSQL(String sql) { + return sql.replace("[TABLE_NAME]", HBaseTestsSuite.TEST_TABLE_1); + } + +} diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_A.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_A.java new file mode 100644 index 00000000000..da950927036 --- /dev/null +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_A.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.hbase; + +import org.apache.drill.exec.DrillIntegrationTestBase; +import org.junit.Test; + +/** + * + */ +@DrillIntegrationTestBase.ClusterScope(scope = DrillIntegrationTestBase.Scope.SUITE, bits = -1, width = -1) +public class IntegrationTest_A extends DrillIntegrationTestBase { + + @Test + public void a_x() throws Exception { + System.out.println("\nTEST A_X()"); + } + + @Test + public void a_y() throws Exception { + System.out.println("\nTEST A_Y()"); + } + + @Test + public void a_z() throws Exception { + System.out.println("\nTEST A_Z()"); + } + +} diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_B.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_B.java new file mode 100644 index 00000000000..0d79e75640a --- /dev/null +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_B.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.hbase; + +import org.apache.drill.exec.DrillIntegrationTestBase; +import org.junit.Test; + +/** + * + */ +@DrillIntegrationTestBase.ClusterScope(scope = DrillIntegrationTestBase.Scope.SUITE, bits = -1, width = -1) +public class IntegrationTest_B extends DrillIntegrationTestBase { + + @Test + public void b_x() throws Exception { + System.out.println("\nTEST B_X()"); + } + + @Test + public void b_y() throws Exception { + System.out.println("\nTEST B_Y()"); + } + + @Test + public void b_z() throws Exception { + System.out.println("\nTEST B_Z()"); + } +} diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown_NewTestFramework.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown_NewTestFramework.java new file mode 100644 index 00000000000..30c866a2de0 --- /dev/null +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown_NewTestFramework.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.hbase; + +import org.apache.drill.exec.DrillIntegrationTestBase.Scope; +import org.apache.drill.exec.DrillIntegrationTestBase.ClusterScope; + +import org.junit.Test; + +@ClusterScope(scope = Scope.SUITE, bits = 1, width = -1) +public class TestHBaseProjectPushDown_NewTestFramework extends BaseHBaseTest_NewTestFramework { + + @Test + public void x() throws Exception { + setColumnWidth(8); + System.out.println("\n\nBLAH BLAH BLAH TEST X"); + } + + @Test + public void y() throws Exception { + setColumnWidth(8); + System.out.println("\n\nBLAH BLAH BLAH TEST Y"); + } + + @Test + public void z() throws Exception { + setColumnWidth(8); + System.out.println("\n\nBLAH BLAH BLAH TEST Z"); + } + + @org.junit.Ignore + @Test + public void testRowKeyPushDown() throws Exception{ + setColumnWidth(8); + runHBaseSQLVerifyCount("SELECT\n" + + "row_key\n" + + "FROM\n" + + " hbase.`[TABLE_NAME]` tableName" + , 7); + } + + @org.junit.Ignore + @Test + public void testColumnWith1RowPushDown() throws Exception{ + setColumnWidth(6); + runHBaseSQLVerifyCount("SELECT\n" + + "t.f2.c7 as `t.f2.c7`\n" + + "FROM\n" + + " hbase.`[TABLE_NAME]` t" + , 1); + } + + @org.junit.Ignore + @Test + public void testRowKeyAndColumnPushDown() throws Exception{ + setColumnWidths(new int[] {8, 9, 6, 2, 6}); + runHBaseSQLVerifyCount("SELECT\n" + + "row_key, t.f.c1*31 as `t.f.c1*31`, t.f.c2 as `t.f.c2`, 5 as `5`, 'abc' as `'abc'`\n" + + "FROM\n" + + " hbase.`[TABLE_NAME]` t" + , 7); + } + + @org.junit.Ignore + @Test + public void testColumnFamilyPushDown() throws Exception{ + setColumnWidths(new int[] {8, 74, 38}); + runHBaseSQLVerifyCount("SELECT\n" + + "row_key, f, f2\n" + + "FROM\n" + + " hbase.`[TABLE_NAME]` tableName" + , 7); + } + +} diff --git a/contrib/storage-hbase/src/test/resources/logback.xml b/contrib/storage-hbase/src/test/resources/logback.xml index 6ef172b0613..5d22c9c83b2 100644 --- a/contrib/storage-hbase/src/test/resources/logback.xml +++ b/contrib/storage-hbase/src/test/resources/logback.xml @@ -51,6 +51,11 @@ + + + + + diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java index d428920ecdc..ae408194584 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java +++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java @@ -59,13 +59,17 @@ import com.google.common.base.Preconditions; import com.google.common.io.Resources; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import static org.hamcrest.core.StringContains.containsString; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; public class BaseTestQuery extends ExecTest { - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseTestQuery.class); + + private static final Logger logger = LoggerFactory.getLogger(BaseTestQuery.class); protected static final String TEMP_SCHEMA = "dfs_test.tmp"; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/DrillIntegrationTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/exec/DrillIntegrationTestBase.java index e836debf7b2..82c9b0a7514 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/DrillIntegrationTestBase.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/DrillIntegrationTestBase.java @@ -18,12 +18,10 @@ package org.apache.drill.exec; import com.google.common.base.Charsets; -import com.google.common.base.Preconditions; import com.google.common.io.Files; import com.google.common.io.Resources; import org.apache.drill.QueryTestUtil; -import org.apache.drill.TestBuilder; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.client.DrillClient; @@ -43,12 +41,15 @@ import org.apache.drill.exec.util.VectorUtil; import org.apache.drill.test.DrillTestBase; +import org.junit.After; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.Closeable; import java.io.File; import java.io.IOException; import java.io.PrintWriter; @@ -76,16 +77,15 @@ public abstract class DrillIntegrationTestBase extends DrillTestBase { private static final Logger logger = LoggerFactory.getLogger(DrillIntegrationTestBase.class); protected static final String TEMP_SCHEMA = "dfs_test.tmp"; - private static final String ENABLE_FULL_CACHE = "drill.exec.test.use-full-cache"; - private static final int MAX_WIDTH_PER_NODE = 2; - protected static DrillClient client; - protected static Drillbit[] bits; - protected static RemoteServiceSet serviceSet; - protected static DrillConfig config; protected static BufferAllocator allocator; + private static Scope scope; + private static DrillTestCluster cluster; + + private int[] columnWidths = new int[] { 8 }; + @SuppressWarnings("serial") private static final Properties TEST_CONFIGURATIONS = new Properties() { { @@ -96,8 +96,11 @@ public abstract class DrillIntegrationTestBase extends DrillTestBase { /* *** Cluster Scope Settings *** */ - private static final int DEFAULT_MIN_NUM_DRILLBITS = 1; - private static final int DEFAULT_MAX_NUM_DRILLBITS = 3; + public static final int MIN_NUM_DRILLBITS = 1; + public static final int MAX_NUM_DRILLBITS = 3; + + public static final int MIN_PARALLELIZATION_WIDTH = 1; + public static final int MAX_PARALLELIZATION_WIDTH = 3; @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.TYPE}) @@ -109,23 +112,25 @@ public abstract class DrillIntegrationTestBase extends DrillTestBase { Scope scope() default Scope.SUITE; /** - * Returns the number of drillbits in the cluster. Default is -1 which means - * a random number of drillbits is used, where the minimum and maximum number of nodes - * are either the specified ones or the default ones if not specified. + * Sets the number of drillbits to create in the test cluster. + * + * Default is -1, which indicates that the test framework should choose a + * random value in the inclusive range [{@link DrillIntegrationTestBase#MIN_NUM_DRILLBITS}, + * {@link DrillIntegrationTestBase#MAX_NUM_DRILLBITS}]. */ - int numDrillbits() default -1; + int bits() default -1; /** - * Returns the minimum number of drillbits in the cluster. Default is -1. - * Ignored when {@link ClusterScope#numDrillbits()} is set. + * Sets the value of {@link ExecConstants#MAX_WIDTH_PER_NODE_KEY}, which determines + * the level of parallelization per drillbit. + * + * Default is -1, which indicates that the test framework should choose a + * random value in the inclusive range [{@link DrillIntegrationTestBase#MIN_PARALLELIZATION_WIDTH}, + * {@link DrillIntegrationTestBase#MAX_PARALLELIZATION_WIDTH}]. */ - int minNumDrillbits() default -1; + int width() default -1; - /** - * Returns the maximum number of drillbits in the cluster. Default is -1. - * Ignored when {@link ClusterScope#numDrillbits()} is set. - */ - int maxNumDrillbits() default -1; + // XXX - WHAT IF THESE VALUES CHANGE FROM ONE CLASS TO THE NEXT? WE HAVE TO UPDATE THE CLUSTER } /** @@ -139,28 +144,19 @@ public enum Scope { TEST } - private Scope getCurrentClusterScope() { - return getCurrentClusterScope(this.getClass()); - } - private static Scope getCurrentClusterScope(Class clazz) { ClusterScope annotation = getAnnotation(clazz, ClusterScope.class); return annotation == null ? Scope.SUITE : annotation.scope(); } - private int getNumDrillbits() { - ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class); - return annotation == null ? -1 : annotation.numDrillbits(); + private static int getNumDrillbits() { + ClusterScope annotation = getAnnotation(getTestClass(), ClusterScope.class); + return annotation == null ? -1 : annotation.bits(); } - private int getMinNumDrillbits() { - ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class); - return annotation == null || annotation.minNumDrillbits() == -1 ? DEFAULT_MIN_NUM_DRILLBITS : annotation.minNumDrillbits(); - } - - private int getMaxNumDrillbits() { - ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class); - return annotation == null || annotation.maxNumDrillbits() == -1 ? DEFAULT_MAX_NUM_DRILLBITS : annotation.maxNumDrillbits(); + private static int getParallelizationWidth() { + ClusterScope annotation = getAnnotation(getTestClass(), ClusterScope.class); + return annotation == null ? -1 : annotation.width(); } private static A getAnnotation(Class clazz, Class annotationClass) { @@ -174,160 +170,138 @@ private static A getAnnotation(Class clazz, Class a return getAnnotation(clazz.getSuperclass(), annotationClass); } - /** - * Number of Drillbits in test cluster. Default is 1. - * - * Tests can update the cluster size through {@link #updateTestCluster(int, DrillConfig)} - */ - private static int drillbitCount = 1; - - /** - * Location of the dfs_test.tmp schema on local filesystem. - */ - private static String dfsTestTmpSchemaLocation; + protected static BufferAllocator getAllocator() { + return allocator; + } - private int[] columnWidths = new int[] { 8 }; + protected static DrillTestCluster getCluster() { + return cluster; + } @BeforeClass - public static void setupDefaultTestCluster() throws Exception { - config = DrillConfig.create(TEST_CONFIGURATIONS); - openClient(); - } - - protected static void updateTestCluster(int newDrillbitCount, DrillConfig newConfig) { - Preconditions.checkArgument(newDrillbitCount > 0, "Number of Drillbits must be at least one"); - if (drillbitCount != newDrillbitCount || config != null) { - // TODO: Currently we have to shutdown the existing Drillbit cluster before starting a new one with the given - // Drillbit count. Revisit later to avoid stopping the cluster. - try { - closeClient(); - drillbitCount = newDrillbitCount; - if (newConfig != null) { - // For next test class, updated DrillConfig will be replaced by default DrillConfig in BaseTestQuery as part - // of the @BeforeClass method of test class. - config = newConfig; - } - openClient(); - } catch(Exception e) { - throw new RuntimeException("Failure while updating the test Drillbit cluster.", e); - } + public static void beforeClass() throws Exception { + + logger.debug("Initializing test framework for suite: {}", getTestClass()); + + if (cluster != null) { + logger.info("Re-using existing test cluster of {} drillbits", cluster.bits().length); + return; } - } - /** - * Useful for tests that require a DrillbitContext to get/add storage plugins, options etc. - * - * @return DrillbitContext of first Drillbit in the cluster. - */ - protected static DrillbitContext getDrillbitContext() { - Preconditions.checkState(bits != null && bits[0] != null, "Drillbits are not setup."); - return bits[0].getContext(); - } + allocator = new TopLevelAllocator(DrillConfig.create(TEST_CONFIGURATIONS)); + scope = getCurrentClusterScope(getTestClass()); - protected static Properties cloneDefaultTestConfigProperties() { - final Properties props = new Properties(); - for(String propName : TEST_CONFIGURATIONS.stringPropertyNames()) { - props.put(propName, TEST_CONFIGURATIONS.getProperty(propName)); + int size = getNumDrillbits(); + if (size <= 0 || size > MAX_NUM_DRILLBITS) { + size = randomIntBetween(MIN_NUM_DRILLBITS, MAX_NUM_DRILLBITS); } - return props; - } + int width = getParallelizationWidth(); + if (width <= 0 || width > MAX_PARALLELIZATION_WIDTH) { + width = randomIntBetween(MIN_PARALLELIZATION_WIDTH, MAX_PARALLELIZATION_WIDTH); + } - protected static String getDfsTestTmpSchemaLocation() { - return dfsTestTmpSchemaLocation; + switch (scope) { + case SUITE: + case TEST: + cluster = new DrillTestCluster(size, width, DrillConfig.create(TEST_CONFIGURATIONS)); + break; + default: + fail("Unsupported cluster scope: " + scope); + } } - private static void resetClientAndBit() throws Exception{ - closeClient(); - openClient(); + @Before + public void beforeDrillIntegrationTest() throws Exception { + ; } - private static void openClient() throws Exception { - allocator = new TopLevelAllocator(config); - if (config.hasPath(ENABLE_FULL_CACHE) && config.getBoolean(ENABLE_FULL_CACHE)) { - serviceSet = RemoteServiceSet.getServiceSetWithFullCache(config, allocator); - } else { - serviceSet = RemoteServiceSet.getLocalServiceSet(); + @AfterClass + public static void afterClass() throws IOException, InterruptedException { + if (cluster != null) { + cluster.close(); } + } - dfsTestTmpSchemaLocation = TestUtilities.createTempDir(); + protected static final class DrillTestCluster implements Closeable { - bits = new Drillbit[drillbitCount]; - for(int i = 0; i < drillbitCount; i++) { - bits[i] = new Drillbit(config, serviceSet); - bits[i].run(); + private final Drillbit[] bits; + private final DrillClient client; + private final DrillConfig config; + private final BufferAllocator allocator; + private final RemoteServiceSet serviceSet; + private final String tmpSchemaLocation; - final StoragePluginRegistry pluginRegistry = bits[i].getContext().getStorage(); - TestUtilities.updateDfsTestTmpSchemaLocation(pluginRegistry, dfsTestTmpSchemaLocation); - TestUtilities.makeDfsTmpSchemaImmutable(pluginRegistry); - } + public DrillTestCluster(int size, int width, DrillConfig config) throws Exception { - client = QueryTestUtil.createClient(config, serviceSet, MAX_WIDTH_PER_NODE, null); - } + logger.info("Initializing test cluster of {} drillbits (parallelization: {})", size, width); - /** - * Close the current client and open a new client using the given properties. All tests executed - * after this method call use the new client. - */ - public static void updateClient(Properties properties) throws Exception { - Preconditions.checkState(bits != null && bits[0] != null, "Drillbits are not setup."); - if (client != null) { - client.close(); - client = null; - } + this.config = config; + this.allocator = new TopLevelAllocator(config); + this.tmpSchemaLocation = TestUtilities.createTempDir(); - client = QueryTestUtil.createClient(config, serviceSet, MAX_WIDTH_PER_NODE, properties); - } + if (config.hasPath(ENABLE_FULL_CACHE) && config.getBoolean(ENABLE_FULL_CACHE)) { + serviceSet = RemoteServiceSet.getServiceSetWithFullCache(config, allocator); + } + else { + serviceSet = RemoteServiceSet.getLocalServiceSet(); + } - /* - * Close the current client and open a new client for the given user. All tests executed - * after this method call use the new client. - * @param user - */ - public static void updateClient(String user) throws Exception { - final Properties props = new Properties(); - props.setProperty("user", user); - updateClient(props); - } + bits = new Drillbit[size]; + for (int i = 0; i < bits.length; i++) { + logger.debug("Initializing drillbit #{}", i); + bits[i] = new Drillbit(config, serviceSet); + bits[i].run(); + StoragePluginRegistry registry = bits[i].getContext().getStorage(); + TestUtilities.updateDfsTestTmpSchemaLocation(registry, tmpSchemaLocation); + TestUtilities.makeDfsTmpSchemaImmutable(registry); + } - protected static BufferAllocator getAllocator() { - return allocator; - } + client = QueryTestUtil.createClient(config, serviceSet, width, null); + } - public static TestBuilder newTest() { - return testBuilder(); - } + public DrillClient client() { + return client; + } - public static TestBuilder testBuilder() { - return new TestBuilder(allocator); - } + public Drillbit[] bits() { + return bits; + } - @AfterClass - public static void closeClient() throws IOException, InterruptedException { - if (client != null) { - client.close(); + public DrillConfig config() { + return config; } - if (bits != null) { - for(Drillbit bit : bits) { - if (bit != null) { - bit.close(); - } - } + public BufferAllocator allocator() { + return allocator; + } + + public Drillbit randomBit() { + return randomFrom(bits); + } + + public DrillbitContext randomDrillBitContext() { + return randomBit().getContext(); } - if(serviceSet != null) { - serviceSet.close(); + public RemoteServiceSet serviceSet() { + return serviceSet; } - if (allocator != null) { - allocator.close(); + + @Override + public void close() throws IOException { + cluster.client().close(); + for (Drillbit bit : cluster.bits()) { + bit.close(); + } + cluster.serviceSet().close(); + cluster.allocator().close(); } } - @AfterClass - public static void resetDrillbitCount() { - // some test classes assume this value to be 1 and will fail if run along other tests that increase it - drillbitCount = 1; + @After + public void afterDrillIntegrationTest() { + } protected static void runSQL(String sql) throws Exception { @@ -350,26 +324,26 @@ protected static List testPhysicalWithResults(String physical) t public static List testRunAndReturn(UserBitShared.QueryType type, String query) throws Exception{ query = QueryTestUtil.normalizeQuery(query); - return client.runQuery(type, query); + return cluster.client().runQuery(type, query); } public static int testRunAndPrint(final UserBitShared.QueryType type, final String query) throws Exception { - return QueryTestUtil.testRunAndPrint(client, type, query); + return QueryTestUtil.testRunAndPrint(cluster.client(), type, query); } protected static void testWithListener(UserBitShared.QueryType type, String query, UserResultsListener resultListener) { - QueryTestUtil.testWithListener(client, type, query, resultListener); + QueryTestUtil.testWithListener(cluster.client(), type, query, resultListener); } protected static void testNoResult(String query, Object... args) throws Exception { testNoResult(1, query, args); } - protected static void testNoResult(int interation, String query, Object... args) throws Exception { + protected static void testNoResult(int iteration, String query, Object... args) throws Exception { query = String.format(query, args); logger.debug("Running query:\n--------------\n" + query); - for (int i = 0; i < interation; i++) { - List results = client.runQuery(UserBitShared.QueryType.SQL, query); + for (int i = 0; i < iteration; i++) { + List results = cluster.client().runQuery(UserBitShared.QueryType.SQL, query); for (QueryDataBatch queryDataBatch : results) { queryDataBatch.release(); } @@ -377,11 +351,11 @@ protected static void testNoResult(int interation, String query, Object... args) } public static void test(String query, Object... args) throws Exception { - QueryTestUtil.test(client, String.format(query, args)); + QueryTestUtil.test(cluster.client(), String.format(query, args)); } public static void test(final String query) throws Exception { - QueryTestUtil.test(client, query); + QueryTestUtil.test(cluster.client(), query); } protected static int testLogical(String query) throws Exception{ @@ -547,25 +521,4 @@ protected int printResult(List results) throws SchemaChangeExcep System.out.println("Total record count: " + rowCount); return rowCount; } - - protected static String getResultString(List results, String delimiter) - throws SchemaChangeException { - StringBuilder formattedResults = new StringBuilder(); - boolean includeHeader = true; - RecordBatchLoader loader = new RecordBatchLoader(getAllocator()); - for(QueryDataBatch result : results) { - loader.load(result.getHeader().getDef(), result.getData()); - if (loader.getRecordCount() <= 0) { - continue; - } - VectorUtil.appendVectorAccessibleContent(loader, formattedResults, delimiter, includeHeader); - if (!includeHeader) { - includeHeader = false; - } - loader.clear(); - result.release(); - } - - return formattedResults.toString(); - } } From 99d2289cebaeaef789437f1baa5d0c288e8df73f Mon Sep 17 00:00:00 2001 From: andrew Date: Sat, 29 Aug 2015 13:21:06 -0700 Subject: [PATCH 4/5] Checkpoint --- .../org/apache/drill/test/DrillTestBase.java | 13 ----- .../apache/drill/hbase/IntegrationTest_A.java | 2 +- .../apache/drill/hbase/IntegrationTest_B.java | 2 +- .../drill/exec/DrillIntegrationTestBase.java | 55 +++++++++---------- 4 files changed, 29 insertions(+), 43 deletions(-) diff --git a/common/src/test/java/org/apache/drill/test/DrillTestBase.java b/common/src/test/java/org/apache/drill/test/DrillTestBase.java index 86f3d56fc2a..7df53793ebc 100644 --- a/common/src/test/java/org/apache/drill/test/DrillTestBase.java +++ b/common/src/test/java/org/apache/drill/test/DrillTestBase.java @@ -91,7 +91,6 @@ public static void initDrillTest() throws Exception { @AfterClass public static void finishDrillTest() throws InterruptedException{ logger.info(String.format("Test Class done (%s): %s.", memWatcher.getMemString(true), className)); - LOG_OUTCOME.sleepIfFailure(); } @Before @@ -205,7 +204,6 @@ public String getMemString(boolean runGC) { private static class TestLogReporter extends TestWatcher { private DrillTest.MemWatcher memWatcher; - private int failureCount = 0; @Override protected void starting(Description description) { @@ -217,23 +215,12 @@ protected void starting(Description description) { @Override protected void failed(Throwable e, Description description) { logger.error(String.format("Test Failed (%s): %s", memWatcher.getMemString(), description.getDisplayName()), e); - failureCount++; } @Override public void succeeded(Description description) { logger.info(String.format("Test Succeeded (%s): %s", memWatcher.getMemString(), description.getDisplayName())); } - - public void sleepIfFailure() throws InterruptedException { - if(failureCount > 0){ - Thread.sleep(2000); - failureCount = 0; - } else { - // pause to get logger to catch up. - Thread.sleep(250); - } - } } private static class SystemManager { diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_A.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_A.java index da950927036..098548b2767 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_A.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_A.java @@ -23,7 +23,7 @@ /** * */ -@DrillIntegrationTestBase.ClusterScope(scope = DrillIntegrationTestBase.Scope.SUITE, bits = -1, width = -1) +@DrillIntegrationTestBase.ClusterScope(scope = DrillIntegrationTestBase.Scope.GLOBAL, bits = -1, width = -1) public class IntegrationTest_A extends DrillIntegrationTestBase { @Test diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_B.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_B.java index 0d79e75640a..f7727c64b14 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_B.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_B.java @@ -23,7 +23,7 @@ /** * */ -@DrillIntegrationTestBase.ClusterScope(scope = DrillIntegrationTestBase.Scope.SUITE, bits = -1, width = -1) +@DrillIntegrationTestBase.ClusterScope(scope = DrillIntegrationTestBase.Scope.GLOBAL, bits = -1, width = -1) public class IntegrationTest_B extends DrillIntegrationTestBase { @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/DrillIntegrationTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/exec/DrillIntegrationTestBase.java index 82c9b0a7514..ccbfcb5ff4e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/DrillIntegrationTestBase.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/DrillIntegrationTestBase.java @@ -97,19 +97,19 @@ public abstract class DrillIntegrationTestBase extends DrillTestBase { /* *** Cluster Scope Settings *** */ public static final int MIN_NUM_DRILLBITS = 1; - public static final int MAX_NUM_DRILLBITS = 3; + public static final int MAX_NUM_DRILLBITS = 5; public static final int MIN_PARALLELIZATION_WIDTH = 1; - public static final int MAX_PARALLELIZATION_WIDTH = 3; + public static final int MAX_PARALLELIZATION_WIDTH = 5; @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.TYPE}) public @interface ClusterScope { /** - * Returns the scope. {@link Scope#SUITE} is default. + * Returns the scope. {@link Scope#GLOBAL} is default. */ - Scope scope() default Scope.SUITE; + Scope scope() default Scope.GLOBAL; /** * Sets the number of drillbits to create in the test cluster. @@ -129,8 +129,6 @@ public abstract class DrillIntegrationTestBase extends DrillTestBase { * {@link DrillIntegrationTestBase#MAX_PARALLELIZATION_WIDTH}]. */ int width() default -1; - - // XXX - WHAT IF THESE VALUES CHANGE FROM ONE CLASS TO THE NEXT? WE HAVE TO UPDATE THE CLUSTER } /** @@ -138,15 +136,19 @@ public abstract class DrillIntegrationTestBase extends DrillTestBase { * {@link ClusterScope} annotations on {@link DrillIntegrationTestBase} subclasses. */ public enum Scope { - /** A cluster shared across all methods in a single test suite */ - SUITE, - /** A cluster that is exclusive to an individual test */ - TEST + /** + * A cluster that is shared across all test suites + */ + GLOBAL, + /** + * A cluster shared across all methods in a single test suite, but not shared across test suites + */ + SUITE } private static Scope getCurrentClusterScope(Class clazz) { ClusterScope annotation = getAnnotation(clazz, ClusterScope.class); - return annotation == null ? Scope.SUITE : annotation.scope(); + return annotation == null ? Scope.GLOBAL : annotation.scope(); } private static int getNumDrillbits() { @@ -181,14 +183,14 @@ protected static DrillTestCluster getCluster() { @BeforeClass public static void beforeClass() throws Exception { - logger.debug("Initializing test framework for suite: {}", getTestClass()); + logger.debug("Initializing test framework for class: {}", getTestClass()); + allocator = new TopLevelAllocator(DrillConfig.create(TEST_CONFIGURATIONS)); if (cluster != null) { logger.info("Re-using existing test cluster of {} drillbits", cluster.bits().length); return; } - allocator = new TopLevelAllocator(DrillConfig.create(TEST_CONFIGURATIONS)); scope = getCurrentClusterScope(getTestClass()); int size = getNumDrillbits(); @@ -202,8 +204,8 @@ public static void beforeClass() throws Exception { } switch (scope) { + case GLOBAL: case SUITE: - case TEST: cluster = new DrillTestCluster(size, width, DrillConfig.create(TEST_CONFIGURATIONS)); break; default: @@ -223,6 +225,15 @@ public static void afterClass() throws IOException, InterruptedException { } } + @After + public void afterDrillIntegrationTest() { + logger.info("Executing post-test cleanup: [{}]", getTestClass()); + if (allocator != null) { + allocator.close(); + allocator = null; + } + } + protected static final class DrillTestCluster implements Closeable { private final Drillbit[] bits; @@ -234,18 +245,12 @@ protected static final class DrillTestCluster implements Closeable { public DrillTestCluster(int size, int width, DrillConfig config) throws Exception { - logger.info("Initializing test cluster of {} drillbits (parallelization: {})", size, width); + logger.info("Initializing test cluster of {} drillbits (parallelization width: {})", size, width); this.config = config; this.allocator = new TopLevelAllocator(config); this.tmpSchemaLocation = TestUtilities.createTempDir(); - - if (config.hasPath(ENABLE_FULL_CACHE) && config.getBoolean(ENABLE_FULL_CACHE)) { - serviceSet = RemoteServiceSet.getServiceSetWithFullCache(config, allocator); - } - else { - serviceSet = RemoteServiceSet.getLocalServiceSet(); - } + this.serviceSet = RemoteServiceSet.getLocalServiceSet(); bits = new Drillbit[size]; for (int i = 0; i < bits.length; i++) { @@ -299,11 +304,6 @@ public void close() throws IOException { } } - @After - public void afterDrillIntegrationTest() { - - } - protected static void runSQL(String sql) throws Exception { SilentListener listener = new SilentListener(); testWithListener(UserBitShared.QueryType.SQL, sql, listener); @@ -447,7 +447,6 @@ public static String getTempDir(final String dirName) { return dir.getAbsolutePath() + File.separator + dirName; } - protected static void setSessionOption(final String option, final String value) { try { runSQL(String.format("alter session set `%s` = %s", option, value)); From b9d564fbf96bcd72c31926eff3c44adfb6b37207 Mon Sep 17 00:00:00 2001 From: andrew Date: Wed, 2 Sep 2015 09:28:44 -0700 Subject: [PATCH 5/5] Checkpoint --- common/pom.xml | 57 --- .../common/expression/PathSegmentTests.java | 4 +- .../common/expression/parser/TreeTest.java | 4 +- .../drill/storage/CheckStorageConfig.java | 4 +- .../java/org/apache/drill/test/DrillTest.java | 187 ------- .../org/apache/drill/test/DrillTestBase.java | 351 +++++++------ .../apache/drill/test/StoreClassNameRule.java | 60 +++ contrib/pom.xml | 89 ---- contrib/storage-hbase/pom.xml | 2 + .../org/apache/drill/hbase/BaseHBaseTest.java | 6 +- .../hbase/BaseHBaseTest_NewTestFramework.java | 105 ---- .../apache/drill/hbase/IntegrationTest_A.java | 44 -- .../apache/drill/hbase/IntegrationTest_B.java | 43 -- ...HBaseProjectPushDown_NewTestFramework.java | 90 ---- .../drill/hbase/TestHBaseRegexParser.java | 4 +- .../drill/exec/fn/hive/TestHiveUDFs.java | 4 +- .../store/hive/HiveTestDataGenerator.java | 6 +- ...ery.java => DrillIntegrationTestBase.java} | 468 ++++++++++-------- ...java => DrillIntegrationTestBase_XXX.java} | 64 ++- .../org/apache/drill/DrillTestWrapper.java | 26 +- .../java/org/apache/drill/PlanTestBase.java | 2 +- .../org/apache/drill/TestAggNullable.java | 2 +- .../apache/drill/TestAllocationException.java | 2 +- .../org/apache/drill/TestAltSortQueries.java | 2 +- .../java/org/apache/drill/TestBugFixes.java | 2 +- .../java/org/apache/drill/TestBuilder.java | 8 +- .../drill/TestDisabledFunctionality.java | 2 +- .../org/apache/drill/TestExampleQueries.java | 2 +- .../org/apache/drill/TestFrameworkTest.java | 2 +- .../org/apache/drill/TestFunctionsQuery.java | 2 +- .../java/org/apache/drill/TestInList.java | 3 +- .../org/apache/drill/TestJoinNullable.java | 3 +- .../apache/drill/TestOutOfMemoryOutcome.java | 2 +- .../org/apache/drill/TestSchemaChange.java | 2 +- .../org/apache/drill/TestStarQueries.java | 2 +- .../java/org/apache/drill/TestTextJoin.java | 2 +- .../org/apache/drill/TestTpchDistributed.java | 2 +- .../drill/TestTpchDistributedStreaming.java | 2 +- .../org/apache/drill/TestTpchExplain.java | 2 +- .../java/org/apache/drill/TestTpchLimit0.java | 2 +- .../org/apache/drill/TestTpchSingleMode.java | 2 +- .../java/org/apache/drill/TestUnionAll.java | 2 +- .../org/apache/drill/TestUnionDistinct.java | 2 +- .../java/org/apache/drill/exec/ExecTest.java | 4 +- .../drill/exec/TestQueriesOnLargeFile.java | 4 +- .../drill/exec/TestRepeatedReaders.java | 4 +- .../drill/exec/TestWindowFunctions.java | 4 +- .../exec/compile/TestClassTransformation.java | 4 +- .../compile/TestLargeFileCompilation.java | 4 +- .../drill/exec/expr/TestLogicalExprSerDe.java | 4 +- .../expr/TestSchemaPathMaterialization.java | 4 +- .../expr/fn/impl/TestStringFunctions.java | 4 +- .../exec/fn/impl/TestAggregateFunctions.java | 5 +- .../exec/fn/impl/TestCastEmptyStrings.java | 4 +- .../drill/exec/fn/impl/TestCastFunctions.java | 6 +- .../exec/fn/impl/TestContextFunctions.java | 4 +- .../exec/fn/impl/TestNewDateFunctions.java | 4 +- .../impl/TestNewSimpleRepeatedFunctions.java | 4 +- .../exec/nested/TestFastComplexSchema.java | 4 +- .../exec/nested/TestNestedComplexSchema.java | 4 +- .../physical/impl/TestConvertFunctions.java | 6 +- .../exec/physical/impl/agg/TestHashAggr.java | 4 +- .../impl/broadcastsender/TestBroadcast.java | 4 +- .../impl/filter/TestLargeInClause.java | 4 +- .../physical/impl/flatten/TestFlatten.java | 8 +- .../impl/join/TestHashJoinAdvanced.java | 5 +- .../impl/join/TestMergeJoinAdvanced.java | 4 +- .../impl/limit/TestLimitWithExchanges.java | 4 +- .../exec/physical/impl/sort/TestSort.java | 4 +- .../impl/svremover/TestSVRemover.java | 4 +- .../physical/impl/window/TestWindowFrame.java | 4 +- .../impl/writer/TestParquetWriter.java | 5 +- .../writer/TestParquetWriterEmptyFiles.java | 8 +- .../exec/physical/impl/writer/TestWriter.java | 4 +- .../impl/xsort/TestSimpleExternalSort.java | 4 +- .../security/TestCustomUserAuthenticator.java | 4 +- .../exec/server/TestDrillbitResilience.java | 10 +- .../apache/drill/exec/server/TestOptions.java | 4 +- .../drill/exec/server/TestTpcdsSf1Leaks.java | 4 +- .../drill/exec/sql/TestBaseViewSupport.java | 4 +- .../org/apache/drill/exec/sql/TestCTAS.java | 4 +- .../apache/drill/exec/sql/TestInfoSchema.java | 4 +- .../exec/sql/TestSimpleCastFunctions.java | 4 +- .../apache/drill/exec/sql/TestWithClause.java | 4 +- .../drill/exec/store/TestTimedRunnable.java | 4 +- .../drill/exec/store/avro/AvroFormatTest.java | 4 +- .../apache/drill/exec/store/dfs/TestGlob.java | 4 +- .../Drill2283InfoSchemaVarchar1BugTest.java | 4 +- .../exec/store/json/TestJsonRecordReader.java | 5 +- .../parquet/ParquetRecordReaderTest.java | 4 +- .../store/parquet/TestParquetComplex.java | 5 +- .../exec/store/parquet/TestParquetScan.java | 4 +- .../TestColumnReaderFactory.java | 4 +- .../parquet/columnreaders/TestDateReader.java | 4 +- .../parquet2/TestDrillParquetReader.java | 4 +- .../drill/exec/store/sys/TestSystemTable.java | 4 +- .../exec/store/text/TestNewTextReader.java | 4 +- .../drill/exec/store/text/TestTextColumn.java | 4 +- .../testing/TestCountDownLatchInjection.java | 4 +- .../exec/testing/TestExceptionInjection.java | 4 +- .../exec/testing/TestPauseInjection.java | 8 +- .../drill/exec/testing/TestResourceLeak.java | 4 +- .../vector/complex/TestEmptyPopulation.java | 4 +- .../fn/TestJsonReaderWithSparseFiles.java | 4 +- .../complex/writer/TestComplexToJson.java | 4 +- .../complex/writer/TestComplexTypeReader.java | 4 +- .../complex/writer/TestComplexTypeWriter.java | 4 +- .../complex/writer/TestExtendedTypes.java | 6 +- .../vector/complex/writer/TestJsonReader.java | 4 +- .../exec/work/batch/TestSpoolingBuffer.java | 5 +- .../fragment/TestFragmentExecutorCancel.java | 4 +- .../apache/drill/jdbc/DrillResultSetTest.java | 29 +- .../org/apache/drill/jdbc/DriverTest.java | 4 +- .../TracingProxyDriverClassLoadingTest.java | 4 +- .../jdbc/proxy/TracingProxyDriverTest.java | 4 +- pom.xml | 87 +++- 116 files changed, 832 insertions(+), 1270 deletions(-) delete mode 100644 common/src/test/java/org/apache/drill/test/DrillTest.java create mode 100644 common/src/test/java/org/apache/drill/test/StoreClassNameRule.java delete mode 100644 contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest_NewTestFramework.java delete mode 100644 contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_A.java delete mode 100644 contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_B.java delete mode 100644 contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown_NewTestFramework.java rename exec/java-exec/src/test/java/org/apache/drill/{BaseTestQuery.java => DrillIntegrationTestBase.java} (51%) rename exec/java-exec/src/test/java/org/apache/drill/{exec/DrillIntegrationTestBase.java => DrillIntegrationTestBase_XXX.java} (90%) diff --git a/common/pom.xml b/common/pom.xml index 8f09a14af4a..85235a15be3 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -142,63 +142,6 @@ - - maven-surefire-plugin - 2.15 - - false - true - - - - com.carrotsearch.randomizedtesting - junit4-maven-plugin - ${carrotsearch.testframework} - - true - - - - - - - - - - - tests - test - - junit4 - - - - - - - - - - - - - **/*Tests.class - **/*Test.class - - - - - maven-jar-plugin diff --git a/common/src/test/java/org/apache/drill/common/expression/PathSegmentTests.java b/common/src/test/java/org/apache/drill/common/expression/PathSegmentTests.java index 97cd578c9e9..afa77b38247 100644 --- a/common/src/test/java/org/apache/drill/common/expression/PathSegmentTests.java +++ b/common/src/test/java/org/apache/drill/common/expression/PathSegmentTests.java @@ -19,10 +19,10 @@ import static org.junit.Assert.assertEquals; -import org.apache.drill.test.DrillTest; +import org.apache.drill.test.DrillTestBase; import org.junit.Test; -public class PathSegmentTests extends DrillTest { +public class PathSegmentTests extends DrillTestBase { protected PathSegment makeArraySegment(final int len, final PathSegment tail) { PathSegment node = tail; for (int i = 0; i < len; i++) { diff --git a/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java b/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java index 7cf14777827..73e19d5da11 100644 --- a/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java +++ b/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java @@ -25,10 +25,10 @@ import org.apache.drill.common.expression.ExpressionStringBuilder; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.parser.ExprParser.parse_return; -import org.apache.drill.test.DrillTest; +import org.apache.drill.test.DrillTestBase; import org.junit.Test; -public class TreeTest extends DrillTest { +public class TreeTest extends DrillTestBase { @Test public void testIfWithCase() throws Exception{ diff --git a/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java b/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java index a3a8257dc2b..2234cf16152 100644 --- a/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java +++ b/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java @@ -26,12 +26,12 @@ import org.apache.drill.common.logical.StoragePluginConfig; import org.apache.drill.common.util.FileUtils; import org.apache.drill.common.util.PathScanner; -import org.apache.drill.test.DrillTest; +import org.apache.drill.test.DrillTestBase; import org.junit.Test; import com.google.common.collect.Lists; -public class CheckStorageConfig extends DrillTest { +public class CheckStorageConfig extends DrillTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CheckStorageConfig.class); @Test diff --git a/common/src/test/java/org/apache/drill/test/DrillTest.java b/common/src/test/java/org/apache/drill/test/DrillTest.java deleted file mode 100644 index 95ba9363092..00000000000 --- a/common/src/test/java/org/apache/drill/test/DrillTest.java +++ /dev/null @@ -1,187 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.test; - -import java.lang.management.BufferPoolMXBean; -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryMXBean; -import java.util.List; - -import org.apache.drill.common.util.DrillStringUtils; -import org.apache.drill.common.util.RepeatTestRule; -import org.apache.drill.common.util.TestTools; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.rules.TestName; -import org.junit.rules.TestRule; -import org.junit.rules.TestWatcher; -import org.junit.runner.Description; -import org.slf4j.Logger; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; - -public class DrillTest { -// private static final Logger logger = org.slf4j.LoggerFactory.getLogger(DrillTest.class); - - protected static final ObjectMapper objectMapper; - static { - System.setProperty("line.separator", "\n"); - objectMapper = new ObjectMapper(); - } - - static final SystemManager manager = new SystemManager(); - - static final Logger testReporter = org.slf4j.LoggerFactory.getLogger("org.apache.drill.TestReporter"); - static final TestLogReporter LOG_OUTCOME = new TestLogReporter(); - - static MemWatcher memWatcher; - static String className; - - @Rule public final TestRule TIMEOUT = TestTools.getTimeoutRule(50000); - @Rule public final TestLogReporter logOutcome = LOG_OUTCOME; - - @Rule public final TestRule REPEAT_RULE = TestTools.getRepeatRule(false); - - @Rule public TestName TEST_NAME = new TestName(); - - @Before - public void printID() throws Exception { - System.out.printf("Running %s#%s\n", getClass().getName(), TEST_NAME.getMethodName()); - } - - @BeforeClass - public static void initDrillTest() throws Exception { - memWatcher = new MemWatcher(); - } - - @AfterClass - public static void finiDrillTest() throws InterruptedException{ - testReporter.info(String.format("Test Class done (%s): %s.", memWatcher.getMemString(true), className)); - LOG_OUTCOME.sleepIfFailure(); - } - - protected static class MemWatcher { - private long startDirect; - private long startHeap; - private long startNonHeap; - - public MemWatcher() { - startDirect = manager.getMemDirect(); - startHeap = manager.getMemHeap(); - startNonHeap = manager.getMemNonHeap(); - } - - public Object getMemString() { - return getMemString(false); - } - - public String getMemString(boolean runGC) { - if (runGC) { - Runtime.getRuntime().gc(); - } - long endDirect = manager.getMemDirect(); - long endHeap = manager.getMemHeap(); - long endNonHeap = manager.getMemNonHeap(); - return String.format("d: %s(%s), h: %s(%s), nh: %s(%s)", // - DrillStringUtils.readable(endDirect - startDirect), DrillStringUtils.readable(endDirect), // - DrillStringUtils.readable(endHeap - startHeap), DrillStringUtils.readable(endHeap), // - DrillStringUtils.readable(endNonHeap - startNonHeap), DrillStringUtils.readable(endNonHeap) // - ); - } - - } - - private static class TestLogReporter extends TestWatcher { - - private MemWatcher memWatcher; - private int failureCount = 0; - - @Override - protected void starting(Description description) { - super.starting(description); - className = description.getClassName(); - memWatcher = new MemWatcher(); - } - - @Override - protected void failed(Throwable e, Description description) { - testReporter.error(String.format("Test Failed (%s): %s", memWatcher.getMemString(), description.getDisplayName()), e); - failureCount++; - } - - @Override - public void succeeded(Description description) { - testReporter.info(String.format("Test Succeeded (%s): %s", memWatcher.getMemString(), description.getDisplayName())); - } - - public void sleepIfFailure() throws InterruptedException { - if(failureCount > 0){ - Thread.sleep(2000); - failureCount = 0; - } else { - // pause to get logger to catch up. - Thread.sleep(250); - } - } - - } - - public static String escapeJsonString(String original) { - try { - return objectMapper.writeValueAsString(original); - } catch (JsonProcessingException e) { - return original; - } - } - - private static class SystemManager { - - final BufferPoolMXBean directBean; - final MemoryMXBean memoryBean; - - public SystemManager(){ - memoryBean = ManagementFactory.getMemoryMXBean(); - BufferPoolMXBean localBean = null; - List pools = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class); - for(BufferPoolMXBean b : pools){ - if(b.getName().equals("direct")){ - localBean = b; - - } - } - directBean = localBean; - } - - public long getMemDirect() { - return directBean.getMemoryUsed(); - } - - public long getMemHeap() { - return memoryBean.getHeapMemoryUsage().getUsed(); - } - - public long getMemNonHeap() { - return memoryBean.getNonHeapMemoryUsage().getUsed(); - } - - } - -} diff --git a/common/src/test/java/org/apache/drill/test/DrillTestBase.java b/common/src/test/java/org/apache/drill/test/DrillTestBase.java index 7df53793ebc..4cba293c365 100644 --- a/common/src/test/java/org/apache/drill/test/DrillTestBase.java +++ b/common/src/test/java/org/apache/drill/test/DrillTestBase.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,10 +17,27 @@ */ package org.apache.drill.test; -import org.junit.Before; +import java.lang.management.BufferPoolMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.util.Collection; +import java.util.List; +import java.util.Random; + +import com.carrotsearch.randomizedtesting.RandomizedContext; +import com.carrotsearch.randomizedtesting.RandomizedRunner; +import com.carrotsearch.randomizedtesting.RandomizedTest; +import com.carrotsearch.randomizedtesting.annotations.Listeners; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import com.carrotsearch.randomizedtesting.generators.RandomPicks; + +import org.apache.drill.common.util.DrillStringUtils; +import org.apache.drill.common.util.TestTools; + import org.junit.After; -import org.junit.BeforeClass; import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Rule; import org.junit.rules.TestName; @@ -29,27 +46,11 @@ import org.junit.runner.Description; import org.junit.runner.RunWith; -import org.apache.lucene.util.TestRuleStoreClassName; - -import com.carrotsearch.randomizedtesting.RandomizedTest; -import com.carrotsearch.randomizedtesting.RandomizedRunner; -import com.carrotsearch.randomizedtesting.RandomizedContext; -import com.carrotsearch.randomizedtesting.generators.RandomPicks; -import com.carrotsearch.randomizedtesting.annotations.Listeners; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; - -import org.apache.drill.common.util.TestTools; -import org.apache.drill.common.util.DrillStringUtils; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.management.BufferPoolMXBean; -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryMXBean; -import java.util.Collection; -import java.util.List; -import java.util.Random; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; /** * Abstract base class for all Drill tests. @@ -64,192 +65,212 @@ @ThreadLeakScope(ThreadLeakScope.Scope.NONE) public abstract class DrillTestBase { - private static final Logger logger = LoggerFactory.getLogger("org.apache.drill.TestReporter"); - private static final TestLogReporter LOG_OUTCOME = new TestLogReporter(); - - static { - System.setProperty("line.separator", "\n"); - } - - protected static final SystemManager manager = new SystemManager(); - protected static MemWatcher memWatcher; - protected static String className; - - @Rule public final TestRule TIMEOUT = TestTools.getTimeoutRule(50000); - @Rule public final TestLogReporter logOutcome = LOG_OUTCOME; - @Rule public final TestRule REPEAT_RULE = TestTools.getRepeatRule(false); - @Rule public TestName TEST_NAME = new TestName(); - - @ClassRule - public static final TestRuleStoreClassName classNameRule = new TestRuleStoreClassName(); + private static final Logger logger = LoggerFactory.getLogger("org.apache.drill.TestReporter"); - @BeforeClass - public static void initDrillTest() throws Exception { - memWatcher = new MemWatcher(); - } + static { + System.setProperty("line.separator", "\n"); - @AfterClass - public static void finishDrillTest() throws InterruptedException{ - logger.info(String.format("Test Class done (%s): %s.", memWatcher.getMemString(true), className)); + java.lang.Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + Logger socket = LoggerFactory.getLogger("SOCKET"); + if (socket != null) { + // XXX - Add socket flush here. + } + } + }); + } + + private static final ObjectMapper objectMapper = new ObjectMapper(); + private static final SystemManager manager = new SystemManager(); + private static final TestLogReporter LOG_OUTCOME = new TestLogReporter(); + private static MemWatcher memWatcher; + protected static String className; + + @Rule public final TestRule TIMEOUT = TestTools.getTimeoutRule(50000); + @Rule public final TestLogReporter logOutcome = LOG_OUTCOME; + @Rule public final TestRule REPEAT_RULE = TestTools.getRepeatRule(false); + @Rule public TestName TEST_NAME = new TestName(); + + @ClassRule + private static final StoreClassNameRule classNameRule = new StoreClassNameRule(); + + @BeforeClass + public static void beforeDrillTestBaseClass() throws Exception { + memWatcher = new MemWatcher(); + } + + @AfterClass + public static void afterDrillTestBaseClass() throws InterruptedException{ + logger.info(String.format("Test Class done (%s): %s.", memWatcher.getMemString(true), className)); + } + + @Before + public void beforeDrillTestBase() throws Exception { + System.out.printf("Running %s#%s\n", getClass().getName(), TEST_NAME.getMethodName()); + } + + @After + public void afterDrillTestBase() { + // Optionally add anything here that needs to be cleared after each test. + } + + protected static Class getTestClass() { + return classNameRule.getTestClass(); + } + + /* *** Utility Classes *** */ + + protected static class MemWatcher { + private long startDirect; + private long startHeap; + private long startNonHeap; + + public MemWatcher() { + startDirect = manager.getMemDirect(); + startHeap = manager.getMemHeap(); + startNonHeap = manager.getMemNonHeap(); } - @Before - public void beforeDrillTestBase() throws Exception { - System.out.printf("Running %s#%s\n", getClass().getName(), TEST_NAME.getMethodName()); + public Object getMemString() { + return getMemString(false); } - @After - public void afterDrillTestBase() { - // Optionally add anything here that needs to be cleared after each test. + public String getMemString(boolean runGC) { + if (runGC) { + Runtime.getRuntime().gc(); + } + long endDirect = manager.getMemDirect(); + long endHeap = manager.getMemHeap(); + long endNonHeap = manager.getMemNonHeap(); + return String.format("d: %s(%s), h: %s(%s), nh: %s(%s)", // + DrillStringUtils.readable(endDirect - startDirect), DrillStringUtils.readable(endDirect), // + DrillStringUtils.readable(endHeap - startHeap), DrillStringUtils.readable(endHeap), // + DrillStringUtils.readable(endNonHeap - startNonHeap), DrillStringUtils.readable(endNonHeap) // + ); } + } - public static Class getTestClass() { - return classNameRule.getTestClass(); - } + private static class TestLogReporter extends TestWatcher { - /* *** Randomization Utilities *** */ + private MemWatcher memWatcher; - public static Random random() { - return RandomizedContext.current().getRandom(); + @Override + protected void starting(Description description) { + super.starting(description); + className = description.getClassName(); + memWatcher = new MemWatcher(); } - public static boolean randomBoolean() { - return random().nextBoolean(); + @Override + protected void failed(Throwable e, Description description) { + logger.error(String.format("Test Failed (%s): %s", memWatcher.getMemString(), description.getDisplayName()), e); } - public static byte randomByte() { - return (byte) random().nextInt(); + @Override + public void succeeded(Description description) { + logger.info(String.format("Test Succeeded (%s): %s", memWatcher.getMemString(), description.getDisplayName())); } + } - public static short randomShort() { - return (short) random().nextInt(); + public static String escapeJsonString(String original) { + try { + return objectMapper.writeValueAsString(original); + } catch (JsonProcessingException e) { + return original; } + } - public static int randomInt() { - return random().nextInt(); - } + private static class SystemManager { - public static float randomFloat() { - return random().nextFloat(); - } + final BufferPoolMXBean directBean; + final MemoryMXBean memoryBean; - public static double randomDouble() { - return random().nextDouble(); - } - - public static long randomLong() { - return random().nextLong(); - } + public SystemManager(){ + memoryBean = ManagementFactory.getMemoryMXBean(); + BufferPoolMXBean localBean = null; + List pools = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class); + for(BufferPoolMXBean b : pools){ + if(b.getName().equals("direct")){ + localBean = b; - public static int randomInt(int max) { - return RandomizedTest.randomInt(max); - } - - public static int randomIntBetween(int min, int max) { - return RandomizedTest.randomIntBetween(min, max); + } + } + directBean = localBean; } - public static String randomAscii() { - return RandomizedTest.randomAsciiOfLength(32); + public long getMemDirect() { + return directBean.getMemoryUsed(); } - public static String randomAsciiOfLength(int codeUnits) { - return RandomizedTest.randomAsciiOfLength(codeUnits); + public long getMemHeap() { + return memoryBean.getHeapMemoryUsage().getUsed(); } - public static T randomFrom(List list) { - return RandomPicks.randomFrom(random(), list); + public long getMemNonHeap() { + return memoryBean.getNonHeapMemoryUsage().getUsed(); } + } - public static T randomFrom(T[] array) { - return RandomPicks.randomFrom(random(), array); - } + /* *** Randomization Utilities *** */ - public static T randomFrom(Collection collection) { - return RandomPicks.randomFrom(random(), collection); - } + public static Random random() { + return RandomizedContext.current().getRandom(); + } - /* *** Utility Classes *** */ + public static boolean randomBoolean() { + return random().nextBoolean(); + } - protected static class MemWatcher { - private long startDirect; - private long startHeap; - private long startNonHeap; + public static byte randomByte() { + return (byte) random().nextInt(); + } - public MemWatcher() { - startDirect = manager.getMemDirect(); - startHeap = manager.getMemHeap(); - startNonHeap = manager.getMemNonHeap(); - } + public static short randomShort() { + return (short) random().nextInt(); + } - public Object getMemString() { - return getMemString(false); - } + public static int randomInt() { + return random().nextInt(); + } - public String getMemString(boolean runGC) { - if (runGC) { - Runtime.getRuntime().gc(); - } - long endDirect = manager.getMemDirect(); - long endHeap = manager.getMemHeap(); - long endNonHeap = manager.getMemNonHeap(); - return String.format("d: %s(%s), h: %s(%s), nh: %s(%s)", // - DrillStringUtils.readable(endDirect - startDirect), DrillStringUtils.readable(endDirect), // - DrillStringUtils.readable(endHeap - startHeap), DrillStringUtils.readable(endHeap), // - DrillStringUtils.readable(endNonHeap - startNonHeap), DrillStringUtils.readable(endNonHeap) // - ); - } - } + public static float randomFloat() { + return random().nextFloat(); + } - private static class TestLogReporter extends TestWatcher { + public static double randomDouble() { + return random().nextDouble(); + } - private DrillTest.MemWatcher memWatcher; + public static long randomLong() { + return random().nextLong(); + } - @Override - protected void starting(Description description) { - super.starting(description); - className = description.getClassName(); - memWatcher = new DrillTest.MemWatcher(); - } + public static int randomInt(int max) { + return RandomizedTest.randomInt(max); + } - @Override - protected void failed(Throwable e, Description description) { - logger.error(String.format("Test Failed (%s): %s", memWatcher.getMemString(), description.getDisplayName()), e); - } + public static int randomIntBetween(int min, int max) { + return RandomizedTest.randomIntBetween(min, max); + } - @Override - public void succeeded(Description description) { - logger.info(String.format("Test Succeeded (%s): %s", memWatcher.getMemString(), description.getDisplayName())); - } - } + public static String randomAscii() { + return RandomizedTest.randomAsciiOfLength(32); + } - private static class SystemManager { - - final BufferPoolMXBean directBean; - final MemoryMXBean memoryBean; - - public SystemManager() { - memoryBean = ManagementFactory.getMemoryMXBean(); - BufferPoolMXBean localBean = null; - List pools = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class); - for(BufferPoolMXBean b : pools) { - if (b.getName().equals("direct")) { - localBean = b; - } - } - directBean = localBean; - } + public static String randomAsciiOfLength(int codeUnits) { + return RandomizedTest.randomAsciiOfLength(codeUnits); + } - public long getMemDirect() { - return directBean.getMemoryUsed(); - } + public static T randomFrom(List list) { + return RandomPicks.randomFrom(random(), list); + } - public long getMemHeap() { - return memoryBean.getHeapMemoryUsage().getUsed(); - } + public static T randomFrom(T[] array) { + return RandomPicks.randomFrom(random(), array); + } - public long getMemNonHeap() { - return memoryBean.getNonHeapMemoryUsage().getUsed(); - } - } + public static T randomFrom(Collection collection) { + return RandomPicks.randomFrom(random(), collection); + } } diff --git a/common/src/test/java/org/apache/drill/test/StoreClassNameRule.java b/common/src/test/java/org/apache/drill/test/StoreClassNameRule.java new file mode 100644 index 00000000000..5092dc7d30d --- /dev/null +++ b/common/src/test/java/org/apache/drill/test/StoreClassNameRule.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.test; + +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +/** + * + */ +public class StoreClassNameRule implements TestRule { + + private volatile Description description; + + @Override + public Statement apply(final Statement s, final Description d) { + if (!d.isSuite()) { + throw new IllegalArgumentException("This is a @ClassRule (applies to suites only)."); + } + + return new Statement() { + @Override + public void evaluate() throws Throwable { + try { + description = d; + s.evaluate(); + } finally { + description = null; + } + } + }; + } + + /** + * Returns the test class currently executing in this rule. + */ + public Class getTestClass() { + Description localDescription = description; + if (localDescription == null) { + throw new RuntimeException("The rule is not currently executing."); + } + return localDescription.getTestClass(); + } +} diff --git a/contrib/pom.xml b/contrib/pom.xml index 1cb1d729ec6..7b30bc2d281 100644 --- a/contrib/pom.xml +++ b/contrib/pom.xml @@ -29,95 +29,6 @@ contrib/Parent Pom pom - - 2.1.16 - .local-${project.version}-execution-hints.log - - - - - - com.carrotsearch.randomizedtesting - randomizedtesting-runner - ${carrotsearch.testframework} - test - - - com.carrotsearch.randomizedtesting - junit4-maven-plugin - ${carrotsearch.testframework} - test - - - - commons-logging - commons-logging-api - - - - - - - - - maven-surefire-plugin - 2.15 - - false - true - - - - com.carrotsearch.randomizedtesting - junit4-maven-plugin - ${carrotsearch.testframework} - - true - - - - - - - - - - - tests - test - - junit4 - - - - - - - - - - - - - **/*Tests.class - **/*Test.class - - - - - - - - storage-hbase storage-hive diff --git a/contrib/storage-hbase/pom.xml b/contrib/storage-hbase/pom.xml index d02777185be..b808b4954cd 100644 --- a/contrib/storage-hbase/pom.xml +++ b/contrib/storage-hbase/pom.xml @@ -74,6 +74,7 @@ + diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java index df83c56f7b6..b28ffa5d795 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.util.List; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.rpc.user.QueryDataBatch; @@ -36,7 +36,7 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; -public class BaseHBaseTest extends BaseTestQuery { +public class BaseHBaseTest extends DrillIntegrationTestBase { private static final String HBASE_STORAGE_PLUGIN_NAME = "hbase"; @@ -55,7 +55,7 @@ public static void setUpBeforeClass() throws Exception { HBaseTestsSuite.configure(true, true); HBaseTestsSuite.initCluster(); - final StoragePluginRegistry pluginRegistry = getDrillbitContext().getStorage(); + final StoragePluginRegistry pluginRegistry = getCluster().randomDrillBitContext().getStorage(); storagePlugin = (HBaseStoragePlugin) pluginRegistry.getPlugin(HBASE_STORAGE_PLUGIN_NAME); storagePluginConfig = storagePlugin.getConfig(); storagePluginConfig.setEnabled(true); diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest_NewTestFramework.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest_NewTestFramework.java deleted file mode 100644 index 10e5593fc6c..00000000000 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest_NewTestFramework.java +++ /dev/null @@ -1,105 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.hbase; - -import com.google.common.base.Charsets; -import com.google.common.io.Files; -import org.apache.drill.BaseTestQuery; -import org.apache.drill.common.util.FileUtils; -import org.apache.drill.exec.DrillIntegrationTestBase; -import org.apache.drill.exec.exception.SchemaChangeException; -import org.apache.drill.exec.rpc.user.QueryDataBatch; -import org.apache.drill.exec.store.StoragePluginRegistry; -import org.apache.drill.exec.store.hbase.HBaseStoragePlugin; -import org.apache.drill.exec.store.hbase.HBaseStoragePluginConfig; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; - -import java.io.IOException; -import java.util.List; - -public class BaseHBaseTest_NewTestFramework extends DrillIntegrationTestBase { - - private static final String HBASE_STORAGE_PLUGIN_NAME = "hbase"; - - protected static Configuration conf = HBaseConfiguration.create(); - - protected static HBaseStoragePlugin storagePlugin; - - protected static HBaseStoragePluginConfig storagePluginConfig; - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - /* - * Change the following to HBaseTestsSuite.configure(false, true) - * if you want to test against an externally running HBase cluster. - */ - HBaseTestsSuite.configure(true, true); - HBaseTestsSuite.initCluster(); - - final StoragePluginRegistry pluginRegistry = getCluster().randomDrillBitContext().getStorage(); - storagePlugin = (HBaseStoragePlugin) pluginRegistry.getPlugin(HBASE_STORAGE_PLUGIN_NAME); - storagePluginConfig = storagePlugin.getConfig(); - storagePluginConfig.setEnabled(true); - storagePluginConfig.setZookeeperPort(HBaseTestsSuite.getZookeeperPort()); - pluginRegistry.createOrUpdate(HBASE_STORAGE_PLUGIN_NAME, storagePluginConfig, true); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - HBaseTestsSuite.tearDownCluster(); - } - - protected String getPlanText(String planFile, String tableName) throws IOException { - return Files.toString(FileUtils.getResourceAsFile(planFile), Charsets.UTF_8) - .replaceFirst("\"hbase\\.zookeeper\\.property\\.clientPort\".*:.*\\d+", "\"hbase.zookeeper.property.clientPort\" : " + HBaseTestsSuite.getZookeeperPort()) - .replace("[TABLE_NAME]", tableName); - } - - protected void runHBasePhysicalVerifyCount(String planFile, String tableName, int expectedRowCount) throws Exception{ - String physicalPlan = getPlanText(planFile, tableName); - List results = testPhysicalWithResults(physicalPlan); - printResultAndVerifyRowCount(results, expectedRowCount); - } - - protected List runHBaseSQLlWithResults(String sql) throws Exception { - sql = canonizeHBaseSQL(sql); - System.out.println("Running query:\n" + sql); - return testSqlWithResults(sql); - } - - protected void runHBaseSQLVerifyCount(String sql, int expectedRowCount) throws Exception{ - List results = runHBaseSQLlWithResults(sql); - printResultAndVerifyRowCount(results, expectedRowCount); - } - - private void printResultAndVerifyRowCount(List results, int expectedRowCount) throws SchemaChangeException { - int rowCount = printResult(results); - if (expectedRowCount != -1) { - Assert.assertEquals(expectedRowCount, rowCount); - } - } - - protected String canonizeHBaseSQL(String sql) { - return sql.replace("[TABLE_NAME]", HBaseTestsSuite.TEST_TABLE_1); - } - -} diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_A.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_A.java deleted file mode 100644 index 098548b2767..00000000000 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_A.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.hbase; - -import org.apache.drill.exec.DrillIntegrationTestBase; -import org.junit.Test; - -/** - * - */ -@DrillIntegrationTestBase.ClusterScope(scope = DrillIntegrationTestBase.Scope.GLOBAL, bits = -1, width = -1) -public class IntegrationTest_A extends DrillIntegrationTestBase { - - @Test - public void a_x() throws Exception { - System.out.println("\nTEST A_X()"); - } - - @Test - public void a_y() throws Exception { - System.out.println("\nTEST A_Y()"); - } - - @Test - public void a_z() throws Exception { - System.out.println("\nTEST A_Z()"); - } - -} diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_B.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_B.java deleted file mode 100644 index f7727c64b14..00000000000 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/IntegrationTest_B.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.hbase; - -import org.apache.drill.exec.DrillIntegrationTestBase; -import org.junit.Test; - -/** - * - */ -@DrillIntegrationTestBase.ClusterScope(scope = DrillIntegrationTestBase.Scope.GLOBAL, bits = -1, width = -1) -public class IntegrationTest_B extends DrillIntegrationTestBase { - - @Test - public void b_x() throws Exception { - System.out.println("\nTEST B_X()"); - } - - @Test - public void b_y() throws Exception { - System.out.println("\nTEST B_Y()"); - } - - @Test - public void b_z() throws Exception { - System.out.println("\nTEST B_Z()"); - } -} diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown_NewTestFramework.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown_NewTestFramework.java deleted file mode 100644 index 30c866a2de0..00000000000 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown_NewTestFramework.java +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.hbase; - -import org.apache.drill.exec.DrillIntegrationTestBase.Scope; -import org.apache.drill.exec.DrillIntegrationTestBase.ClusterScope; - -import org.junit.Test; - -@ClusterScope(scope = Scope.SUITE, bits = 1, width = -1) -public class TestHBaseProjectPushDown_NewTestFramework extends BaseHBaseTest_NewTestFramework { - - @Test - public void x() throws Exception { - setColumnWidth(8); - System.out.println("\n\nBLAH BLAH BLAH TEST X"); - } - - @Test - public void y() throws Exception { - setColumnWidth(8); - System.out.println("\n\nBLAH BLAH BLAH TEST Y"); - } - - @Test - public void z() throws Exception { - setColumnWidth(8); - System.out.println("\n\nBLAH BLAH BLAH TEST Z"); - } - - @org.junit.Ignore - @Test - public void testRowKeyPushDown() throws Exception{ - setColumnWidth(8); - runHBaseSQLVerifyCount("SELECT\n" - + "row_key\n" - + "FROM\n" - + " hbase.`[TABLE_NAME]` tableName" - , 7); - } - - @org.junit.Ignore - @Test - public void testColumnWith1RowPushDown() throws Exception{ - setColumnWidth(6); - runHBaseSQLVerifyCount("SELECT\n" - + "t.f2.c7 as `t.f2.c7`\n" - + "FROM\n" - + " hbase.`[TABLE_NAME]` t" - , 1); - } - - @org.junit.Ignore - @Test - public void testRowKeyAndColumnPushDown() throws Exception{ - setColumnWidths(new int[] {8, 9, 6, 2, 6}); - runHBaseSQLVerifyCount("SELECT\n" - + "row_key, t.f.c1*31 as `t.f.c1*31`, t.f.c2 as `t.f.c2`, 5 as `5`, 'abc' as `'abc'`\n" - + "FROM\n" - + " hbase.`[TABLE_NAME]` t" - , 7); - } - - @org.junit.Ignore - @Test - public void testColumnFamilyPushDown() throws Exception{ - setColumnWidths(new int[] {8, 74, 38}); - runHBaseSQLVerifyCount("SELECT\n" - + "row_key, f, f2\n" - + "FROM\n" - + " hbase.`[TABLE_NAME]` tableName" - , 7); - } - -} diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseRegexParser.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseRegexParser.java index a925d0e92a3..57604c75ba1 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseRegexParser.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseRegexParser.java @@ -25,10 +25,10 @@ import java.util.regex.Pattern; import org.apache.drill.exec.store.hbase.HBaseRegexParser; -import org.apache.drill.test.DrillTest; +import org.apache.drill.test.DrillTestBase; import org.junit.Test; -public class TestHBaseRegexParser extends DrillTest { +public class TestHBaseRegexParser extends DrillTestBase { @Test public void testLikeExprToRegex() throws Exception { diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestHiveUDFs.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestHiveUDFs.java index 3ce9a6d650f..7b166e05739 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestHiveUDFs.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestHiveUDFs.java @@ -22,7 +22,7 @@ import java.util.List; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.exec.record.RecordBatchLoader; import org.apache.drill.exec.rpc.user.QueryDataBatch; import org.apache.drill.exec.vector.Float4Vector; @@ -35,7 +35,7 @@ import com.google.common.base.Charsets; import com.google.common.io.Resources; -public class TestHiveUDFs extends BaseTestQuery { +public class TestHiveUDFs extends DrillIntegrationTestBase { @Test public void testGenericUDF() throws Throwable { diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java index 3be846ddbbf..0bb239dc479 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java @@ -25,7 +25,7 @@ import java.util.Map; import org.apache.commons.io.FileUtils; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.exceptions.DrillException; import org.apache.drill.exec.store.StoragePluginRegistry; import org.apache.hadoop.fs.FileSystem; @@ -36,7 +36,7 @@ import com.google.common.collect.Maps; -import static org.apache.drill.BaseTestQuery.getTempDir; +import static org.apache.drill.DrillIntegrationTestBase.getTempDir; import static org.apache.drill.exec.hive.HiveTestUtilities.executeQuery; public class HiveTestDataGenerator { @@ -148,7 +148,7 @@ private void generateTestData() throws Exception { "STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' " + "OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' " + "TBLPROPERTIES ('avro.schema.url'='file:///%s')", - BaseTestQuery.getPhysicalFileFromResource("avro_test_schema.json").replace('\\', '/')); + DrillIntegrationTestBase.getPhysicalFileFromResource("avro_test_schema.json").replace('\\', '/')); executeQuery(hiveDriver, avroCreateQuery); executeQuery(hiveDriver, "INSERT INTO TABLE db1.avro SELECT * FROM default.kv"); diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/DrillIntegrationTestBase.java similarity index 51% rename from exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java rename to exec/java-exec/src/test/java/org/apache/drill/DrillIntegrationTestBase.java index ae408194584..e52379f8c6d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java +++ b/exec/java-exec/src/test/java/org/apache/drill/DrillIntegrationTestBase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,29 +17,22 @@ */ package org.apache.drill; -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.net.URL; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicInteger; - +import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; import com.google.common.io.Files; +import com.google.common.io.Resources; + import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.ExecConstants; -import org.apache.drill.exec.ExecTest; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.memory.TopLevelAllocator; +import org.apache.drill.exec.metrics.DrillMetrics; import org.apache.drill.exec.proto.UserBitShared; -import org.apache.drill.exec.proto.UserBitShared.QueryId; -import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState; -import org.apache.drill.exec.proto.UserBitShared.QueryType; import org.apache.drill.exec.record.RecordBatchLoader; +import org.apache.drill.exec.rpc.RpcException; import org.apache.drill.exec.rpc.user.ConnectionThrottle; import org.apache.drill.exec.rpc.user.QueryDataBatch; import org.apache.drill.exec.rpc.user.UserResultsListener; @@ -49,32 +42,48 @@ import org.apache.drill.exec.store.StoragePluginRegistry; import org.apache.drill.exec.util.TestUtilities; import org.apache.drill.exec.util.VectorUtil; +import org.apache.drill.test.DrillTestBase; + +import org.junit.After; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; -import org.junit.rules.TestRule; -import org.junit.rules.TestWatcher; -import org.junit.runner.Description; - -import com.google.common.base.Charsets; -import com.google.common.base.Preconditions; -import com.google.common.io.Resources; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.lang.annotation.Annotation; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.net.URL; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; + import static org.hamcrest.core.StringContains.containsString; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; -public class BaseTestQuery extends ExecTest { +/** + * Abstract base class for integration tests. + */ +public abstract class DrillIntegrationTestBase extends DrillTestBase { - private static final Logger logger = LoggerFactory.getLogger(BaseTestQuery.class); + private static final Logger logger = LoggerFactory.getLogger(DrillIntegrationTestBase.class); protected static final String TEMP_SCHEMA = "dfs_test.tmp"; - private static final String ENABLE_FULL_CACHE = "drill.exec.test.use-full-cache"; - private static final int MAX_WIDTH_PER_NODE = 2; + private static DrillTestCluster cluster; + + private int[] columnWidths = new int[] { 8 }; @SuppressWarnings("serial") private static final Properties TEST_CONFIGURATIONS = new Properties() { @@ -84,221 +93,303 @@ public class BaseTestQuery extends ExecTest { } }; - public final TestRule resetWatcher = new TestWatcher() { - @Override - protected void failed(Throwable e, Description description) { - try { - resetClientAndBit(); - } catch (Exception e1) { - throw new RuntimeException("Failure while resetting client.", e1); - } - } - }; - - protected static DrillClient client; - protected static Drillbit[] bits; - protected static RemoteServiceSet serviceSet; - protected static DrillConfig config; - protected static BufferAllocator allocator; + /* *** Cluster Scope Settings *** */ + + public static final int MIN_NUM_DRILLBITS = 1; + public static final int MAX_NUM_DRILLBITS = 5; + public static final int DEFAULT_NUM_DRILLBITS = 2; + public static final int RANDOM_NUM_DRILLBITS = -1; + + public static final int MIN_PARALLELIZATION_WIDTH = 1; + public static final int MAX_PARALLELIZATION_WIDTH = 5; + public static final int DEFAULT_PARALLELIZATION_WIDTH = 2; + public static final int RANDOM_PARALLELIZATION_WIDTH = -1; + + @Retention(RetentionPolicy.RUNTIME) + @Target({ElementType.TYPE}) + public @interface ClusterScope { + + /** + * Returns the scope. {@link Scope#GLOBAL} is default. + */ + Scope scope() default Scope.GLOBAL; + + /** + * Sets the number of drillbits to create in the test cluster. + * + * Default is DEFAULT_NUM_DRILLBITS. + * A value of RANDOM_NUM_DRILLBITS indicates that the test framework should choose a + * random value in the inclusive range [{@link DrillIntegrationTestBase#MIN_NUM_DRILLBITS}, + * {@link DrillIntegrationTestBase#MAX_NUM_DRILLBITS}]. + */ + int bits() default DEFAULT_NUM_DRILLBITS; + + /** + * Sets the value of {@link ExecConstants#MAX_WIDTH_PER_NODE_KEY}, which determines + * the level of parallelization per drillbit. + * + * Default is DEFAULT_PARALLELIZATION_WIDTH. + * A value of RANDOM_PARALLELIZATION_WIDTH indicates that the test framework should choose a + * random value in the inclusive range [{@link DrillIntegrationTestBase#MIN_PARALLELIZATION_WIDTH}, + * {@link DrillIntegrationTestBase#MAX_PARALLELIZATION_WIDTH}]. + */ + int width() default DEFAULT_PARALLELIZATION_WIDTH; + } /** - * Number of Drillbits in test cluster. Default is 1. - * - * Tests can update the cluster size through {@link #updateTestCluster(int, DrillConfig)} + * The scope of a test cluster used together with + * {@link ClusterScope} annotations on {@link DrillIntegrationTestBase} subclasses. */ - private static int drillbitCount = 1; + public enum Scope { + /** + * A cluster that is shared across all test suites + */ + GLOBAL, + /** + * A cluster shared across all methods in a single test suite, but not shared across test suites + */ + SUITE + } - /** - * Location of the dfs_test.tmp schema on local filesystem. - */ - private static String dfsTestTmpSchemaLocation; + private static Scope getCurrentClusterScope(Class clazz) { + ClusterScope annotation = getAnnotation(clazz, ClusterScope.class); + return annotation == null ? Scope.GLOBAL : annotation.scope(); + } - private int[] columnWidths = new int[] { 8 }; + private static int getNumDrillbits() { + ClusterScope annotation = getAnnotation(getTestClass(), ClusterScope.class); + return annotation == null ? -1 : annotation.bits(); + } - @BeforeClass - public static void setupDefaultTestCluster() throws Exception { - config = DrillConfig.create(TEST_CONFIGURATIONS); - openClient(); + private static int getParallelizationWidth() { + ClusterScope annotation = getAnnotation(getTestClass(), ClusterScope.class); + return annotation == null ? -1 : annotation.width(); } - protected static void updateTestCluster(int newDrillbitCount, DrillConfig newConfig) { - Preconditions.checkArgument(newDrillbitCount > 0, "Number of Drillbits must be at least one"); - if (drillbitCount != newDrillbitCount || config != null) { - // TODO: Currently we have to shutdown the existing Drillbit cluster before starting a new one with the given - // Drillbit count. Revisit later to avoid stopping the cluster. - try { - closeClient(); - drillbitCount = newDrillbitCount; - if (newConfig != null) { - // For next test class, updated DrillConfig will be replaced by default DrillConfig in BaseTestQuery as part - // of the @BeforeClass method of test class. - config = newConfig; - } - openClient(); - } catch(Exception e) { - throw new RuntimeException("Failure while updating the test Drillbit cluster.", e); - } + private static A getAnnotation(Class clazz, Class annotationClass) { + if (clazz == Object.class || clazz == DrillIntegrationTestBase.class) { + return null; + } + A annotation = clazz.getAnnotation(annotationClass); + if (annotation != null) { + return annotation; } + return getAnnotation(clazz.getSuperclass(), annotationClass); } - /** - * Useful for tests that require a DrillbitContext to get/add storage plugins, options etc. - * - * @return DrillbitContext of first Drillbit in the cluster. - */ - protected static DrillbitContext getDrillbitContext() { - Preconditions.checkState(bits != null && bits[0] != null, "Drillbits are not setup."); - return bits[0].getContext(); + public static TestBuilder newTest() { + return testBuilder(); } - protected static Properties cloneDefaultTestConfigProperties() { - final Properties props = new Properties(); - for(String propName : TEST_CONFIGURATIONS.stringPropertyNames()) { - props.put(propName, TEST_CONFIGURATIONS.getProperty(propName)); - } - - return props; + public static TestBuilder testBuilder() { + assert cluster != null; + return new TestBuilder(cluster.allocator()); } - protected static String getDfsTestTmpSchemaLocation() { - return dfsTestTmpSchemaLocation; + protected static BufferAllocator getAllocator() { + assert cluster != null; + return cluster.allocator(); } - private static void resetClientAndBit() throws Exception{ - closeClient(); - openClient(); + protected static DrillTestCluster getCluster() { + return cluster; } - private static void openClient() throws Exception { - allocator = new TopLevelAllocator(config); - if (config.hasPath(ENABLE_FULL_CACHE) && config.getBoolean(ENABLE_FULL_CACHE)) { - serviceSet = RemoteServiceSet.getServiceSetWithFullCache(config, allocator); - } else { - serviceSet = RemoteServiceSet.getLocalServiceSet(); + @BeforeClass + public static void beforeClass() throws Exception { + + logger.debug("Initializing test framework for class: {}", getTestClass()); + + if (cluster != null) { + logger.info("Re-using existing test cluster of {} drillbits", cluster.bits().length); + return; } - dfsTestTmpSchemaLocation = TestUtilities.createTempDir(); + DrillConfig config = DrillConfig.create(TEST_CONFIGURATIONS); - bits = new Drillbit[drillbitCount]; - for(int i = 0; i < drillbitCount; i++) { - bits[i] = new Drillbit(config, serviceSet); - bits[i].run(); + int size = getNumDrillbits(); + if (size <= RANDOM_NUM_DRILLBITS || size > MAX_NUM_DRILLBITS) { + size = randomIntBetween(MIN_NUM_DRILLBITS, MAX_NUM_DRILLBITS); + } - final StoragePluginRegistry pluginRegistry = bits[i].getContext().getStorage(); - TestUtilities.updateDfsTestTmpSchemaLocation(pluginRegistry, dfsTestTmpSchemaLocation); - TestUtilities.makeDfsTmpSchemaImmutable(pluginRegistry); + int width = getParallelizationWidth(); + if (width <= RANDOM_PARALLELIZATION_WIDTH || width > MAX_PARALLELIZATION_WIDTH) { + width = randomIntBetween(MIN_PARALLELIZATION_WIDTH, MAX_PARALLELIZATION_WIDTH); } - client = QueryTestUtil.createClient(config, serviceSet, MAX_WIDTH_PER_NODE, null); - } + Scope scope = getCurrentClusterScope(getTestClass()); - /** - * Close the current client and open a new client using the given properties. All tests executed - * after this method call use the new client. - * - * @param properties - */ - public static void updateClient(Properties properties) throws Exception { - Preconditions.checkState(bits != null && bits[0] != null, "Drillbits are not setup."); - if (client != null) { - client.close(); - client = null; + switch (scope) { + case GLOBAL: + case SUITE: + cluster = new DrillTestCluster(size, width, config); + break; + default: + fail("Unsupported cluster scope: " + scope); } - - client = QueryTestUtil.createClient(config, serviceSet, MAX_WIDTH_PER_NODE, properties); } - /* - * Close the current client and open a new client for the given user. All tests executed - * after this method call use the new client. - * @param user - */ - public static void updateClient(String user) throws Exception { - final Properties props = new Properties(); - props.setProperty("user", user); - updateClient(props); + @Before + public void beforeDrillIntegrationTest() throws Exception { + ; } - protected static BufferAllocator getAllocator() { - return allocator; + @AfterClass + public static void afterClass() throws IOException, InterruptedException { + if (cluster != null) { + cluster.close(); + } } - public static TestBuilder newTest() { - return testBuilder(); + @After + public void afterDrillIntegrationTest() { + logger.info("Executing post-test cleanup: [{}]", getTestClass()); + DrillMetrics.resetMetrics(); } - public static TestBuilder testBuilder() { - return new TestBuilder(allocator); - } + protected static final class DrillTestCluster implements Closeable { - @AfterClass - public static void closeClient() throws IOException, InterruptedException { - if (client != null) { - client.close(); - } + private final Drillbit[] bits; + private final DrillClient client; + private final DrillConfig config; + private final RemoteServiceSet serviceSet; + private final BufferAllocator allocator; + private final int width; + private final String tmpSchemaLocation; + + public DrillTestCluster(int size, int width, DrillConfig config) throws Exception { + + logger.info("Initializing test cluster of {} drillbits (parallelization width: {})", size, width); + + this.config = config; + this.width = width; + this.tmpSchemaLocation = TestUtilities.createTempDir(); + this.allocator = new TopLevelAllocator(config); + this.serviceSet = RemoteServiceSet.getLocalServiceSet(); - if (bits != null) { - for(Drillbit bit : bits) { - if (bit != null) { - bit.close(); - } + bits = new Drillbit[size]; + for (int i = 0; i < bits.length; i++) { + logger.debug("Initializing drillbit #{}", i); + bits[i] = new Drillbit(config, serviceSet); + bits[i].run(); + StoragePluginRegistry registry = bits[i].getContext().getStorage(); + TestUtilities.updateDfsTestTmpSchemaLocation(registry, tmpSchemaLocation); + TestUtilities.makeDfsTmpSchemaImmutable(registry); } + + client = QueryTestUtil.createClient(config, serviceSet, width, null); + } + + public DrillClient client() { + return client; + } + + public DrillClient getNewClient(Properties properties) throws RpcException { + return QueryTestUtil.createClient(config, serviceSet, width, properties); + } + + public Drillbit[] bits() { + return bits; + } + + public DrillConfig config() { + return this.config; + } + + public Drillbit randomBit() { + return randomFrom(bits); + } + + public DrillbitContext randomDrillBitContext() { + return randomBit().getContext(); + } + + public RemoteServiceSet serviceSet() { + return serviceSet; + } + + public BufferAllocator allocator() { + return allocator; } - if(serviceSet != null) { - serviceSet.close(); + public String tmpSchemaLocation() { + return tmpSchemaLocation; } - if (allocator != null) { - allocator.close(); + + @Override + public void close() throws IOException { + cluster.client.close(); + for (Drillbit bit : cluster.bits()) { + bit.close(); + } + cluster.serviceSet.close(); + cluster.allocator.close(); } } - @AfterClass - public static void resetDrillbitCount() { - // some test classes assume this value to be 1 and will fail if run along other tests that increase it - drillbitCount = 1; + /** + * Useful for tests that require a DrillbitContext to get/add storage plugins, options etc. + * + * @return DrillbitContext of a random Drillbit in the cluster. + */ + protected static DrillbitContext getDrillbitContext() { + Preconditions.checkState(cluster != null, "Test cluster is not running."); + return cluster.randomDrillBitContext(); + } + + protected static Properties cloneDefaultTestConfigProperties() { + final Properties props = new Properties(); + for(String propName : TEST_CONFIGURATIONS.stringPropertyNames()) { + props.put(propName, TEST_CONFIGURATIONS.getProperty(propName)); + } + + return props; + } + + protected static void updateTestCluster(int newDrillbitCount, DrillConfig newConfig) { + fail("Please use @ClusterScope annotations to set cluster properties"); } protected static void runSQL(String sql) throws Exception { SilentListener listener = new SilentListener(); - testWithListener(QueryType.SQL, sql, listener); + testWithListener(UserBitShared.QueryType.SQL, sql, listener); listener.waitForCompletion(); } protected static List testSqlWithResults(String sql) throws Exception{ - return testRunAndReturn(QueryType.SQL, sql); + return testRunAndReturn(UserBitShared.QueryType.SQL, sql); } protected static List testLogicalWithResults(String logical) throws Exception{ - return testRunAndReturn(QueryType.LOGICAL, logical); + return testRunAndReturn(UserBitShared.QueryType.LOGICAL, logical); } protected static List testPhysicalWithResults(String physical) throws Exception{ - return testRunAndReturn(QueryType.PHYSICAL, physical); + return testRunAndReturn(UserBitShared.QueryType.PHYSICAL, physical); } - public static List testRunAndReturn(QueryType type, String query) throws Exception{ + public static List testRunAndReturn(UserBitShared.QueryType type, String query) throws Exception{ query = QueryTestUtil.normalizeQuery(query); - return client.runQuery(type, query); + return cluster.client().runQuery(type, query); } - public static int testRunAndPrint(final QueryType type, final String query) throws Exception { - return QueryTestUtil.testRunAndPrint(client, type, query); + public static int testRunAndPrint(final UserBitShared.QueryType type, final String query) throws Exception { + return QueryTestUtil.testRunAndPrint(cluster.client(), type, query); } - protected static void testWithListener(QueryType type, String query, UserResultsListener resultListener) { - QueryTestUtil.testWithListener(client, type, query, resultListener); + protected static void testWithListener(UserBitShared.QueryType type, String query, UserResultsListener resultListener) { + QueryTestUtil.testWithListener(cluster.client(), type, query, resultListener); } protected static void testNoResult(String query, Object... args) throws Exception { testNoResult(1, query, args); } - protected static void testNoResult(int interation, String query, Object... args) throws Exception { + protected static void testNoResult(int iteration, String query, Object... args) throws Exception { query = String.format(query, args); logger.debug("Running query:\n--------------\n" + query); - for (int i = 0; i < interation; i++) { - List results = client.runQuery(QueryType.SQL, query); + for (int i = 0; i < iteration; i++) { + List results = cluster.client().runQuery(UserBitShared.QueryType.SQL, query); for (QueryDataBatch queryDataBatch : results) { queryDataBatch.release(); } @@ -306,23 +397,23 @@ protected static void testNoResult(int interation, String query, Object... args) } public static void test(String query, Object... args) throws Exception { - QueryTestUtil.test(client, String.format(query, args)); + QueryTestUtil.test(cluster.client(), String.format(query, args)); } public static void test(final String query) throws Exception { - QueryTestUtil.test(client, query); + QueryTestUtil.test(cluster.client(), query); } protected static int testLogical(String query) throws Exception{ - return testRunAndPrint(QueryType.LOGICAL, query); + return testRunAndPrint(UserBitShared.QueryType.LOGICAL, query); } protected static int testPhysical(String query) throws Exception{ - return testRunAndPrint(QueryType.PHYSICAL, query); + return testRunAndPrint(UserBitShared.QueryType.PHYSICAL, query); } protected static int testSql(String query) throws Exception{ - return testRunAndPrint(QueryType.SQL, query); + return testRunAndPrint(UserBitShared.QueryType.SQL, query); } protected static void testPhysicalFromFile(String file) throws Exception{ @@ -330,7 +421,7 @@ protected static void testPhysicalFromFile(String file) throws Exception{ } protected static List testPhysicalFromFileWithResults(String file) throws Exception { - return testRunAndReturn(QueryType.PHYSICAL, getFile(file)); + return testRunAndReturn(UserBitShared.QueryType.PHYSICAL, getFile(file)); } protected static void testLogicalFromFile(String file) throws Exception{ @@ -379,15 +470,13 @@ public static String getFile(String resource) throws IOException{ /** * Copy the resource (ex. file on classpath) to a physical file on FileSystem. - * @param resource - * @return * @throws IOException */ public static String getPhysicalFileFromResource(final String resource) throws IOException { final File file = File.createTempFile("tempfile", ".txt"); file.deleteOnExit(); PrintWriter printWriter = new PrintWriter(file); - printWriter.write(BaseTestQuery.getFile(resource)); + printWriter.write(DrillIntegrationTestBase.getFile(resource)); printWriter.close(); return file.getPath(); @@ -395,7 +484,6 @@ public static String getPhysicalFileFromResource(final String resource) throws I /** * Create a temp directory to store the given dirName - * @param dirName * @return Full path including temp parent directory and given directory name. */ public static String getTempDir(final String dirName) { @@ -405,7 +493,6 @@ public static String getTempDir(final String dirName) { return dir.getAbsolutePath() + File.separator + dirName; } - protected static void setSessionOption(final String option, final String value) { try { runSQL(String.format("alter session set `%s` = %s", option, value)); @@ -427,7 +514,7 @@ public void submissionFailed(UserException ex) { } @Override - public void queryCompleted(QueryState state) { + public void queryCompleted(UserBitShared.QueryResult.QueryState state) { System.out.println("Query completed successfully with row count: " + count.get()); latch.countDown(); } @@ -442,7 +529,7 @@ public void dataArrived(QueryDataBatch result, ConnectionThrottle throttle) { } @Override - public void queryIdArrived(QueryId queryId) {} + public void queryIdArrived(UserBitShared.QueryId queryId) {} public int waitForCompletion() throws Exception { latch.await(); @@ -479,25 +566,4 @@ protected int printResult(List results) throws SchemaChangeExcep System.out.println("Total record count: " + rowCount); return rowCount; } - - protected static String getResultString(List results, String delimiter) - throws SchemaChangeException { - StringBuilder formattedResults = new StringBuilder(); - boolean includeHeader = true; - RecordBatchLoader loader = new RecordBatchLoader(getAllocator()); - for(QueryDataBatch result : results) { - loader.load(result.getHeader().getDef(), result.getData()); - if (loader.getRecordCount() <= 0) { - continue; - } - VectorUtil.appendVectorAccessibleContent(loader, formattedResults, delimiter, includeHeader); - if (!includeHeader) { - includeHeader = false; - } - loader.clear(); - result.release(); - } - - return formattedResults.toString(); - } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/DrillIntegrationTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/DrillIntegrationTestBase_XXX.java similarity index 90% rename from exec/java-exec/src/test/java/org/apache/drill/exec/DrillIntegrationTestBase.java rename to exec/java-exec/src/test/java/org/apache/drill/DrillIntegrationTestBase_XXX.java index ccbfcb5ff4e..48d9ce58e4b 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/DrillIntegrationTestBase.java +++ b/exec/java-exec/src/test/java/org/apache/drill/DrillIntegrationTestBase_XXX.java @@ -15,19 +15,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec; +package org.apache.drill; import com.google.common.base.Charsets; import com.google.common.io.Files; import com.google.common.io.Resources; -import org.apache.drill.QueryTestUtil; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.UserException; +import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.memory.TopLevelAllocator; +import org.apache.drill.exec.metrics.DrillMetrics; import org.apache.drill.exec.proto.UserBitShared; import org.apache.drill.exec.record.RecordBatchLoader; import org.apache.drill.exec.rpc.user.ConnectionThrottle; @@ -72,14 +73,11 @@ /** * Abstract base class for integration tests. */ -public abstract class DrillIntegrationTestBase extends DrillTestBase { +public abstract class DrillIntegrationTestBase_XXX extends DrillTestBase { - private static final Logger logger = LoggerFactory.getLogger(DrillIntegrationTestBase.class); + private static final Logger logger = LoggerFactory.getLogger(DrillIntegrationTestBase_XXX.class); protected static final String TEMP_SCHEMA = "dfs_test.tmp"; - private static final String ENABLE_FULL_CACHE = "drill.exec.test.use-full-cache"; - - protected static BufferAllocator allocator; private static Scope scope; private static DrillTestCluster cluster; @@ -96,11 +94,15 @@ public abstract class DrillIntegrationTestBase extends DrillTestBase { /* *** Cluster Scope Settings *** */ - public static final int MIN_NUM_DRILLBITS = 1; - public static final int MAX_NUM_DRILLBITS = 5; + public static final int MIN_NUM_DRILLBITS = 1; + public static final int MAX_NUM_DRILLBITS = 5; + public static final int DEFAULT_NUM_DRILLBITS = 2; + public static final int RANDOM_NUM_DRILLBITS = -1; - public static final int MIN_PARALLELIZATION_WIDTH = 1; - public static final int MAX_PARALLELIZATION_WIDTH = 5; + public static final int MIN_PARALLELIZATION_WIDTH = 1; + public static final int MAX_PARALLELIZATION_WIDTH = 5; + public static final int DEFAULT_PARALLELIZATION_WIDTH = 2; + public static final int RANDOM_PARALLELIZATION_WIDTH = -1; @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.TYPE}) @@ -114,26 +116,28 @@ public abstract class DrillIntegrationTestBase extends DrillTestBase { /** * Sets the number of drillbits to create in the test cluster. * - * Default is -1, which indicates that the test framework should choose a - * random value in the inclusive range [{@link DrillIntegrationTestBase#MIN_NUM_DRILLBITS}, - * {@link DrillIntegrationTestBase#MAX_NUM_DRILLBITS}]. + * Default is DEFAULT_NUM_DRILLBITS. + * A value of RANDOM_NUM_DRILLBITS indicates that the test framework should choose a + * random value in the inclusive range [{@link DrillIntegrationTestBase_XXX#MIN_NUM_DRILLBITS}, + * {@link DrillIntegrationTestBase_XXX#MAX_NUM_DRILLBITS}]. */ - int bits() default -1; + int bits() default DEFAULT_NUM_DRILLBITS; /** * Sets the value of {@link ExecConstants#MAX_WIDTH_PER_NODE_KEY}, which determines * the level of parallelization per drillbit. * - * Default is -1, which indicates that the test framework should choose a - * random value in the inclusive range [{@link DrillIntegrationTestBase#MIN_PARALLELIZATION_WIDTH}, - * {@link DrillIntegrationTestBase#MAX_PARALLELIZATION_WIDTH}]. + * Default is DEFAULT_PARALLELIZATION_WIDTH. + * A value of RANDOM_PARALLELIZATION_WIDTH indicates that the test framework should choose a + * random value in the inclusive range [{@link DrillIntegrationTestBase_XXX#MIN_PARALLELIZATION_WIDTH}, + * {@link DrillIntegrationTestBase_XXX#MAX_PARALLELIZATION_WIDTH}]. */ - int width() default -1; + int width() default DEFAULT_PARALLELIZATION_WIDTH; } /** * The scope of a test cluster used together with - * {@link ClusterScope} annotations on {@link DrillIntegrationTestBase} subclasses. + * {@link ClusterScope} annotations on {@link DrillIntegrationTestBase_XXX} subclasses. */ public enum Scope { /** @@ -162,7 +166,7 @@ private static int getParallelizationWidth() { } private static A getAnnotation(Class clazz, Class annotationClass) { - if (clazz == Object.class || clazz == DrillIntegrationTestBase.class) { + if (clazz == Object.class || clazz == DrillIntegrationTestBase_XXX.class) { return null; } A annotation = clazz.getAnnotation(annotationClass); @@ -172,10 +176,6 @@ private static A getAnnotation(Class clazz, Class a return getAnnotation(clazz.getSuperclass(), annotationClass); } - protected static BufferAllocator getAllocator() { - return allocator; - } - protected static DrillTestCluster getCluster() { return cluster; } @@ -184,7 +184,6 @@ protected static DrillTestCluster getCluster() { public static void beforeClass() throws Exception { logger.debug("Initializing test framework for class: {}", getTestClass()); - allocator = new TopLevelAllocator(DrillConfig.create(TEST_CONFIGURATIONS)); if (cluster != null) { logger.info("Re-using existing test cluster of {} drillbits", cluster.bits().length); @@ -194,12 +193,12 @@ public static void beforeClass() throws Exception { scope = getCurrentClusterScope(getTestClass()); int size = getNumDrillbits(); - if (size <= 0 || size > MAX_NUM_DRILLBITS) { + if (size <= RANDOM_NUM_DRILLBITS || size > MAX_NUM_DRILLBITS) { size = randomIntBetween(MIN_NUM_DRILLBITS, MAX_NUM_DRILLBITS); } int width = getParallelizationWidth(); - if (width <= 0 || width > MAX_PARALLELIZATION_WIDTH) { + if (width <= RANDOM_PARALLELIZATION_WIDTH || width > MAX_PARALLELIZATION_WIDTH) { width = randomIntBetween(MIN_PARALLELIZATION_WIDTH, MAX_PARALLELIZATION_WIDTH); } @@ -228,10 +227,7 @@ public static void afterClass() throws IOException, InterruptedException { @After public void afterDrillIntegrationTest() { logger.info("Executing post-test cleanup: [{}]", getTestClass()); - if (allocator != null) { - allocator.close(); - allocator = null; - } + DrillMetrics.resetMetrics(); } protected static final class DrillTestCluster implements Closeable { @@ -430,7 +426,7 @@ public static String getPhysicalFileFromResource(final String resource) throws I final File file = File.createTempFile("tempfile", ".txt"); file.deleteOnExit(); PrintWriter printWriter = new PrintWriter(file); - printWriter.write(DrillIntegrationTestBase.getFile(resource)); + printWriter.write(DrillIntegrationTestBase_XXX.getFile(resource)); printWriter.close(); return file.getPath(); @@ -504,7 +500,7 @@ protected void setColumnWidths(int[] columnWidths) { protected int printResult(List results) throws SchemaChangeException { int rowCount = 0; - RecordBatchLoader loader = new RecordBatchLoader(getAllocator()); + RecordBatchLoader loader = new RecordBatchLoader(cluster.allocator()); for(QueryDataBatch result : results) { rowCount += result.getHeader().getRowCount(); loader.load(result.getHeader().getDef(), result.getData()); diff --git a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java index ff752745c8c..a8f98878e7c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java +++ b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java @@ -54,7 +54,7 @@ * the BaseTestQuery class, and instance of the builder is accessible through the testBuilder() method. */ public class DrillTestWrapper { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseTestQuery.class); + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillIntegrationTestBase.class); // TODO - when in JSON, read baseline in all text mode to avoid precision loss for decimal values @@ -282,8 +282,8 @@ protected void compareUnorderedResults() throws Exception { List actualRecords = new ArrayList<>(); try { - BaseTestQuery.test(testOptionSettingQueries); - actual = BaseTestQuery.testRunAndReturn(queryType, query); + DrillIntegrationTestBase.test(testOptionSettingQueries); + actual = DrillIntegrationTestBase.testRunAndReturn(queryType, query); checkNumBatches(actual); @@ -293,8 +293,8 @@ protected void compareUnorderedResults() throws Exception { // If baseline data was not provided to the test builder directly, we must run a query for the baseline, this includes // the cases where the baseline is stored in a file. if (baselineRecords == null) { - BaseTestQuery.test(baselineOptionSettingQueries); - expected = BaseTestQuery.testRunAndReturn(baselineQueryType, testBuilder.getValidationQuery()); + DrillIntegrationTestBase.test(baselineOptionSettingQueries); + expected = DrillIntegrationTestBase.testRunAndReturn(baselineQueryType, testBuilder.getValidationQuery()); addToMaterializedResults(expectedRecords, expected, loader, schema); } else { expectedRecords = baselineRecords; @@ -333,8 +333,8 @@ public void compareMergedOnHeapVectors() throws Exception { Map expectedSuperVectors; try { - BaseTestQuery.test(testOptionSettingQueries); - actual = BaseTestQuery.testRunAndReturn(queryType, query); + DrillIntegrationTestBase.test(testOptionSettingQueries); + actual = DrillIntegrationTestBase.testRunAndReturn(queryType, query); checkNumBatches(actual); @@ -346,8 +346,8 @@ public void compareMergedOnHeapVectors() throws Exception { // If baseline data was not provided to the test builder directly, we must run a query for the baseline, this includes // the cases where the baseline is stored in a file. if (baselineRecords == null) { - BaseTestQuery.test(baselineOptionSettingQueries); - expected = BaseTestQuery.testRunAndReturn(baselineQueryType, testBuilder.getValidationQuery()); + DrillIntegrationTestBase.test(baselineOptionSettingQueries); + expected = DrillIntegrationTestBase.testRunAndReturn(baselineQueryType, testBuilder.getValidationQuery()); expectedSuperVectors = addToCombinedVectorResults(expected, loader, schema); } else { // data is built in the TestBuilder in a row major format as it is provided by the user @@ -374,8 +374,8 @@ public void compareResultsHyperVector() throws Exception { RecordBatchLoader loader = new RecordBatchLoader(getAllocator()); BatchSchema schema = null; - BaseTestQuery.test(testOptionSettingQueries); - List results = BaseTestQuery.testRunAndReturn(queryType, query); + DrillIntegrationTestBase.test(testOptionSettingQueries); + List results = DrillIntegrationTestBase.testRunAndReturn(queryType, query); checkNumBatches(results); @@ -384,8 +384,8 @@ public void compareResultsHyperVector() throws Exception { Map actualSuperVectors = addToHyperVectorMap(results, loader, schema); - BaseTestQuery.test(baselineOptionSettingQueries); - List expected = BaseTestQuery.testRunAndReturn(baselineQueryType, testBuilder.getValidationQuery()); + DrillIntegrationTestBase.test(baselineOptionSettingQueries); + List expected = DrillIntegrationTestBase.testRunAndReturn(baselineQueryType, testBuilder.getValidationQuery()); Map expectedSuperVectors = addToHyperVectorMap(expected, loader, schema); diff --git a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java index f9e0b008142..9fbef6adbd2 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java +++ b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java @@ -37,7 +37,7 @@ import com.google.common.base.Strings; -public class PlanTestBase extends BaseTestQuery { +public class PlanTestBase extends DrillIntegrationTestBase { protected static final String OPTIQ_FORMAT = "text"; protected static final String JSON_FORMAT = "json"; diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestAggNullable.java b/exec/java-exec/src/test/java/org/apache/drill/TestAggNullable.java index 34850ba4013..e58422d0824 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestAggNullable.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestAggNullable.java @@ -22,7 +22,7 @@ import org.apache.drill.common.util.TestTools; import org.junit.Test; -public class TestAggNullable extends BaseTestQuery{ +public class TestAggNullable extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestAggNullable.class); static final String WORKING_PATH = TestTools.getWorkingPath(); diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java b/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java index 10fd5da5e45..41e2cbd06d0 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java @@ -32,7 +32,7 @@ * Run several tpch queries and inject an OutOfMemoryException in ScanBatch that will cause an OUT_OF_MEMORY outcome to * be propagated downstream. Make sure the proper "memory error" message is sent to the client. */ -public class TestAllocationException extends BaseTestQuery { +public class TestAllocationException extends DrillIntegrationTestBase { private static final String SINGLE_MODE = "ALTER SESSION SET `planner.disable_exchanges` = true"; diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestAltSortQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestAltSortQueries.java index a554db2a2e0..a9f8fd42cf7 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestAltSortQueries.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestAltSortQueries.java @@ -19,7 +19,7 @@ import org.junit.Test; -public class TestAltSortQueries extends BaseTestQuery{ +public class TestAltSortQueries extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestAltSortQueries.class); @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java b/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java index b5ed1d1c071..834ca666496 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java @@ -22,7 +22,7 @@ import org.junit.Ignore; import org.junit.Test; -public class TestBugFixes extends BaseTestQuery { +public class TestBugFixes extends DrillIntegrationTestBase { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestBugFixes.class); @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java index 330ff127c03..2d9d7a8bca1 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java @@ -140,14 +140,14 @@ public TestBuilder sqlQuery(String query, Object... replacements) { } public TestBuilder sqlQueryFromFile(String queryFile) throws IOException { - String query = BaseTestQuery.getFile(queryFile); + String query = DrillIntegrationTestBase.getFile(queryFile); this.query = query; this.queryType = UserBitShared.QueryType.SQL; return this; } public TestBuilder physicalPlanFromFile(String queryFile) throws IOException { - String query = BaseTestQuery.getFile(queryFile); + String query = DrillIntegrationTestBase.getFile(queryFile); this.query = query; this.queryType = UserBitShared.QueryType.PHYSICAL; return this; @@ -348,7 +348,7 @@ public BaselineQueryTestBuilder sqlBaselineQuery(String baselineQuery) { // provide a path to a file containing a SQL query to use as a baseline public BaselineQueryTestBuilder sqlBaselineQueryFromFile(String baselineQueryFilename) throws IOException { - String baselineQuery = BaseTestQuery.getFile(baselineQueryFilename); + String baselineQuery = DrillIntegrationTestBase.getFile(baselineQueryFilename); return new BaselineQueryTestBuilder(baselineQuery, UserBitShared.QueryType.SQL, allocator, query, queryType, ordered, approximateEquality, baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries, highPerformanceComparison, expectedNumBatches); } @@ -356,7 +356,7 @@ public BaselineQueryTestBuilder sqlBaselineQueryFromFile(String baselineQueryFil // as physical plans are verbose, this is the only option provided for specifying them, we should enforce // that physical plans, or any large JSON strings do not live in the Java source as literals public BaselineQueryTestBuilder physicalPlanBaselineQueryFromFile(String baselinePhysicalPlanPath) throws IOException { - String baselineQuery = BaseTestQuery.getFile(baselinePhysicalPlanPath); + String baselineQuery = DrillIntegrationTestBase.getFile(baselinePhysicalPlanPath); return new BaselineQueryTestBuilder(baselineQuery, UserBitShared.QueryType.PHYSICAL, allocator, query, queryType, ordered, approximateEquality, baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries, highPerformanceComparison, expectedNumBatches); } diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestDisabledFunctionality.java b/exec/java-exec/src/test/java/org/apache/drill/TestDisabledFunctionality.java index f53cc0f28eb..5c2a2c1f2dd 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestDisabledFunctionality.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestDisabledFunctionality.java @@ -25,7 +25,7 @@ import org.apache.drill.exec.work.foreman.UnsupportedRelOperatorException; import org.junit.Test; -public class TestDisabledFunctionality extends BaseTestQuery{ +public class TestDisabledFunctionality extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestExampleQueries.class); @Test(expected = UserException.class) // see DRILL-2054 diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java index 6b74ecfdc5e..ec2dec2d374 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java @@ -29,7 +29,7 @@ import java.math.BigDecimal; import static org.apache.drill.TestBuilder.listOf; -public class TestExampleQueries extends BaseTestQuery { +public class TestExampleQueries extends DrillIntegrationTestBase { // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestExampleQueries.class); @Test // see DRILL-2328 diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java b/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java index deeb7cbca75..5c0f1e494fe 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java @@ -35,7 +35,7 @@ // currently using it with the assumption that the csv and json readers are well tested, and handling diverse // types in the test framework would require doing some redundant work to enable casting outside of Drill or // some better tooling to generate parquet files that have all of the parquet types -public class TestFrameworkTest extends BaseTestQuery{ +public class TestFrameworkTest extends DrillIntegrationTestBase { private static String CSV_COLS = " cast(columns[0] as bigint) employee_id, columns[1] as first_name, columns[2] as last_name "; diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java b/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java index e81c661c5f7..1bfac96f2b1 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java @@ -27,7 +27,7 @@ import java.math.BigDecimal; -public class TestFunctionsQuery extends BaseTestQuery { +public class TestFunctionsQuery extends DrillIntegrationTestBase { // enable decimal data type @BeforeClass diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestInList.java b/exec/java-exec/src/test/java/org/apache/drill/TestInList.java index b6218ec4426..78abef6e7f7 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestInList.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestInList.java @@ -19,10 +19,9 @@ import static org.junit.Assert.assertEquals; -import org.apache.drill.common.util.TestTools; import org.junit.Test; -public class TestInList extends BaseTestQuery{ +public class TestInList extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestInList.class); @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestJoinNullable.java b/exec/java-exec/src/test/java/org/apache/drill/TestJoinNullable.java index 320a992a2f1..1600e073b6e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestJoinNullable.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestJoinNullable.java @@ -20,10 +20,9 @@ import static org.junit.Assert.assertEquals; import org.apache.drill.common.util.TestTools; -import org.junit.Ignore; import org.junit.Test; -public class TestJoinNullable extends BaseTestQuery{ +public class TestJoinNullable extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJoinNullable.class); static final String WORKING_PATH = TestTools.getWorkingPath(); diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestOutOfMemoryOutcome.java b/exec/java-exec/src/test/java/org/apache/drill/TestOutOfMemoryOutcome.java index b270a8bfa08..1ffe9c58e67 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestOutOfMemoryOutcome.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestOutOfMemoryOutcome.java @@ -28,7 +28,7 @@ * Run several tpch queries and inject an OutOfMemoryException in ScanBatch that will cause an OUT_OF_MEMORY outcome to * be propagated downstream. Make sure the proper "memory error" message is sent to the client. */ -public class TestOutOfMemoryOutcome extends BaseTestQuery{ +public class TestOutOfMemoryOutcome extends DrillIntegrationTestBase { private static final String SINGLE_MODE = "ALTER SESSION SET `planner.disable_exchanges` = true"; diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestSchemaChange.java b/exec/java-exec/src/test/java/org/apache/drill/TestSchemaChange.java index 4465e79cfff..15da4262b72 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestSchemaChange.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestSchemaChange.java @@ -21,7 +21,7 @@ import org.junit.Ignore; import org.junit.Test; -public class TestSchemaChange extends BaseTestQuery { +public class TestSchemaChange extends DrillIntegrationTestBase { protected static final String WORKING_PATH = TestTools.getWorkingPath(); protected static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources"; diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java index b9dcacd8d22..cdeba20621c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java @@ -25,7 +25,7 @@ import static org.junit.Assert.assertEquals; -public class TestStarQueries extends BaseTestQuery{ +public class TestStarQueries extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestStarQueries.class); static final String WORKING_PATH = TestTools.getWorkingPath(); static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources"; diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTextJoin.java b/exec/java-exec/src/test/java/org/apache/drill/TestTextJoin.java index 7fd9c8ecd86..26f81c7ddda 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestTextJoin.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestTextJoin.java @@ -20,7 +20,7 @@ import org.apache.drill.common.util.TestTools; import org.junit.Test; -public class TestTextJoin extends BaseTestQuery{ +public class TestTextJoin extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTextJoin.class); static final String WORKING_PATH = TestTools.getWorkingPath(); diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java index 3ac956c27cb..51b09735061 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java @@ -20,7 +20,7 @@ import org.junit.Ignore; import org.junit.Test; -public class TestTpchDistributed extends BaseTestQuery { +public class TestTpchDistributed extends DrillIntegrationTestBase { // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTpchDistributed.class); private static void testDistributed(final String fileName) throws Exception { diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedStreaming.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedStreaming.java index 94433f49793..31644fd6ae9 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedStreaming.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedStreaming.java @@ -20,7 +20,7 @@ import org.junit.Ignore; import org.junit.Test; -public class TestTpchDistributedStreaming extends BaseTestQuery{ +public class TestTpchDistributedStreaming extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTpchDistributedStreaming.class); private void testDistributed(String fileName) throws Exception{ diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchExplain.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchExplain.java index e10417de3c5..6d22594d8ac 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestTpchExplain.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchExplain.java @@ -20,7 +20,7 @@ import org.junit.Ignore; import org.junit.Test; -public class TestTpchExplain extends BaseTestQuery{ +public class TestTpchExplain extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTpchExplain.class); private static final String EXPLAIN_PREFIX = "EXPLAIN PLAN FOR "; diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchLimit0.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchLimit0.java index 22471c8abbd..13cd80fbd1c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestTpchLimit0.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchLimit0.java @@ -20,7 +20,7 @@ import org.junit.Ignore; import org.junit.Test; -public class TestTpchLimit0 extends BaseTestQuery{ +public class TestTpchLimit0 extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTpchLimit0.class); private void testLimitZero(String fileName) throws Exception { diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchSingleMode.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchSingleMode.java index abefb16b161..07d19efe838 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestTpchSingleMode.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchSingleMode.java @@ -20,7 +20,7 @@ import org.junit.Ignore; import org.junit.Test; -public class TestTpchSingleMode extends BaseTestQuery{ +public class TestTpchSingleMode extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTpchSingleMode.class); private static final String SINGLE_MODE = "ALTER SESSION SET `planner.disable_exchanges` = true;"; diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java index 3e5bfa2a937..d727b369408 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java @@ -24,7 +24,7 @@ import org.apache.drill.exec.work.foreman.UnsupportedRelOperatorException; import org.junit.Test; -public class TestUnionAll extends BaseTestQuery{ +public class TestUnionAll extends DrillIntegrationTestBase { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestUnionAll.class); @Test // Simple Union-All over two scans diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestUnionDistinct.java b/exec/java-exec/src/test/java/org/apache/drill/TestUnionDistinct.java index add9787b029..f7e7214d953 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestUnionDistinct.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestUnionDistinct.java @@ -24,7 +24,7 @@ import org.apache.drill.exec.work.foreman.UnsupportedRelOperatorException; import org.junit.Test; -public class TestUnionDistinct extends BaseTestQuery { +public class TestUnionDistinct extends DrillIntegrationTestBase { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestUnionDistinct.class); @Test // Simple Unionover two scans diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java index 8a1aecb121a..cda8f60fbc7 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java @@ -18,10 +18,10 @@ package org.apache.drill.exec; import org.apache.drill.exec.metrics.DrillMetrics; -import org.apache.drill.test.DrillTest; +import org.apache.drill.test.DrillTestBase; import org.junit.After; -public class ExecTest extends DrillTest { +public class ExecTest extends DrillTestBase { @After public void clear(){ diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestQueriesOnLargeFile.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestQueriesOnLargeFile.java index 0a2407349a7..d94ff8d0cfb 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/TestQueriesOnLargeFile.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/TestQueriesOnLargeFile.java @@ -24,7 +24,7 @@ import java.io.PrintWriter; import java.util.List; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.record.RecordBatchLoader; import org.apache.drill.exec.rpc.user.QueryDataBatch; @@ -36,7 +36,7 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; -public class TestQueriesOnLargeFile extends BaseTestQuery { +public class TestQueriesOnLargeFile extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestQueriesOnLargeFile.class); private static File dataFile = null; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestRepeatedReaders.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestRepeatedReaders.java index ddb67bbf485..e14ed2e4fc1 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/TestRepeatedReaders.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/TestRepeatedReaders.java @@ -18,14 +18,14 @@ package org.apache.drill.exec; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.junit.BeforeClass; import org.junit.Test; -public class TestRepeatedReaders extends BaseTestQuery { +public class TestRepeatedReaders extends DrillIntegrationTestBase { static FileSystem fs; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java index 29b702cba95..b544771f9e5 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.util.FileUtils; import org.apache.drill.common.util.TestTools; @@ -27,7 +27,7 @@ import org.junit.Test; -public class TestWindowFunctions extends BaseTestQuery { +public class TestWindowFunctions extends DrillIntegrationTestBase { static final String WORKING_PATH = TestTools.getWorkingPath(); static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources"; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java index f2240cc700c..74b1ee7c0b1 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java @@ -19,7 +19,7 @@ import java.io.IOException; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.exec.compile.ClassTransformer.ClassSet; import org.apache.drill.exec.compile.sig.GeneratorMapping; import org.apache.drill.exec.compile.sig.MappingSet; @@ -35,7 +35,7 @@ import org.junit.BeforeClass; import org.junit.Test; -public class TestClassTransformation extends BaseTestQuery { +public class TestClassTransformation extends DrillIntegrationTestBase { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestClassTransformation.class); private static final int ITERATION_COUNT = Integer.valueOf(System.getProperty("TestClassTransformation.iteration", "1")); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java index e63bdc03a6f..32bb4c5bed6 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.compile; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.util.TestTools; import org.apache.drill.exec.ExecConstants; import org.junit.Ignore; @@ -25,7 +25,7 @@ import org.junit.Test; import org.junit.rules.TestRule; -public class TestLargeFileCompilation extends BaseTestQuery { +public class TestLargeFileCompilation extends DrillIntegrationTestBase { @Rule public final TestRule TIMEOUT = TestTools.getTimeoutRule(150000); // 150secs private static final String LARGE_QUERY_GROUP_BY; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestLogicalExprSerDe.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestLogicalExprSerDe.java index 420fc7523bf..fc50136e6e8 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestLogicalExprSerDe.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestLogicalExprSerDe.java @@ -18,7 +18,7 @@ * limitations under the License. */ -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.exec.ExecConstants; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -27,7 +27,7 @@ /** * Test LogicalExpressions are serialized and deserialized properly when query is planned into multiple fragments. */ -public class TestLogicalExprSerDe extends BaseTestQuery { +public class TestLogicalExprSerDe extends DrillIntegrationTestBase { @BeforeClass public static void setSliceCount() throws Exception { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java index c2ab18a7915..b3a4f16f411 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java @@ -17,12 +17,12 @@ */ package org.apache.drill.exec.expr; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.exec.proto.UserBitShared; import org.junit.Ignore; import org.junit.Test; -public class TestSchemaPathMaterialization extends BaseTestQuery { +public class TestSchemaPathMaterialization extends DrillIntegrationTestBase { @Test public void testSingleProjectionFromMultiLevelRepeatedList() throws Exception { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestStringFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestStringFunctions.java index e1ef7c9da62..f9085b7b63d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestStringFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestStringFunctions.java @@ -17,10 +17,10 @@ */ package org.apache.drill.exec.expr.fn.impl; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.Test; -public class TestStringFunctions extends BaseTestQuery { +public class TestStringFunctions extends DrillIntegrationTestBase { @Test public void testILike() throws Exception { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java index 78ebe0fe05c..c1fcd8dbb04 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java @@ -17,13 +17,12 @@ */ package org.apache.drill.exec.fn.impl; -import org.apache.drill.BaseTestQuery; -import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.util.TestTools; import org.junit.Ignore; import org.junit.Test; -public class TestAggregateFunctions extends BaseTestQuery { +public class TestAggregateFunctions extends DrillIntegrationTestBase { private static final String TEST_RES_PATH = TestTools.getWorkingPath() + "/src/test/resources"; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java index f8a133ab36e..50936f6b646 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java @@ -18,14 +18,14 @@ package org.apache.drill.exec.fn.impl; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -public class TestCastEmptyStrings extends BaseTestQuery { +public class TestCastEmptyStrings extends DrillIntegrationTestBase { // enable decimal data type @BeforeClass public static void enableDecimalDataType() throws Exception { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java index 3ba8743e1d1..6a2141c2bac 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java @@ -18,13 +18,11 @@ package org.apache.drill.exec.fn.impl; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.util.FileUtils; import org.junit.Test; -import java.util.Date; - -public class TestCastFunctions extends BaseTestQuery { +public class TestCastFunctions extends DrillIntegrationTestBase { @Test public void testVarbinaryToDate() throws Exception { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestContextFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestContextFunctions.java index a3c473fa024..f9a6d4a21bb 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestContextFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestContextFunctions.java @@ -17,10 +17,10 @@ */ package org.apache.drill.exec.fn.impl; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.Test; -public class TestContextFunctions extends BaseTestQuery { +public class TestContextFunctions extends DrillIntegrationTestBase { @Test public void userUDFForAnonymousConnection() throws Exception { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewDateFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewDateFunctions.java index 0e9276c7bd6..52c106803c5 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewDateFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewDateFunctions.java @@ -17,13 +17,13 @@ */ package org.apache.drill.exec.fn.impl; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; import org.junit.Test; -public class TestNewDateFunctions extends BaseTestQuery { +public class TestNewDateFunctions extends DrillIntegrationTestBase { DateTime date; DateTimeFormatter formatter; long unixTimeStamp = -1; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewSimpleRepeatedFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewSimpleRepeatedFunctions.java index 05807d6a9e3..b2f9239ecd9 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewSimpleRepeatedFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewSimpleRepeatedFunctions.java @@ -17,10 +17,10 @@ */ package org.apache.drill.exec.fn.impl; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.Test; -public class TestNewSimpleRepeatedFunctions extends BaseTestQuery { +public class TestNewSimpleRepeatedFunctions extends DrillIntegrationTestBase { @Test public void testRepeatedContainsForWildCards() throws Exception { testBuilder(). diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestFastComplexSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestFastComplexSchema.java index 955c93ea736..a50b6c097cd 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestFastComplexSchema.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestFastComplexSchema.java @@ -17,10 +17,10 @@ */ package org.apache.drill.exec.nested; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.Test; -public class TestFastComplexSchema extends BaseTestQuery { +public class TestFastComplexSchema extends DrillIntegrationTestBase { @Test public void test() throws Exception { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestNestedComplexSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestNestedComplexSchema.java index 05a53089a2f..5a6905b5f5e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestNestedComplexSchema.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestNestedComplexSchema.java @@ -17,10 +17,10 @@ */ package org.apache.drill.exec.nested; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.Test; -public class TestNestedComplexSchema extends BaseTestQuery { +public class TestNestedComplexSchema extends DrillIntegrationTestBase { @Test public void testNested1() throws Exception { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java index a911e29128b..6e46679ccb0 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java @@ -31,8 +31,7 @@ import mockit.Injectable; -import org.apache.drill.BaseTestQuery; -import org.apache.drill.TestBuilder; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.exec.compile.ClassTransformer; import org.apache.drill.exec.compile.ClassTransformer.ScalarReplacementOption; import org.apache.drill.exec.expr.fn.impl.DateUtility; @@ -51,14 +50,13 @@ import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.VarCharVector; import org.joda.time.DateTime; -import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; import com.google.common.base.Charsets; import com.google.common.io.Resources; -public class TestConvertFunctions extends BaseTestQuery { +public class TestConvertFunctions extends DrillIntegrationTestBase { // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestConvertFunctions.class); private static final String CONVERSION_TEST_LOGICAL_PLAN = "functions/conv/conversionTestWithLogicalPlan.json"; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggr.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggr.java index 3786bfdebd6..1cd5829ceb1 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggr.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggr.java @@ -18,12 +18,12 @@ package org.apache.drill.exec.physical.impl.agg; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.Ignore; import org.junit.Test; -public class TestHashAggr extends BaseTestQuery{ +public class TestHashAggr extends DrillIntegrationTestBase { @Test public void testQ6() throws Exception{ diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/broadcastsender/TestBroadcast.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/broadcastsender/TestBroadcast.java index f367a69203f..c76dd0548b7 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/broadcastsender/TestBroadcast.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/broadcastsender/TestBroadcast.java @@ -17,10 +17,10 @@ */ package org.apache.drill.exec.physical.impl.broadcastsender; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.Test; -public class TestBroadcast extends BaseTestQuery { +public class TestBroadcast extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestBroadcast.class); String broadcastQuery = "select * from " diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestLargeInClause.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestLargeInClause.java index 22826b4fe38..2644af7439b 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestLargeInClause.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestLargeInClause.java @@ -17,10 +17,10 @@ */ package org.apache.drill.exec.physical.impl.filter; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.Test; -public class TestLargeInClause extends BaseTestQuery { +public class TestLargeInClause extends DrillIntegrationTestBase { private static String getInIntList(int size){ StringBuffer sb = new StringBuffer(); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java index 39e36eb2105..244b711f2d4 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java @@ -22,7 +22,7 @@ import static org.junit.Assert.assertEquals; import com.google.common.collect.Lists; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.TestBuilder; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.fn.interp.TestConstantFolding; @@ -34,7 +34,7 @@ import java.util.List; -public class TestFlatten extends BaseTestQuery { +public class TestFlatten extends DrillIntegrationTestBase { /** * enable this if you have the following files: @@ -58,7 +58,7 @@ public void testFlattenFailure() throws Exception { public void testFlatten_Drill2162_complex() throws Exception { String path = folder.getRoot().toPath().toString(); - String jsonRecords = BaseTestQuery.getFile("flatten/complex_transaction_example_data.json"); + String jsonRecords = DrillIntegrationTestBase.getFile("flatten/complex_transaction_example_data.json"); int numCopies = 700; new TestConstantFolding.SmallFileCreator(folder) .setRecord(jsonRecords) @@ -216,7 +216,7 @@ public void drill1660() throws Exception { public void drill1673() throws Exception { String path = folder.getRoot().toPath().toString(); - String jsonRecords = BaseTestQuery.getFile("store/json/1673.json"); + String jsonRecords = DrillIntegrationTestBase.getFile("store/json/1673.json"); int numCopies = 25000; new TestConstantFolding.SmallFileCreator(folder) .setRecord(jsonRecords) diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java index a70a3f8d5be..88837793a29 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java @@ -19,13 +19,12 @@ package org.apache.drill.exec.physical.impl.join; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; -public class TestHashJoinAdvanced extends BaseTestQuery { +public class TestHashJoinAdvanced extends DrillIntegrationTestBase { // Have to disable merge join, if this testcase is to test "HASH-JOIN". @BeforeClass diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinAdvanced.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinAdvanced.java index c706638db62..eca0ffcca87 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinAdvanced.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinAdvanced.java @@ -17,12 +17,12 @@ */ package org.apache.drill.exec.physical.impl.join; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -public class TestMergeJoinAdvanced extends BaseTestQuery { +public class TestMergeJoinAdvanced extends DrillIntegrationTestBase { // Have to disable hash join to test merge join in this class @BeforeClass diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitWithExchanges.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitWithExchanges.java index 0e4d734294a..9ead07773d3 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitWithExchanges.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitWithExchanges.java @@ -17,10 +17,10 @@ */ package org.apache.drill.exec.physical.impl.limit; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.Test; -public class TestLimitWithExchanges extends BaseTestQuery { +public class TestLimitWithExchanges extends DrillIntegrationTestBase { @Test public void testLimitWithExchanges() throws Exception{ diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSort.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSort.java index 5066c839cd0..d8373c8ca81 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSort.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSort.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.physical.impl.sort; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.util.JsonStringArrayList; import org.apache.drill.exec.util.JsonStringHashMap; @@ -27,7 +27,7 @@ * Placeholder for all sort related test. Can be used as we move * more tests to use the new test framework */ -public class TestSort extends BaseTestQuery { +public class TestSort extends DrillIntegrationTestBase { private static final JsonStringHashMap x = new JsonStringHashMap(); private static final JsonStringArrayList repeated_map = new JsonStringArrayList<>(); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java index 15b13d84e1c..dbff8e3685d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java @@ -19,10 +19,10 @@ import static org.junit.Assert.assertEquals; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.Test; -public class TestSVRemover extends BaseTestQuery { +public class TestSVRemover extends DrillIntegrationTestBase { @Test public void testSelectionVectorRemoval() throws Exception { int numOutputRecords = testPhysical(getFile("remover/test1.json")); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/TestWindowFrame.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/TestWindowFrame.java index 23e9b467895..0fc34830411 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/TestWindowFrame.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/TestWindowFrame.java @@ -19,7 +19,7 @@ import java.util.Properties; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.DrillTestWrapper; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.UserRemoteException; @@ -32,7 +32,7 @@ import org.junit.BeforeClass; import org.junit.Test; -public class TestWindowFrame extends BaseTestQuery { +public class TestWindowFrame extends DrillIntegrationTestBase { private static final String TEST_RES_PATH = TestTools.getWorkingPath() + "/src/test/resources"; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java index 5dc7bec4176..235e490e303 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java @@ -20,7 +20,7 @@ import java.math.BigDecimal; import java.sql.Date; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.fn.interp.TestConstantFolding; import org.apache.drill.exec.planner.physical.PlannerSettings; @@ -30,14 +30,13 @@ import org.joda.time.DateTime; import org.joda.time.Period; import org.junit.AfterClass; -import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -public class TestParquetWriter extends BaseTestQuery { +public class TestParquetWriter extends DrillIntegrationTestBase { // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestParquetWriter.class); @Rule diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriterEmptyFiles.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriterEmptyFiles.java index 2848b68d53f..d57837e4f50 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriterEmptyFiles.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriterEmptyFiles.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.physical.impl.writer; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.exec.ExecConstants; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -26,7 +26,7 @@ import org.junit.BeforeClass; import org.junit.Test; -public class TestParquetWriterEmptyFiles extends BaseTestQuery { +public class TestParquetWriterEmptyFiles extends DrillIntegrationTestBase { private static FileSystem fs; @@ -47,7 +47,7 @@ public void testWriteEmptyFile() throws Exception { try { test("CREATE TABLE dfs_test.tmp.%s AS SELECT * FROM cp.`employee.json` WHERE 1=0", outputFile); - final Path path = new Path(getDfsTestTmpSchemaLocation(), outputFile); + final Path path = new Path(getCluster().tmpSchemaLocation(), outputFile); Assert.assertFalse(fs.exists(path)); } finally { deleteTableIfExists(outputFile); @@ -104,7 +104,7 @@ public void testWriteEmptyFileAfterFlush() throws Exception { private static boolean deleteTableIfExists(String tableName) { try { - Path path = new Path(getDfsTestTmpSchemaLocation(), tableName); + Path path = new Path(getCluster().tmpSchemaLocation(), tableName); if (fs.exists(path)) { return fs.delete(path, true); } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java index f4d505db499..ff4305a5cea 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java @@ -22,7 +22,7 @@ import java.util.List; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.planner.physical.PlannerSettings; @@ -40,7 +40,7 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; -public class TestWriter extends BaseTestQuery { +public class TestWriter extends DrillIntegrationTestBase { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestWriter.class); static FileSystem fs; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java index b34a4667d5b..b436a50ea54 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java @@ -22,7 +22,7 @@ import java.util.List; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.SchemaPath; @@ -43,7 +43,7 @@ import com.google.common.io.Files; @Ignore -public class TestSimpleExternalSort extends BaseTestQuery { +public class TestSimpleExternalSort extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleExternalSort.class); DrillConfig c = DrillConfig.create(); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestCustomUserAuthenticator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestCustomUserAuthenticator.java index d9b463446ee..2b08807003e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestCustomUserAuthenticator.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestCustomUserAuthenticator.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.rpc.user.security; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.rpc.RpcException; @@ -35,7 +35,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; -public class TestCustomUserAuthenticator extends BaseTestQuery { +public class TestCustomUserAuthenticator extends DrillIntegrationTestBase { @BeforeClass public static void setupCluster() { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java index ce09f687241..eb6f3597d59 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java @@ -31,7 +31,7 @@ import java.util.Map; import org.apache.commons.math3.util.Pair; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.QueryTestUtil; import org.apache.drill.SingleRowListener; import org.apache.drill.common.AutoCloseables; @@ -80,7 +80,7 @@ import org.apache.drill.exec.work.foreman.ForemanException; import org.apache.drill.exec.work.foreman.ForemanSetupException; import org.apache.drill.exec.work.fragment.FragmentExecutor; -import org.apache.drill.test.DrillTest; +import org.apache.drill.test.DrillTestBase; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -94,7 +94,7 @@ * Test how resilient drillbits are to throwing exceptions during various phases of query * execution by injecting exceptions at various points, and to cancellations in various phases. */ -public class TestDrillbitResilience extends DrillTest { +public class TestDrillbitResilience extends DrillTestBase { private static final Logger logger = org.slf4j.LoggerFactory.getLogger(TestDrillbitResilience.class); private static ZookeeperHelper zkHelper; @@ -864,7 +864,7 @@ public void memoryLeaksWhenCancelled() { .build(); String query = null; try { - query = BaseTestQuery.getFile("queries/tpch/09.sql"); + query = DrillIntegrationTestBase.getFile("queries/tpch/09.sql"); query = query.substring(0, query.length() - 1); // drop the ";" } catch (final IOException e) { fail("Failed to get query file: " + e); @@ -910,7 +910,7 @@ public void memoryLeaksWhenFailed() { String query = null; try { - query = BaseTestQuery.getFile("queries/tpch/09.sql"); + query = DrillIntegrationTestBase.getFile("queries/tpch/09.sql"); query = query.substring(0, query.length() - 1); // drop the ";" } catch (final IOException e) { fail("Failed to get query file: " + e); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptions.java index 71ac653b623..8ec886af61f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptions.java @@ -17,11 +17,11 @@ */ package org.apache.drill.exec.server; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.exec.ExecConstants; import org.junit.Test; -public class TestOptions extends BaseTestQuery{ +public class TestOptions extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestOptions.class); @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestTpcdsSf1Leaks.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestTpcdsSf1Leaks.java index ba19e0d7d48..e177d2d8916 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestTpcdsSf1Leaks.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestTpcdsSf1Leaks.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.server; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import static org.apache.drill.exec.ExecConstants.SLICE_TARGET; import static org.apache.drill.exec.ExecConstants.SLICE_TARGET_DEFAULT; @@ -43,7 +43,7 @@ * */ @Ignore -public class TestTpcdsSf1Leaks extends BaseTestQuery { +public class TestTpcdsSf1Leaks extends DrillIntegrationTestBase { @Rule final public TestRule TIMEOUT = new Timeout(0); // wait forever diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestBaseViewSupport.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestBaseViewSupport.java index a8f5bbb2f67..71f01d48177 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestBaseViewSupport.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestBaseViewSupport.java @@ -18,7 +18,7 @@ package org.apache.drill.exec.sql; import com.google.common.base.Strings; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.TestBuilder; import java.util.List; @@ -28,7 +28,7 @@ * Base class for view tests. It has utility methods which can be used when writing tests for views on tables * in different storage engines such as Hive, HBase etc. */ -public class TestBaseViewSupport extends BaseTestQuery { +public class TestBaseViewSupport extends DrillIntegrationTestBase { private static AtomicInteger viewSeqNum = new AtomicInteger(0); /** diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestCTAS.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestCTAS.java index 33288e367cd..c9943c39990 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestCTAS.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestCTAS.java @@ -18,12 +18,12 @@ package org.apache.drill.exec.sql; import org.apache.commons.io.FileUtils; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.Test; import java.io.File; -public class TestCTAS extends BaseTestQuery { +public class TestCTAS extends DrillIntegrationTestBase { @Test // DRILL-2589 public void withDuplicateColumnsInDef1() throws Exception { ctasErrorTestHelper("CREATE TABLE %s.%s AS SELECT region_id, region_id FROM cp.`region.json`", diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java index 8e7498f6d18..0ba5c3201dd 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java @@ -18,7 +18,7 @@ package org.apache.drill.exec.sql; import com.google.common.collect.ImmutableList; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.TestBuilder; import org.junit.Test; @@ -31,7 +31,7 @@ * -- USE schema * -- SHOW FILES */ -public class TestInfoSchema extends BaseTestQuery { +public class TestInfoSchema extends DrillIntegrationTestBase { @Test public void selectFromAllTables() throws Exception{ test("select * from INFORMATION_SCHEMA.SCHEMATA"); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestSimpleCastFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestSimpleCastFunctions.java index 4dae7fe6377..3bc97a7097f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestSimpleCastFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestSimpleCastFunctions.java @@ -17,10 +17,10 @@ */ package org.apache.drill.exec.sql; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.Test; -public class TestSimpleCastFunctions extends BaseTestQuery { +public class TestSimpleCastFunctions extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleCastFunctions.class); @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestWithClause.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestWithClause.java index 3a3204587d3..5238c3517b5 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestWithClause.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestWithClause.java @@ -17,11 +17,11 @@ */ package org.apache.drill.exec.sql; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.Ignore; import org.junit.Test; -public class TestWithClause extends BaseTestQuery { +public class TestWithClause extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestWithClause.class); @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestTimedRunnable.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestTimedRunnable.java index 44bb8b651f9..cd92cc1cc23 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestTimedRunnable.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestTimedRunnable.java @@ -20,7 +20,7 @@ import com.google.common.collect.Lists; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.util.TestTools; -import org.apache.drill.test.DrillTest; +import org.apache.drill.test.DrillTestBase; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestRule; @@ -35,7 +35,7 @@ /** * Unit testing for {@link TimedRunnable}. */ -public class TestTimedRunnable extends DrillTest { +public class TestTimedRunnable extends DrillTestBase { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTimedRunnable.class); @Rule diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java index 2d2522ba8d4..90f9be59d10 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java @@ -17,14 +17,14 @@ */ package org.apache.drill.exec.store.avro; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.Test; /** * Unit tests for Avro record reader. */ -public class AvroFormatTest extends BaseTestQuery { +public class AvroFormatTest extends DrillIntegrationTestBase { // XXX // 1. Need to test nested field names with same name as top-level names for conflict. diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestGlob.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestGlob.java index 8b4f7e722be..ec43318fae5 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestGlob.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestGlob.java @@ -17,11 +17,11 @@ */ package org.apache.drill.exec.store.dfs; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.util.TestTools; import org.junit.Test; -public class TestGlob extends BaseTestQuery { +public class TestGlob extends DrillIntegrationTestBase { String MULTILEVEL = TestTools.getWorkingPath() + "/../java-exec/src/test/resources/multilevel"; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/Drill2283InfoSchemaVarchar1BugTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/Drill2283InfoSchemaVarchar1BugTest.java index 5f5d2a347b4..4db796eefe5 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/Drill2283InfoSchemaVarchar1BugTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/Drill2283InfoSchemaVarchar1BugTest.java @@ -17,10 +17,10 @@ */ package org.apache.drill.exec.store.ischema; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.Test; -public class Drill2283InfoSchemaVarchar1BugTest extends BaseTestQuery { +public class Drill2283InfoSchemaVarchar1BugTest extends DrillIntegrationTestBase { @Test public void testInfoSchemaStringsNotOfLength1() throws Exception { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java index 8639e1cc9f8..7efa7b8a6cd 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java @@ -17,8 +17,7 @@ */ package org.apache.drill.exec.store.json; -import org.apache.drill.BaseTestQuery; -import org.apache.drill.TestBuilder; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.proto.UserBitShared; import org.junit.Test; @@ -27,7 +26,7 @@ import static org.junit.Assert.assertTrue; -public class TestJsonRecordReader extends BaseTestQuery{ +public class TestJsonRecordReader extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJsonRecordReader.class); @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java index af1b8960c8b..07c91b48a8f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java @@ -32,7 +32,7 @@ import mockit.Injectable; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.SchemaPath; @@ -82,7 +82,7 @@ import com.google.common.io.Files; @Ignore -public class ParquetRecordReaderTest extends BaseTestQuery { +public class ParquetRecordReaderTest extends DrillIntegrationTestBase { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRecordReaderTest.class); static final boolean VERBOSE_DEBUG = false; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java index 6397ef77664..c17916af872 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java @@ -17,11 +17,10 @@ */ package org.apache.drill.exec.store.parquet; -import org.apache.drill.BaseTestQuery; -import org.apache.drill.exec.proto.UserBitShared; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.Test; -public class TestParquetComplex extends BaseTestQuery { +public class TestParquetComplex extends DrillIntegrationTestBase { private static final String DATAFILE = "cp.`store/parquet/complex/complex.parquet`"; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetScan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetScan.java index 69759e6706b..27bf45b2579 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetScan.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetScan.java @@ -18,7 +18,7 @@ package org.apache.drill.exec.store.parquet; import com.google.common.io.Resources; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -26,7 +26,7 @@ import org.junit.BeforeClass; import org.junit.Test; -public class TestParquetScan extends BaseTestQuery { +public class TestParquetScan extends DrillIntegrationTestBase { static FileSystem fs; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java index 4dff9285022..c8ecd9d455d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java @@ -17,13 +17,13 @@ */ package org.apache.drill.exec.store.parquet.columnreaders; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -public class TestColumnReaderFactory extends BaseTestQuery { +public class TestColumnReaderFactory extends DrillIntegrationTestBase { // enable decimal data type @BeforeClass public static void enableDecimalDataType() throws Exception { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestDateReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestDateReader.java index e95842cc982..1d8f6572fda 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestDateReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestDateReader.java @@ -17,10 +17,10 @@ */ package org.apache.drill.exec.store.parquet.columnreaders; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.Test; -public class TestDateReader extends BaseTestQuery { +public class TestDateReader extends DrillIntegrationTestBase { /** * check if DateReader works well with dictionary encoding. diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java index 05ca7fc2a78..9cc6b89f6f9 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.store.parquet2; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -25,7 +25,7 @@ import java.math.BigDecimal; -public class TestDrillParquetReader extends BaseTestQuery { +public class TestDrillParquetReader extends DrillIntegrationTestBase { // enable decimal data type @BeforeClass public static void enableDecimalDataType() throws Exception { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java index e86fc286fa6..ffbb2fa329a 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java @@ -17,11 +17,11 @@ */ package org.apache.drill.exec.store.sys; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.exec.ExecConstants; import org.junit.Test; -public class TestSystemTable extends BaseTestQuery { +public class TestSystemTable extends DrillIntegrationTestBase { // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSystemTable.class); @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestNewTextReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestNewTextReader.java index e63e5288e4a..5c2042ff5c7 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestNewTextReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestNewTextReader.java @@ -21,12 +21,12 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.exceptions.UserRemoteException; import org.apache.drill.exec.proto.UserBitShared.DrillPBError.ErrorType; import org.junit.Test; -public class TestNewTextReader extends BaseTestQuery { +public class TestNewTextReader extends DrillIntegrationTestBase { @Test public void fieldDelimiterWithinQuotes() throws Exception { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestTextColumn.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestTextColumn.java index 882033a1152..1b2177fd03f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestTextColumn.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestTextColumn.java @@ -23,7 +23,7 @@ import java.util.Arrays; import java.util.List; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.record.RecordBatchLoader; import org.apache.drill.exec.record.VectorWrapper; @@ -31,7 +31,7 @@ import org.apache.drill.exec.vector.ValueVector; import org.junit.Test; -public class TestTextColumn extends BaseTestQuery{ +public class TestTextColumn extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTextColumn.class); @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestCountDownLatchInjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestCountDownLatchInjection.java index c911f791956..e6cf650268c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestCountDownLatchInjection.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestCountDownLatchInjection.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.testing; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.concurrent.ExtendedLatch; import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.proto.UserBitShared.UserCredentials; @@ -31,7 +31,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -public class TestCountDownLatchInjection extends BaseTestQuery { +public class TestCountDownLatchInjection extends DrillIntegrationTestBase { private static final UserSession session = UserSession.Builder.newBuilder() .withCredentials(UserCredentials.newBuilder() diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestExceptionInjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestExceptionInjection.java index 40620c2175e..047db042d84 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestExceptionInjection.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestExceptionInjection.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.testing; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.exec.ZookeeperHelper; import org.apache.drill.exec.exception.DrillbitStartupException; @@ -35,7 +35,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -public class TestExceptionInjection extends BaseTestQuery { +public class TestExceptionInjection extends DrillIntegrationTestBase { private static final String NO_THROW_FAIL = "Didn't throw expected exception"; private static final UserSession session = UserSession.Builder.newBuilder() diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestPauseInjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestPauseInjection.java index f07f67697d4..bdab4ec3dc1 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestPauseInjection.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestPauseInjection.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.testing; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.concurrent.ExtendedLatch; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.exec.ZookeeperHelper; @@ -39,14 +39,14 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -public class TestPauseInjection extends BaseTestQuery { +public class TestPauseInjection extends DrillIntegrationTestBase { private static final UserSession session = UserSession.Builder.newBuilder() .withCredentials(UserCredentials.newBuilder() .setUserName("foo") .build()) .withUserProperties(UserProperties.getDefaultInstance()) - .withOptionManager(bits[0].getContext().getOptionManager()) + .withOptionManager(getCluster().randomBit().getContext().getOptionManager()) .build(); /** @@ -126,7 +126,7 @@ public void pauseInjected() { ControlsInjectionUtil.setControls(session, controls); - final QueryContext queryContext = new QueryContext(session, bits[0].getContext()); + final QueryContext queryContext = new QueryContext(session, getCluster().randomBit().getContext()); (new ResumingThread(queryContext, trigger, ex, expectedDuration)).start(); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestResourceLeak.java b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestResourceLeak.java index d7e317cfb92..56f51248570 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestResourceLeak.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestResourceLeak.java @@ -36,7 +36,7 @@ import org.apache.drill.exec.memory.TopLevelAllocator; import org.apache.drill.exec.server.Drillbit; import org.apache.drill.exec.server.RemoteServiceSet; -import org.apache.drill.test.DrillTest; +import org.apache.drill.test.DrillTestBase; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -47,7 +47,7 @@ import java.net.URL; import java.util.Properties; -public class TestResourceLeak extends DrillTest { +public class TestResourceLeak extends DrillTestBase { private static DrillClient client; private static Drillbit bit; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/TestEmptyPopulation.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/TestEmptyPopulation.java index f7f2706ed6b..2504db97c2d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/TestEmptyPopulation.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/TestEmptyPopulation.java @@ -19,7 +19,7 @@ import static org.apache.drill.TestBuilder.listOf; import static org.apache.drill.TestBuilder.mapOf; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.memory.TopLevelAllocator; import org.apache.drill.exec.vector.UInt4Vector; @@ -30,7 +30,7 @@ import org.mockito.runners.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) -public class TestEmptyPopulation extends BaseTestQuery { +public class TestEmptyPopulation extends DrillIntegrationTestBase { private UInt4Vector offsets; private UInt4Vector.Accessor accessor; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java index 808d98cfc0f..029bfa2d395 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java @@ -21,7 +21,7 @@ import java.util.List; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.exec.record.RecordBatchLoader; import org.apache.drill.exec.rpc.user.QueryDataBatch; import org.apache.drill.exec.util.JsonStringArrayList; @@ -29,7 +29,7 @@ import org.apache.drill.exec.vector.ValueVector; import org.junit.Test; -public class TestJsonReaderWithSparseFiles extends BaseTestQuery { +public class TestJsonReaderWithSparseFiles extends DrillIntegrationTestBase { private interface Function { void apply(T param); } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexToJson.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexToJson.java index 656429e2988..8e3d3e1c690 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexToJson.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexToJson.java @@ -22,7 +22,7 @@ import java.util.List; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.proto.UserBitShared.RecordBatchDef; @@ -30,7 +30,7 @@ import org.apache.drill.exec.rpc.user.QueryDataBatch; import org.junit.Test; -public class TestComplexToJson extends BaseTestQuery { +public class TestComplexToJson extends DrillIntegrationTestBase { @Test public void test() throws Exception { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java index 521a41d468d..4ce0a0b46c6 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java @@ -18,10 +18,10 @@ package org.apache.drill.exec.vector.complex.writer; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.Test; -public class TestComplexTypeReader extends BaseTestQuery{ +public class TestComplexTypeReader extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestComplexTypeReader.class); @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeWriter.java index cb7bef2597a..ce92de21e8d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeWriter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeWriter.java @@ -18,10 +18,10 @@ package org.apache.drill.exec.vector.complex.writer; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.junit.Test; -public class TestComplexTypeWriter extends BaseTestQuery{ +public class TestComplexTypeWriter extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestComplexTypeReader.class); @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java index f403108604e..93d61471820 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java @@ -24,12 +24,12 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.util.TestTools; import org.apache.drill.exec.ExecConstants; import org.junit.Test; -public class TestExtendedTypes extends BaseTestQuery { +public class TestExtendedTypes extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestExtendedTypes.class); @Test @@ -52,7 +52,7 @@ public void checkReadWriteExtended() throws Exception { // check that original file and new file match. final byte[] originalData = Files.readAllBytes(Paths.get(originalFile)); - final byte[] newData = Files.readAllBytes(Paths.get(BaseTestQuery.getDfsTestTmpSchemaLocation() + '/' + newTable + final byte[] newData = Files.readAllBytes(Paths.get(DrillIntegrationTestBase.getDfsTestTmpSchemaLocation() + '/' + newTable + "/0_0_0.json")); assertEquals(new String(originalData), new String(newData)); } finally { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java index 7d6c71c5094..fa04d194502 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java @@ -31,7 +31,7 @@ import java.util.List; import java.util.zip.GZIPOutputStream; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.exception.SchemaChangeException; @@ -49,7 +49,7 @@ import com.google.common.io.Files; import org.junit.rules.TemporaryFolder; -public class TestJsonReader extends BaseTestQuery { +public class TestJsonReader extends DrillIntegrationTestBase { // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJsonReader.class); private static final boolean VERBOSE_DEBUG = false; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/TestSpoolingBuffer.java b/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/TestSpoolingBuffer.java index 271f29f7229..7cfdc445aef 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/TestSpoolingBuffer.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/TestSpoolingBuffer.java @@ -21,10 +21,9 @@ import java.util.List; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.util.FileUtils; -import org.apache.drill.exec.ExecTest; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.rpc.user.QueryDataBatch; import org.apache.drill.exec.server.Drillbit; @@ -34,7 +33,7 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; -public class TestSpoolingBuffer extends BaseTestQuery { +public class TestSpoolingBuffer extends DrillIntegrationTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSpoolingBuffer.class); @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/work/fragment/TestFragmentExecutorCancel.java b/exec/java-exec/src/test/java/org/apache/drill/exec/work/fragment/TestFragmentExecutorCancel.java index eb5cc499f22..21240ff1b8c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/work/fragment/TestFragmentExecutorCancel.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/work/fragment/TestFragmentExecutorCancel.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.work.fragment; -import org.apache.drill.BaseTestQuery; +import org.apache.drill.DrillIntegrationTestBase; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.testing.ControlsInjectionUtil; @@ -28,7 +28,7 @@ * Run several tpch queries and inject an OutOfMemoryException in ScanBatch that will cause an OUT_OF_MEMORY outcome to * be propagated downstream. Make sure the proper "memory error" message is sent to the client. */ -public class TestFragmentExecutorCancel extends BaseTestQuery { +public class TestFragmentExecutorCancel extends DrillIntegrationTestBase { @Test public void testCancelNonRunningFragments() throws Exception{ diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/DrillResultSetTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/DrillResultSetTest.java index 2df173e7221..cf9257e106d 100644 --- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/DrillResultSetTest.java +++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/DrillResultSetTest.java @@ -20,46 +20,23 @@ import static org.junit.Assert.*; import java.sql.Connection; -import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Properties; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; - -import net.hydromatic.linq4j.Ord; - -import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.logical.LogicalPlan; -import org.apache.drill.common.logical.data.LogicalOperator; -import org.apache.drill.common.util.Hook; + import org.apache.drill.exec.ExecConstants; import org.apache.drill.jdbc.test.JdbcAssert; -import org.apache.drill.test.DrillTest; +import org.apache.drill.test.DrillTestBase; import org.junit.AfterClass; -import org.junit.Assert; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.core.StringContains.containsString; -import com.google.common.base.Function; -import com.google.common.base.Predicate; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.google.common.collect.ImmutableSet.Builder; - -public class DrillResultSetTest extends DrillTest { +public class DrillResultSetTest extends DrillTestBase { // TODO: Move Jetty status server disabling to DrillTest. private static final String STATUS_SERVER_PROPERTY_NAME = diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/DriverTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/DriverTest.java index 7935215b900..c4f3a814ab0 100644 --- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/DriverTest.java +++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/DriverTest.java @@ -23,7 +23,7 @@ import java.util.Properties; import org.apache.drill.exec.ExecConstants; -import org.apache.drill.test.DrillTest; +import org.apache.drill.test.DrillTestBase; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Ignore; @@ -39,7 +39,7 @@ /** * (Some) unit and integration tests for org.apache.drill.jdbc.Driver. */ -public class DriverTest extends DrillTest { +public class DriverTest extends DrillTestBase { // TODO: Move Jetty status server disabling to DrillTest. private static final String STATUS_SERVER_PROPERTY_NAME = diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/proxy/TracingProxyDriverClassLoadingTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/proxy/TracingProxyDriverClassLoadingTest.java index 13becac3db2..fd1711b9118 100644 --- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/proxy/TracingProxyDriverClassLoadingTest.java +++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/proxy/TracingProxyDriverClassLoadingTest.java @@ -17,7 +17,7 @@ */ package org.apache.drill.jdbc.proxy; -import org.apache.drill.test.DrillTest; +import org.apache.drill.test.DrillTestBase; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -39,7 +39,7 @@ /** * Test of TracingProxyDriver's loading of driver class. */ -public class TracingProxyDriverClassLoadingTest extends DrillTest { +public class TracingProxyDriverClassLoadingTest extends DrillTestBase { @Ignore( "except when run in own JVM (so Drill Driver not already loaded)" ) @Test diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/proxy/TracingProxyDriverTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/proxy/TracingProxyDriverTest.java index 6e8a17c039a..b1eeffb355e 100644 --- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/proxy/TracingProxyDriverTest.java +++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/proxy/TracingProxyDriverTest.java @@ -17,7 +17,7 @@ */ package org.apache.drill.jdbc.proxy; -import org.apache.drill.test.DrillTest; +import org.apache.drill.test.DrillTestBase; import java.io.ByteArrayOutputStream; import java.io.PrintStream; @@ -44,7 +44,7 @@ /** * Test of TracingProxyDriver other than loading of driver classes. */ -public class TracingProxyDriverTest extends DrillTest { +public class TracingProxyDriverTest extends DrillTestBase { private static Driver proxyDriver; private static Connection proxyConnection; diff --git a/pom.xml b/pom.xml index 8d4b318f26c..b172c04a3e4 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,8 @@ 1.7.6 2 1.6.0rc3-drill-r0.3 - + 2.1.16 + .local-${project.version}-execution-hints.log + true + + + + com.carrotsearch.randomizedtesting + junit4-maven-plugin + ${carrotsearch.testframework} + + max + true + + -Xms512m -Xmx3g -Ddrill.exec.http.enabled=false -Ddrill.exec.sys.store.provider.local.write=false -Dorg.apache.drill.exec.server.Drillbit.system_options="org.apache.drill.exec.compile.ClassTransformer.scalar_replacement=on" -XX:MaxPermSize=256M -XX:MaxDirectMemorySize=3072M - -XX:+CMSClassUnloadingEnabled -ea - ${forkCount} - true + -XX:+CMSClassUnloadingEnabled -ea + ./exec/jdbc/src/test/resources/storage-plugins.json - - ${project.build.directory} - + + + + + + + + + + tests + test + + junit4 + + + + + + + + + + + + + **/*Tests.class + **/*Test.class + + + + org.apache.maven.plugins @@ -555,6 +603,25 @@ 0.9.44 test + + com.carrotsearch.randomizedtesting + randomizedtesting-runner + ${carrotsearch.testframework} + test + + + com.carrotsearch.randomizedtesting + junit4-maven-plugin + ${carrotsearch.testframework} + test + + + + commons-logging + commons-logging-api + + +