Skip to content
Permalink
Browse files
Merge pull request #1 from agozhiy/resurrection
Resurrection
  • Loading branch information
jnturton committed May 10, 2022
2 parents 9fffe6a + 3261cf5 commit 85fd1ebd153db34638fcb0a9b49980dab7e7c22b
Showing 135 changed files with 2,908 additions and 1,558 deletions.
@@ -21,6 +21,7 @@ git.properties
framework/resources/Datasources/s3minio/minio/minio_data/.minio.sys
framework/resources/Datasources/s3minio/minio/minio_data/tstbucket/tmp/ppruning/
framework/resources/Datasources/s3minio/minio/minio_data/tstbucket/tmp/gitignore
drill-output/

# Hive stuff
metastore_db
@@ -85,6 +85,18 @@ A test definition file is a JSON file that defines one or a set of tests within
}
</code></pre>

The data described in "datasources" section is prepared in several stages. The stages are defined by "mode" field and executed in the following order:
- **rm** - remove data specified in "dest" field
- **cp** - copy data from "src" in local file system to "dest" in the target fs
- **mkdir** - make directory specified in "dest"
- **gen** - execute command specified in "src". In case of a file with .ddl extension it will be send to Drill through JDBC
- **post_rm** - the same as "rm" but executed later
- **post_cp** - the same as "cp" but executed later
- **dfs_cp** - copy data between dfs locations
- **ddl** - executes specified ddl file through JDBC

Note that local paths are based on DRILL_TEST_DATA_DIR property (by default it is framework/resources), and all dfs paths are based on DRILL_TESTDATA property (by default it is /drill/testdata)

### Special handling for Drill version in regex based baseline verification

The Drill version in a regex based baseline file can be substituted with "{DRILL_VERSION}".
@@ -36,11 +36,25 @@ bin/build_framework -Pdownload

If you've already downloaded the datasets previously, you can simply skip the download.

If you're going to run tests in docker, you'll need to build the docker image first:

```
mvn clean install docker:build -DskipTests
```

## Execute Tests

In the root directory of your repository, execute the following command to run tests:

`bin/run_tests -s <suites> -g <groups> -t <Timeout> -x <Exclude> -n <Concurrency> -d`
```
bin/run_tests -s <suites> -g <groups> -t <Timeout> -x <Exclude> -n <Concurrency> -d`
```

Alternatively, you can run tests in docker:

```
mvn docker:start -Dtest.args="-s <suites> -g <groups> -t <Timeout> -x <Exclude> -n <Concurrency> -d"
```

Example:
<pre><code> bin/run_tests -s Functional/aggregates,Functional/joins -g functional -x hbase -t 180 -n 20 -d
@@ -57,7 +71,8 @@ Example:
-n concurrency (optional)
Here, '20' queries can execute concurrently
-x exclude dependencies (optional)
Here, any 'hbase' test suites within the specified directory are excluded
Here, any test suites within the specified directory that have specified dependencies are excluded.
In case of "-x all" all tests suites that have at least one dependency are excluded
-h help (optional)
Use this option to provide the usage of the command, which includes additional options
</code></pre>
@@ -5,6 +5,7 @@ source conf/drillTestConfig.properties
cd ${bin}/../

default_debug_string="-Xdebug -Xnoagent -Xrunjdwp:transport=dt_socket,address=50000,server=y,suspend=y"
JDBC_DRIVER_CP="${DRILL_HOME}/jars/jdbc-driver/drill-jdbc-all-${DRILL_VERSION}.jar"

ARGS=()
while [[ $# > 0 ]]
@@ -20,6 +21,11 @@ do
DRILL_TEST_FRAMEWORK_JAVA_OPTS="$DRILL_TEST_FRAMEWORK_JAVA_OPTS $default_debug_string"
shift
;;
--jdbcCP)
shift
JDBC_DRIVER_CP=$1
shift
;;
*)
ARGS+=("$1")
shift
@@ -34,7 +40,7 @@ done
# use the following line when testing apache drill JDBC driver
if [[ $HADOOP_VERSION == *"mapr"* ]]
then
java $DRILL_TEST_FRAMEWORK_JAVA_OPTS -Xss40m -cp .:conf/:${DRILL_HOME}/jars/jdbc-driver/drill-jdbc-all-${DRILL_VERSION}.jar:framework/target/framework-1.0.0-SNAPSHOT-shaded.jar -Dfs.mapr.bailout.on.library.mismatch=false -Djava.io.tmpdir=/tmp/drill/tests -Djava.security.auth.login.config=/opt/mapr/conf/mapr.login.conf -Dzookeeper.sasl.client=false org.apache.drill.test.framework.TestDriver "${ARGS[@]}"
java $DRILL_TEST_FRAMEWORK_JAVA_OPTS -Xss40m -cp .:conf/:${JDBC_DRIVER_CP}:framework/target/framework-1.0.0-SNAPSHOT-shaded.jar -Dfs.mapr.bailout.on.library.mismatch=false -Djava.io.tmpdir=/tmp/drill/tests -Djava.security.auth.login.config=/opt/mapr/conf/mapr.login.conf -Dzookeeper.sasl.client=false org.apache.drill.test.framework.TestDriver "${ARGS[@]}"
else
java $DRILL_TEST_FRAMEWORK_JAVA_OPTS -Xss40m -cp .:conf/:${DRILL_HOME}/jars/jdbc-driver/drill-jdbc-all-${DRILL_VERSION}.jar:framework/target/framework-1.0.0-SNAPSHOT-shaded.jar org.apache.drill.test.framework.TestDriver "${ARGS[@]}"
java $DRILL_TEST_FRAMEWORK_JAVA_OPTS -Xss40m -cp .:conf/:${JDBC_DRIVER_CP}:framework/target/framework-1.0.0-SNAPSHOT-shaded.jar org.apache.drill.test.framework.TestDriver "${ARGS[@]}"
fi
@@ -0,0 +1,64 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration>

<!-- Direct log messages to ./drill.log.timestamp file -->
<appender name="file" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>logs/drill.log.%d{yyyy-MM-dd-HH}</fileNamePattern>
</rollingPolicy>
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n</pattern>
</encoder>
</appender>

<!-- Direct log messages to stdout -->
<appender name="Drill" class="ch.qos.logback.core.ConsoleAppender">
<Target>System.out</Target>
<encoder>
<pattern>%m%n</pattern>
</encoder>
</appender>

<!-- Direct log messages to stdout -->
<appender name="stdout" class="ch.qos.logback.core.ConsoleAppender">
<Target>System.out</Target>
<encoder>
<pattern>%m%n</pattern>
</encoder>
</appender>


<logger name="org.apache.zookeeper" level="WARN"/>

<logger name="DrillTestLogger" additivity="false" level="INFO">
<appender-ref ref="file"/>
<appender-ref ref="Drill"/>
</logger>

<logger name="org.apache.curator" level="WARN"/>

<!-- Root logger option -->
<root level="ERROR">
<appender-ref ref="stdout"/>
</root>

</configuration>
@@ -7,12 +7,27 @@
<version>1.0.0-SNAPSHOT</version>
<properties>
<testng.version>6.4</testng.version>
<log4j.version>1.2.17</log4j.version>
<slf4j.version>1.7.26</slf4j.version>
<logback.version>1.2.9</logback.version>
<jackson.version>2.10.0</jackson.version>
<drill.version>${env.DRILL_VERSION}</drill.version>
<hadoop.version>${env.HADOOP_VERSION}</hadoop.version>
<runtimeDepenencies>${env.DRILL_HOME}/conf</runtimeDepenencies>
<guava.version>30.1.1-jre</guava.version>
<drill.version>2.0.0-SNAPSHOT</drill.version>
<hadoop.version>3.2.2</hadoop.version>
<drill.test.framework.root>${project.basedir}/..</drill.test.framework.root>
<drill.home>${project.basedir}/../../drill</drill.home>
<drill.distribution>${drill.home}/distribution/target/apache-drill-${drill.version}/apache-drill-${drill.version}/</drill.distribution>
<drill.jdbc.driver.path>jars/jdbc-driver/drill-jdbc-all-${drill.version}.jar</drill.jdbc.driver.path>
<drill.java.exec.path>jars/drill-java-exec-${drill.version}.jar</drill.java.exec.path>
<m2.home>${env.HOME}/.m2</m2.home>
<runtimeDepenencies>${drill.distribution}/conf</runtimeDepenencies>
<maven.surefire.plugin.version>2.19.1</maven.surefire.plugin.version>
<!-- Docker Plugin Properties -->
<maven.docker.plugin.version>0.38.1</maven.docker.plugin.version>
<docker.base.image>maven:3.8.4-jdk-11</docker.base.image>
<docker.automation.root>/root/testAutomation</docker.automation.root>
<docker.drill.test.framework.home>${docker.automation.root}/drill-test-framework</docker.drill.test.framework.home>
<docker.drill.home>${docker.automation.root}/apache-drill-${drill.version}</docker.drill.home>
<test.args>-s Functional -g empty</test.args>
</properties>
<dependencies>
<dependency>
@@ -24,21 +39,46 @@
<groupId>org.ojai</groupId>
<artifactId>ojai</artifactId>
<version>1.1</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.14</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>${log4j.version}</version>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>${logback.version}</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>apache-log4j-extras</artifactId>
<version>${log4j.version}</version>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
<version>${logback.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
@@ -58,7 +98,12 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>16.0.1</version>
<version>${guava.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -73,6 +118,10 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
@@ -92,7 +141,14 @@
<groupId>org.apache.drill.exec</groupId>
<artifactId>drill-jdbc-all</artifactId>
<version>${drill.version}</version>
<systemPath>${env.DRILL_HOME}/jars/jdbc-driver/drill-jdbc-all-${drill.version}.jar</systemPath>
<systemPath>${drill.distribution}/${drill.jdbc.driver.path}</systemPath>
<scope>system</scope>
</dependency>
<dependency>
<groupId>org.apache.drill.exec</groupId>
<artifactId>drill-java-exec</artifactId>
<version>${drill.version}</version>
<systemPath>${drill.distribution}/${drill.java.exec.path}</systemPath>
<scope>system</scope>
</dependency>
<dependency>
@@ -141,7 +197,7 @@
<id>mapr-releases</id>
<url>http://repository.mapr.com/maven/</url>
<releases>
<enabled>true</enabled>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
@@ -150,6 +206,61 @@
</repositories>
<build>
<plugins>
<plugin>
<groupId>io.fabric8</groupId>
<artifactId>docker-maven-plugin</artifactId>
<version>${maven.docker.plugin.version}</version>
<configuration>
<showLogs>true</showLogs>
<verbose>true</verbose>
<images>
<image>
<name>drill-test-framework</name>
<build>
<nocache>false</nocache>
<from>${docker.base.image}</from>
</build>
<run>
<containerNamePattern>%n_%t_%i</containerNamePattern>
<wait>
<exit>0</exit>
<time>1210000000</time>
</wait>
<volumes>
<bind>
<volume>${drill.test.framework.root}:${docker.drill.test.framework.home}</volume>
<volume>${drill.distribution}:${docker.drill.home}</volume>
<volume>${m2.home}:/root/.m2</volume>
</bind>
</volumes>
<workingDir>${docker.drill.test.framework.home}</workingDir>
<network>
<mode>host</mode>
</network>
<cmd>
<shell>/bin/bash bin/run_tests --jdbcCP ${docker.drill.home}/${drill.jdbc.driver.path} ${test.args}</shell>
</cmd>
</run>
</image>
</images>
</configuration>
<!--<executions>
<execution>
<phase>install</phase>
<goals>
<goal>build</goal>
</goals>
</execution>
<execution>
<id>start</id>
<phase>test</phase>
<goals>
<goal>run</goal>
<goal>stop</goal>
</goals>
</execution>
</executions>-->
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
@@ -1,4 +1,4 @@
CREATE TABLE `complex.json` AS
CREATE TABLE dfs.drillTestDirComplexParquet.`complex.json` AS
SELECT id,
gbyi,
gbyt,

0 comments on commit 85fd1eb

Please sign in to comment.