Merge pull request #1 from agozhiy/resurrection

Resurrection

.gitignore

@@ -21,6 +21,7 @@ git.properties
 framework/resources/Datasources/s3minio/minio/minio_data/.minio.sys
 framework/resources/Datasources/s3minio/minio/minio_data/tstbucket/tmp/ppruning/
 framework/resources/Datasources/s3minio/minio/minio_data/tstbucket/tmp/gitignore
+drill-output/
 
 # Hive stuff
 metastore_db

CONTRIBUTING.md

@@ -85,6 +85,18 @@ A test definition file is a JSON file that defines one or a set of tests within
  }
 </code></pre>
 
+The data described in "datasources" section is prepared in several stages. The stages are defined by "mode" field and executed in the following order:  
+- **rm** - remove data specified in "dest" field
+- **cp** - copy data from "src" in local file system to "dest" in the target fs
+- **mkdir** - make directory specified in "dest"
+- **gen** - execute command specified in "src". In case of a file with .ddl extension it will be send to Drill through JDBC
+- **post_rm** - the same as "rm" but executed later
+- **post_cp** - the same as "cp" but executed later
+- **dfs_cp** - copy data between dfs locations
+- **ddl** - executes specified ddl file through JDBC
+
+Note that local paths are based on DRILL_TEST_DATA_DIR property (by default it is framework/resources), and all dfs paths are based on DRILL_TESTDATA property (by default it is /drill/testdata)  
+
 ### Special handling for Drill version in regex based baseline verification
 
 The Drill version in a regex based baseline file can be substituted with "{DRILL_VERSION}". 

README.md

@@ -36,11 +36,25 @@ bin/build_framework -Pdownload
 
 If you've already downloaded the datasets previously, you can simply skip the download.
 
+If you're going to run tests in docker, you'll need to build the docker image first:
+
+```
+mvn clean install docker:build -DskipTests
+```
+
 ## Execute Tests
 
 In the root directory of your repository, execute the following command to run tests:
 
-`bin/run_tests -s <suites> -g <groups> -t <Timeout> -x <Exclude> -n <Concurrency> -d`
+```
+bin/run_tests -s <suites> -g <groups> -t <Timeout> -x <Exclude> -n <Concurrency> -d`
+```
+
+Alternatively, you can run tests in docker:
+
+```
+mvn docker:start -Dtest.args="-s <suites> -g <groups> -t <Timeout> -x <Exclude> -n <Concurrency> -d"
+```
 
 Example:
  <pre><code> bin/run_tests -s Functional/aggregates,Functional/joins -g functional -x hbase -t 180 -n 20 -d
@@ -57,7 +71,8 @@ Example:
   -n concurrency (optional)
      Here, '20' queries can execute concurrently
   -x exclude dependencies (optional)
-     Here, any 'hbase' test suites within the specified directory are excluded
+     Here, any test suites within the specified directory that have specified dependencies are excluded.
+     In case of "-x all" all tests suites that have at least one dependency are excluded
   -h help (optional)
      Use this option to provide the usage of the command, which includes additional options
 </code></pre>

bin/run_tests

@@ -5,6 +5,7 @@ source conf/drillTestConfig.properties
 cd ${bin}/../
 
 default_debug_string="-Xdebug -Xnoagent -Xrunjdwp:transport=dt_socket,address=50000,server=y,suspend=y"
+JDBC_DRIVER_CP="${DRILL_HOME}/jars/jdbc-driver/drill-jdbc-all-${DRILL_VERSION}.jar"
 
 ARGS=()
 while [[ $# > 0 ]]
@@ -20,6 +21,11 @@ do
     DRILL_TEST_FRAMEWORK_JAVA_OPTS="$DRILL_TEST_FRAMEWORK_JAVA_OPTS $default_debug_string"
     shift
     ;;
+  --jdbcCP)
+    shift
+    JDBC_DRIVER_CP=$1
+    shift
+    ;;
   *)
     ARGS+=("$1")
     shift
@@ -34,7 +40,7 @@ done
 # use the following line when testing apache drill JDBC driver
 if [[ $HADOOP_VERSION == *"mapr"* ]]
 then
-  java $DRILL_TEST_FRAMEWORK_JAVA_OPTS -Xss40m -cp .:conf/:${DRILL_HOME}/jars/jdbc-driver/drill-jdbc-all-${DRILL_VERSION}.jar:framework/target/framework-1.0.0-SNAPSHOT-shaded.jar -Dfs.mapr.bailout.on.library.mismatch=false -Djava.io.tmpdir=/tmp/drill/tests -Djava.security.auth.login.config=/opt/mapr/conf/mapr.login.conf -Dzookeeper.sasl.client=false org.apache.drill.test.framework.TestDriver "${ARGS[@]}"
+  java $DRILL_TEST_FRAMEWORK_JAVA_OPTS -Xss40m -cp .:conf/:${JDBC_DRIVER_CP}:framework/target/framework-1.0.0-SNAPSHOT-shaded.jar -Dfs.mapr.bailout.on.library.mismatch=false -Djava.io.tmpdir=/tmp/drill/tests -Djava.security.auth.login.config=/opt/mapr/conf/mapr.login.conf -Dzookeeper.sasl.client=false org.apache.drill.test.framework.TestDriver "${ARGS[@]}"
 else
-  java $DRILL_TEST_FRAMEWORK_JAVA_OPTS -Xss40m -cp .:conf/:${DRILL_HOME}/jars/jdbc-driver/drill-jdbc-all-${DRILL_VERSION}.jar:framework/target/framework-1.0.0-SNAPSHOT-shaded.jar org.apache.drill.test.framework.TestDriver "${ARGS[@]}"
+  java $DRILL_TEST_FRAMEWORK_JAVA_OPTS -Xss40m -cp .:conf/:${JDBC_DRIVER_CP}:framework/target/framework-1.0.0-SNAPSHOT-shaded.jar org.apache.drill.test.framework.TestDriver "${ARGS[@]}"
 fi

conf/logback.xml

@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
+<configuration>
+
+  <!-- Direct log messages to ./drill.log.timestamp file -->
+  <appender name="file" class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+      <fileNamePattern>logs/drill.log.%d{yyyy-MM-dd-HH}</fileNamePattern>
+    </rollingPolicy>
+    <encoder>
+      <pattern>%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n</pattern>
+    </encoder>
+  </appender>
+
+  <!-- Direct log messages to stdout -->
+  <appender name="Drill" class="ch.qos.logback.core.ConsoleAppender">
+    <Target>System.out</Target>
+    <encoder>
+      <pattern>%m%n</pattern>
+    </encoder>
+  </appender>
+
+  <!-- Direct log messages to stdout -->
+  <appender name="stdout" class="ch.qos.logback.core.ConsoleAppender">
+    <Target>System.out</Target>
+    <encoder>
+      <pattern>%m%n</pattern>
+    </encoder>
+  </appender>
+
+
+  <logger name="org.apache.zookeeper" level="WARN"/>
+
+  <logger name="DrillTestLogger" additivity="false" level="INFO">
+    <appender-ref ref="file"/>
+    <appender-ref ref="Drill"/>
+  </logger>
+
+  <logger name="org.apache.curator" level="WARN"/>
+
+  <!-- Root logger option -->
+  <root level="ERROR">
+    <appender-ref ref="stdout"/>
+  </root>
+
+</configuration>

framework/pom.xml

@@ -7,12 +7,27 @@
   <version>1.0.0-SNAPSHOT</version>
   <properties>
     <testng.version>6.4</testng.version>
-    <log4j.version>1.2.17</log4j.version>
+    <slf4j.version>1.7.26</slf4j.version>
+    <logback.version>1.2.9</logback.version>
     <jackson.version>2.10.0</jackson.version>
-    <drill.version>${env.DRILL_VERSION}</drill.version>
-    <hadoop.version>${env.HADOOP_VERSION}</hadoop.version>
-    <runtimeDepenencies>${env.DRILL_HOME}/conf</runtimeDepenencies>
+    <guava.version>30.1.1-jre</guava.version>
+    <drill.version>2.0.0-SNAPSHOT</drill.version>
+    <hadoop.version>3.2.2</hadoop.version>
+    <drill.test.framework.root>${project.basedir}/..</drill.test.framework.root>
+    <drill.home>${project.basedir}/../../drill</drill.home>
+    <drill.distribution>${drill.home}/distribution/target/apache-drill-${drill.version}/apache-drill-${drill.version}/</drill.distribution>
+    <drill.jdbc.driver.path>jars/jdbc-driver/drill-jdbc-all-${drill.version}.jar</drill.jdbc.driver.path>
+    <drill.java.exec.path>jars/drill-java-exec-${drill.version}.jar</drill.java.exec.path>
+    <m2.home>${env.HOME}/.m2</m2.home>
+    <runtimeDepenencies>${drill.distribution}/conf</runtimeDepenencies>
     <maven.surefire.plugin.version>2.19.1</maven.surefire.plugin.version>
+    <!-- Docker Plugin Properties -->
+    <maven.docker.plugin.version>0.38.1</maven.docker.plugin.version>
+    <docker.base.image>maven:3.8.4-jdk-11</docker.base.image>
+    <docker.automation.root>/root/testAutomation</docker.automation.root>
+    <docker.drill.test.framework.home>${docker.automation.root}/drill-test-framework</docker.drill.test.framework.home>
+    <docker.drill.home>${docker.automation.root}/apache-drill-${drill.version}</docker.drill.home>
+    <test.args>-s Functional -g empty</test.args>
   </properties>
   <dependencies>
     <dependency>
@@ -24,21 +39,46 @@
       <groupId>org.ojai</groupId>
       <artifactId>ojai</artifactId>
       <version>1.1</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-api</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.zookeeper</groupId>
       <artifactId>zookeeper</artifactId>
       <version>3.4.14</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <artifactId>log4j</artifactId>
+          <groupId>log4j</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>${slf4j.version}</version>
     </dependency>
     <dependency>
-      <groupId>log4j</groupId>
-      <artifactId>log4j</artifactId>
-      <version>${log4j.version}</version>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-classic</artifactId>
+      <version>${logback.version}</version>
     </dependency>
     <dependency>
-      <groupId>log4j</groupId>
-      <artifactId>apache-log4j-extras</artifactId>
-      <version>${log4j.version}</version>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-core</artifactId>
+      <version>${logback.version}</version>
     </dependency>
     <dependency>
       <groupId>junit</groupId>
@@ -58,7 +98,12 @@
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
-      <version>16.0.1</version>
+      <version>${guava.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-aws</artifactId>
+      <version>${hadoop.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -73,6 +118,10 @@
           <groupId>org.slf4j</groupId>
           <artifactId>slf4j-log4j12</artifactId>
         </exclusion>
+        <exclusion>
+          <artifactId>log4j</artifactId>
+          <groupId>log4j</groupId>
+        </exclusion>
         <exclusion>
           <groupId>org.codehaus.jackson</groupId>
           <artifactId>jackson-mapper-asl</artifactId>
@@ -92,7 +141,14 @@
       <groupId>org.apache.drill.exec</groupId>
       <artifactId>drill-jdbc-all</artifactId>
       <version>${drill.version}</version>
-      <systemPath>${env.DRILL_HOME}/jars/jdbc-driver/drill-jdbc-all-${drill.version}.jar</systemPath>
+      <systemPath>${drill.distribution}/${drill.jdbc.driver.path}</systemPath>
+      <scope>system</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.drill.exec</groupId>
+      <artifactId>drill-java-exec</artifactId>
+      <version>${drill.version}</version>
+      <systemPath>${drill.distribution}/${drill.java.exec.path}</systemPath>
       <scope>system</scope>
     </dependency>
     <dependency>
@@ -141,7 +197,7 @@
       <id>mapr-releases</id>
       <url>http://repository.mapr.com/maven/</url>
       <releases>
-        <enabled>true</enabled>
+        <enabled>false</enabled>
       </releases>
       <snapshots>
         <enabled>false</enabled>
@@ -150,6 +206,61 @@
   </repositories>
   <build>
     <plugins>
+      <plugin>
+        <groupId>io.fabric8</groupId>
+        <artifactId>docker-maven-plugin</artifactId>
+        <version>${maven.docker.plugin.version}</version>
+        <configuration>
+          <showLogs>true</showLogs>
+          <verbose>true</verbose>
+          <images>
+            <image>
+              <name>drill-test-framework</name>
+              <build>
+                <nocache>false</nocache>
+                <from>${docker.base.image}</from>
+              </build>
+              <run>
+                <containerNamePattern>%n_%t_%i</containerNamePattern>
+                <wait>
+                  <exit>0</exit>
+                  <time>1210000000</time>
+                </wait>
+                <volumes>
+                  <bind>
+                    <volume>${drill.test.framework.root}:${docker.drill.test.framework.home}</volume>
+                    <volume>${drill.distribution}:${docker.drill.home}</volume>
+                    <volume>${m2.home}:/root/.m2</volume>
+                  </bind>
+                </volumes>
+                <workingDir>${docker.drill.test.framework.home}</workingDir>
+                <network>
+                  <mode>host</mode>
+                </network>
+                <cmd>
+                  <shell>/bin/bash bin/run_tests --jdbcCP ${docker.drill.home}/${drill.jdbc.driver.path} ${test.args}</shell>
+                </cmd>
+              </run>
+            </image>
+          </images>
+        </configuration>
+        <!--<executions>
+          <execution>
+            <phase>install</phase>
+            <goals>
+              <goal>build</goal>
+            </goals>
+          </execution>
+          <execution>
+            <id>start</id>
+            <phase>test</phase>
+            <goals>
+              <goal>run</goal>
+              <goal>stop</goal>
+            </goals>
+          </execution>
+        </executions>-->
+      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>

framework/resources/Datasources/ctas/create_tables_complex_parquet.ddl

@@ -1,4 +1,4 @@
-CREATE TABLE `complex.json` AS 
+CREATE TABLE dfs.drillTestDirComplexParquet.`complex.json` AS
   SELECT id, 
          gbyi, 
          gbyt,