KSQL Testing Tool, V1 (#2802)

* Temporary Commit! * Phase 1, refactoring * More tests and some minor changes. * Temporary Commit * Have to commit to merge! * Still working on it * V1-temp * V1 working. * More tests * Minor change * Added docs. * Fix a minor conflict issue. * Minor changes. * Minor changes. * Some refactoring * Fixed a minor regression. * Minor change. * Fixed some FindBug issues. * Fixed more Avro issues! * Applied feedback from Victoria * Fixed the issue with the window keys * more fixes. * More changes * More improvements. * More improvements
confluentinc · May 15, 2019 · 9d55cf6 · 9d55cf6
1 parent 1a36613
commit 9d55cf6
Show file tree

Hide file tree

Showing 29 changed files with 2,208 additions and 29 deletions.
diff --git a/bin/ksql-run-class b/bin/ksql-run-class
@@ -25,7 +25,7 @@ fi
 : "${JAVA_HOME:=""}"
 
 # Development jars. `mvn package` should collect all the required dependency jars here
-for project in ksql-engine ksql-examples ksql-rest-app ksql-cli; do
+for project in ksql-engine ksql-examples ksql-rest-app ksql-cli ksql-functional-tests; do
     for dir in "$base_dir/$project/target/$project"-*-development; do
       KSQL_DIR="$dir/share/java/$project"
       if [ -d "$KSQL_DIR" ]; then
@@ -35,7 +35,7 @@ for project in ksql-engine ksql-examples ksql-rest-app ksql-cli; do
 done
 
 # Production jars - each one is prepended so they will appear in reverse order.  KSQL jars take precedence over other stuff passed in via CLASSPATH env var
-for library in "confluent-common" "ksql-examples" "rest-utils" "ksql-engine" "ksql-rest-app" "ksql-cli" "ksql" "monitoring-interceptors"; do
+for library in "confluent-common" "ksql-examples" "rest-utils" "ksql-engine" "ksql-rest-app" "ksql-cli" "ksql-functional-tests" "ksql" "monitoring-interceptors"; do
   DIR="$base_dir/share/java/$library"
   if [ -d "$DIR" ]; then
     KSQL_CLASSPATH="$DIR/*:$KSQL_CLASSPATH"

diff --git a/bin/ksql-testing-tool b/bin/ksql-testing-tool
@@ -0,0 +1,17 @@
+#!/bin/bash
+# (Copyright) [2019 - 2019] Confluent, Inc.
+
+ #
+# Use shellcheck to lint this file
+#
+set -ue
+
+ base_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )
+: "${KSQL_CONFIG_DIR:="$base_dir/config"}"
+
+ : "${KSQL_LOG4J_OPTS:=""}"
+if [ -z "$KSQL_LOG4J_OPTS" ] && [ -e "$KSQL_CONFIG_DIR/log4j-silent.properties" ]; then
+  export KSQL_LOG4J_OPTS="-Dlog4j.configuration=file:$KSQL_CONFIG_DIR/log4j-silent.properties"
+fi
+
+ exec "$base_dir"/bin/ksql-run-class io.confluent.ksql.test.tools.KsqlTestingTool "$@"
diff --git a/docs/index.rst b/docs/index.rst
@@ -92,6 +92,7 @@ KSQL Documentation
     developer-guide/index
     operations
     capacity-planning
+    ksql-testing-tool
     tutorials/index
     faq
     changelog

diff --git a/docs/testing-tool.rst b/docs/testing-tool.rst
@@ -0,0 +1,91 @@
+.. _ksql-testing-tool:
+
+KSQL Testing Tool
+###########################
+
+Use the KSQL testing tool to test a set of KSQL statements. The KSQL testing tool
+is a command line utility that enables testing KSQL statements without requiring any infrastructure, like |ak-tm| and KSQL clusters.
+The KSQL testing tool is a great way to design your KSQL pipeline and ensure the expected results are generated.
+You can collaborate on designing your KSQL statements by sharing the test files.
+You provide a JSON file that describes a set of KSQL statements, along with the input data and expected output data.
+Run the testing tool from a terminal and pass the test file as a parameter.
+
+.. code:: bash
+
+    ksql-testing-tool /path/to/the/test/file.json
+
+
+Test File Structure
+*******************
+The test file is a JSON file containing the KSQL statements, input data, desired configurations, and the expected results or expected errors.
+The following is a sample test file:
+
+.. code:: json
+    {
+      "comments": [
+        "Add a description of the functionality that this file tests"
+      ],
+      "tests": [
+        {
+          "name": "my first positive test",
+          "description": "an example positive test where the output is verified",
+          "statements": [
+            "CREATE STREAM intput (ID bigint) WITH (kafka_topic='input_topic', value_format='JSON');",
+            "CREATE STREAM output AS SELECT id FROM intput WHERE id < 10;"
+          ],
+          "inputs": [
+            {"topic": "input_topic", "key": 0, "value": {"id": 8}, "timestamp": 0},
+            {"topic": "input_topic", "key": 0, "value": {"id": 10}, "timestamp": 10000},
+            {"topic": "input_topic", "key": 1, "value": {"id": 9}, "timestamp": 30000},
+            {"topic": "input_topic", "key": 1, "value": {"id": 11}, "timestamp": 40000}
+          ],
+          "outputs": [
+            {"topic": "OUTPUT", "key": 0, "value": {"ID": 8}, "timestamp": 0},
+            {"topic": "OUTPUT", "key": 1, "value": {"ID": 9}, "timestamp": 30000}
+          ]
+        },
+        {
+          "name": "my first negative test",
+          "description": "an example negative test where the statement will fail to parse",
+          "statements": [
+            "CREATE STREAM TEST WITH (kafka_topic='test_topic', value_format='DELIMITED');"
+          ],
+          "expectedException": {
+            "type": "io.confluent.ksql.util.KsqlException",
+            "message": "The statement does not define any columns."
+          }
+        }
+      ]
+    }
+
+
+You can have multiple tests in one test file. In addition to name, description, and statements, each test includes
+input topics and their data along with the expected output topics and their data.
+The test file format is the same as the files KSQL code uses for integration tests. For more details on the
+structure of the test file and all possible settings, see the `README.md <https://github.com/confluentinc/ksql/tree/master/ksql-functional-tests>` in the KSQL repository.
+
+Running Tests
+*************
+
+Assume we run the previous test file, which is stored in test.json in the home directory.
+The following command shows how to run the test from the terminal:
+
+.. code:: bash
+
+    ksql-testing-tool ~/test.json
+
+
+Your output should resemble:
+
+.. code:: bash
+
+     >>> Running test: ksql-test - my first positive test
+    	 >>> Test ksql-test - my first positive test passed!
+     >>> Running test: ksql-test - my first negative test
+    	 >>> Test ksql-test - my first negative test passed!
+    All tests passed!
+
+
+For each test case, the testing tool first creates and populates the input topics in its internal simulated Kafka cluster.
+It compiles the KSQL statements, runs them, and compares the generated results with the expected results. If the expected results are generated, the test passes, otherwise it fails.
+The status of each test is printed out into the terminal.
diff --git a/ksql-cli/src/test/java/io/confluent/ksql/cli/SslFunctionalTest.java b/ksql-cli/src/test/java/io/confluent/ksql/cli/SslFunctionalTest.java
@@ -252,4 +252,4 @@ private void closeSilently() {
       }
     }
   }
-}
+}
diff --git a/ksql-functional-tests/pom.xml b/ksql-functional-tests/pom.xml
@@ -11,16 +11,18 @@
 
   <artifactId>ksql-functional-tests</artifactId>
 
+  <properties>
+    <air.main.basedir>${project.parent.basedir}</air.main.basedir>
+    <main-class>io.confluent.ksql.test.tools.KsqlTestingTool</main-class>
+  </properties>
 
   <dependencies>
 
-
     <dependency>
       <groupId>io.confluent.ksql</groupId>
       <artifactId>ksql-engine</artifactId>
     </dependency>
 
-
     <dependency>
       <groupId>io.confluent.ksql</groupId>
       <artifactId>ksql-engine</artifactId>
@@ -42,12 +44,25 @@
       <type>test-jar</type>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.kafka</groupId>
+      <artifactId>kafka-clients</artifactId>
+      <version>${kafka.version}</version>
+      <classifier>test</classifier>
+      <scope>compile</scope>
+    </dependency>
+
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
       <scope>compile</scope>
     </dependency>
 
+    <dependency>
+      <groupId>com.github.rvesse</groupId>
+      <artifactId>airline</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>org.hamcrest</groupId>
       <artifactId>hamcrest-all</artifactId>
@@ -75,4 +90,35 @@
     </dependency>
 
   </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <configuration>
+          <descriptors>
+            <descriptor>src/assembly/development.xml</descriptor>
+            <descriptor>src/assembly/package.xml</descriptor>
+            <descriptor>src/assembly/standalone.xml</descriptor>
+          </descriptors>
+          <archive>
+            <manifest>
+              <mainClass>${main-class}</mainClass>
+            </manifest>
+          </archive>
+          <attach>false</attach>
+        </configuration>
+        <executions>
+          <execution>
+            <id>make-assembly</id>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
 </project>
diff --git a/ksql-functional-tests/src/assembly/development.xml b/ksql-functional-tests/src/assembly/development.xml
@@ -0,0 +1,66 @@
+<!--
+  ~ Copyright 2019 Confluent Inc.
+  ~
+  ~ Licensed under the Confluent Community License (the "License"); you may not use
+  ~ this file except in compliance with the License.  You may obtain a copy of the
+  ~ License at
+  ~
+  ~ http://www.confluent.io/confluent-community-license
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~ WARRANTIES OF ANY KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations under the License.
+  -->
+
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2
+          http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+    <!-- Assembles all dependencies in target/ directory so scripts can easily run in a development
+         environment -->
+    <id>development</id>
+    <formats>
+        <format>dir</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+    <fileSets>
+        <fileSet>
+            <directory>${project.parent.basedir}</directory>
+            <outputDirectory>share/doc/ksql-functional-tests/</outputDirectory>
+            <includes>
+                <include>README*</include>
+                <include>COPYRIGHT*</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>${project.parent.basedir}</directory>
+            <outputDirectory></outputDirectory>
+            <includes>
+                <include>bin/*</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>${project.parent.basedir}/config</directory>
+            <outputDirectory>etc/ksql-functional-tests</outputDirectory>
+            <includes>
+                <include>*</include>
+            </includes>
+        </fileSet>
+    </fileSets>
+    <dependencySets>
+        <dependencySet>
+            <outputDirectory>share/java/ksql-functional-tests</outputDirectory>
+            <useProjectArtifact>true</useProjectArtifact>
+            <!-- Warning: Enabling useTransitiveFiltering is potentially unsafe,
+                 cf. https://github.com/miguno/maven-assembly-transitive-deps-woes
+                 and https://issues.apache.org/jira/browse/MASSEMBLY-504.  If
+                 dependencies happen to be missing in the assembly you may want to
+                 disable transitive filtering. -->
+            <useTransitiveFiltering>true</useTransitiveFiltering>
+            <excludes>
+                <exclude>org.slf4j:slf4j-log4j12</exclude> <!-- Already included by rest-app-->
+            </excludes>
+        </dependencySet>
+    </dependencySets>
+</assembly>
diff --git a/ksql-functional-tests/src/assembly/package.xml b/ksql-functional-tests/src/assembly/package.xml
@@ -0,0 +1,70 @@
+<!--
+  ~ Copyright 2019 Confluent Inc.
+  ~
+  ~ Licensed under the Confluent Community License (the "License"); you may not use
+  ~ this file except in compliance with the License.  You may obtain a copy of the
+  ~ License at
+  ~
+  ~ http://www.confluent.io/confluent-community-license
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~ WARRANTIES OF ANY KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations under the License.
+  -->
+
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2
+          http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+    <!-- Assembles a package that can run the ksql cli given dependency packages. This is used to
+         construct packages OS packages where underlying libraries, like rest-utils, can be shared
+         -->
+    <id>package</id>
+    <formats>
+        <format>dir</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+    <fileSets>
+        <fileSet>
+            <directory>${project.parent.basedir}</directory>
+            <outputDirectory>share/doc/ksql-functional-tests/</outputDirectory>
+            <includes>
+                <include>version.txt</include>
+                <include>COPYRIGHT*</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>${project.parent.basedir}</directory>
+            <outputDirectory></outputDirectory>
+            <includes>
+                <include>bin/*</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>${project.parent.basedir}/config</directory>
+            <outputDirectory>etc/ksql-functional-tests</outputDirectory>
+            <includes>
+                <include>*</include>
+            </includes>
+        </fileSet>
+    </fileSets>
+    <dependencySets>
+        <dependencySet>
+            <outputDirectory>share/java/ksql-functional-tests</outputDirectory>
+            <useProjectArtifact>true</useProjectArtifact>
+            <!-- Warning: Enabling useTransitiveFiltering is potentially unsafe,
+                 cf. https://github.com/miguno/maven-assembly-transitive-deps-woes
+                 and https://issues.apache.org/jira/browse/MASSEMBLY-504.  If
+                 dependencies happen to be missing in the assembly you may want to
+                 disable transitive filtering. -->
+            <useTransitiveFiltering>true</useTransitiveFiltering>
+            <excludes>
+                <exclude>io.confluent.ksql:ksql-functional-tests</exclude>
+                <exclude>io.confluent:rest-utils</exclude>
+                <exclude>io.confluent:common-*</exclude>
+                <exclude>com.google.guava:guava</exclude>
+            </excludes>
+        </dependencySet>
+    </dependencySets>
+</assembly>