add e2e for spark3.3

seatunnel-core/seatunnel-spark-starter/src/main/bin/start-seatunnel-spark-new-connector.sh

@@ -50,6 +50,8 @@ else
     args=$@
 fi
 
+echo "args: ${args}"
+
 CMD=$(java -cp ${APP_JAR} ${APP_MAIN} ${args} | tail -n 1) && EXIT_CODE=$? || EXIT_CODE=$?
 if [ ${EXIT_CODE} -eq 234 ]; then
     # print usage

seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SparkStarter.java

@@ -191,7 +191,7 @@ private List<Path> getPluginsJarDependencies() throws IOException {
             return stream
                     .filter(it -> pluginRootDir.relativize(it).getNameCount() == PLUGIN_LIB_DIR_DEPTH)
                     .filter(it -> it.getParent().endsWith("lib"))
-                    .filter(it -> it.getFileName().endsWith("jar"))
+                    .filter(it -> it.getFileName().toString().endsWith("jar"))
                     .collect(Collectors.toList());
         }
     }

seatunnel-e2e/seatunnel-spark-connector-v2-e2e/pom.xml

@@ -22,26 +22,12 @@
         <version>${revision}</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
-    <packaging>jar</packaging>
+    <packaging>pom</packaging>
+    <modules>
+        <module>seatunnel-spark-connector-v2-e2e-2.4</module>
+        <module>seatunnel-spark-connector-v2-e2e-3.3</module>
+        <module>seatunnel-spark-connector-v2-e2e-common</module>
+    </modules>
 
     <artifactId>seatunnel-spark-connector-v2-e2e</artifactId>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.seatunnel</groupId>
-            <artifactId>seatunnel-core-spark</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.seatunnel</groupId>
-            <artifactId>seatunnel-connectors-v2-dist</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>org.awaitility</groupId>
-            <artifactId>awaitility</artifactId>
-        </dependency>
-    </dependencies>
-
 </project>

seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/pom.xml

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+       http://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>seatunnel-spark-connector-v2-e2e</artifactId>
+        <groupId>org.apache.seatunnel</groupId>
+        <version>${revision}</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>seatunnel-spark-connector-v2-e2e-2.4</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-spark-connector-v2-e2e-common</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.awaitility</groupId>
+            <artifactId>awaitility</artifactId>
+        </dependency>
+    </dependencies>
+</project>

seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.e2e.spark;
+
+import org.apache.seatunnel.e2e.spark.v2.AbstractSparkContainer;
+
+import java.nio.file.Paths;
+
+/**
+ * This class is the base class of SparkEnvironment test. The before method will create a Spark master, and after method will close the Spark master.
+ * You can use {@link SparkContainer#executeSeaTunnelSparkJob} to submit a seatunnel conf and a seatunnel spark job.
+ */
+public abstract class SparkContainer extends AbstractSparkContainer {
+
+    private final String translationJarName = "seatunnel-translation-spark-2.4-dist.jar";
+
+    @Override
+    protected String getTranslationJarPath() {
+        return Paths.get(PROJECT_ROOT_PATH.toString(),
+                "seatunnel-translation", "seatunnel-translation-spark",
+                "seatunnel-translation-spark-2.4-dist", "target",
+                translationJarName).toString();
+    }
+
+    @Override
+    protected String getTranslationJarTargetPath() {
+        return Paths.get(SEATUNNEL_HOME, "plugins", "translation-spark-2.4", "lib",
+                translationJarName).toString();
+    }
+
+    @Override
+    protected String getSparkDockerImage() {
+        return "bitnami/spark:2.4.3";
+    }
+}

seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/pom.xml

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+       http://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>seatunnel-spark-connector-v2-e2e</artifactId>
+        <groupId>org.apache.seatunnel</groupId>
+        <version>${revision}</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>seatunnel-spark-connector-v2-e2e-3.3</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-spark-connector-v2-e2e-common</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.awaitility</groupId>
+            <artifactId>awaitility</artifactId>
+        </dependency>
+    </dependencies>
+</project>

seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.e2e.spark;
+
+import org.apache.seatunnel.e2e.spark.v2.AbstractSparkContainer;
+
+import java.nio.file.Paths;
+
+/**
+ * This class is the base class of SparkEnvironment test. The before method will create a Spark master, and after method will close the Spark master.
+ * You can use {@link SparkContainer#executeSeaTunnelSparkJob} to submit a seatunnel conf and a seatunnel spark job.
+ */
+public abstract class SparkContainer extends AbstractSparkContainer {
+
+    private final String translationJarName = "seatunnel-translation-spark-3.3-dist.jar";
+
+    @Override
+    protected String getTranslationJarPath() {
+        return Paths.get(PROJECT_ROOT_PATH.toString(),
+                "seatunnel-translation", "seatunnel-translation-spark",
+                "seatunnel-translation-spark-3.3-dist", "target",
+                translationJarName).toString();
+    }
+
+    @Override
+    protected String getTranslationJarTargetPath() {
+        return Paths.get(SEATUNNEL_HOME, "plugins", "translation-spark-3.3", "lib",
+                translationJarName).toString();
+    }
+
+    @Override
+    protected String getSparkDockerImage() {
+        return "bitnami/spark:3.3.0";
+    }
+}

seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/src/test/java/org/apache/seatunnel/e2e/spark/fake/FakeSourceToConsoleIT.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.e2e.spark.fake;
+
+import org.apache.seatunnel.e2e.spark.SparkContainer;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.testcontainers.containers.Container;
+
+import java.io.IOException;
+
+/**
+ * This test case is used to verify that the fake source is able to send data to the console.
+ * Make sure the SeaTunnel job can submit successfully on spark engine.
+ */
+public class FakeSourceToConsoleIT extends SparkContainer {
+
+    @Test
+    public void testFakeSourceToConsoleSine() throws IOException, InterruptedException {
+        Container.ExecResult execResult = executeSeaTunnelSparkJob("/fake/fakesource_to_console.conf");
+        Assertions.assertEquals(0, execResult.getExitCode());
+    }
+}

seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/src/test/resources/fake/fakesource_to_console.conf

@@ -0,0 +1,81 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+######
+###### This config file is a demonstration of streaming processing in seatunnel config
+######
+
+env {
+  # You can set spark configuration here
+  # see available properties defined by spark: https://spark.apache.org/docs/latest/configuration.html#available-properties
+  job.mode = "BATCH"
+  spark.app.name = "SeaTunnel"
+  spark.executor.instances = 2
+  spark.executor.cores = 1
+  spark.executor.memory = "1g"
+  spark.master = local
+}
+
+source {
+  # This is a example input plugin **only for test and demonstrate the feature input plugin**
+  FakeSource {
+    result_table_name = "my_dataset"
+    schema = {
+      fields {
+        name = "string"
+        age = "int"
+      }
+    }
+  }
+
+  # You can also use other input plugins, such as hdfs
+  # hdfs {
+  #   result_table_name = "accesslog"
+  #   path = "hdfs://hadoop-cluster-01/nginx/accesslog"
+  #   format = "json"
+  # }
+
+  # If you would like to get more information about how to configure seatunnel and see full list of source plugins,
+  # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource
+}
+
+transform {
+  # split data by specific delimiter
+
+  # you can also use other transform plugins, such as sql
+  # sql {
+  #   sql = "select * from accesslog where request_time > 1000"
+  # }
+
+  # If you would like to get more information about how to configure seatunnel and see full list of transform plugins,
+  # please go to https://seatunnel.apache.org/docs/spark/configuration/transform-plugins/Split
+}
+
+sink {
+  # choose stdout output plugin to output data to console
+  Console {
+    saveMode = "append"
+  }
+
+  # you can also you other output plugins, such as sql
+  # hdfs {
+  #   path = "hdfs://hadoop-cluster-01/nginx/accesslog_processed"
+  #   save_mode = "append"
+  # }
+
+  # If you would like to get more information about how to configure seatunnel and see full list of output plugins,
+  # please go to https://seatunnel.apache.org/docs/spark/configuration/sink-plugins/Console
+}

seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/src/test/resources/log4j.properties

@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Set everything to be logged to the console
+log4j.rootCategory=INFO, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n