diff --git a/seatunnel-core/seatunnel-spark-starter/src/main/bin/start-seatunnel-spark-new-connector.sh b/seatunnel-core/seatunnel-spark-starter/src/main/bin/start-seatunnel-spark-new-connector.sh index a54dc55ee33..965b7f8dfc4 100755 --- a/seatunnel-core/seatunnel-spark-starter/src/main/bin/start-seatunnel-spark-new-connector.sh +++ b/seatunnel-core/seatunnel-spark-starter/src/main/bin/start-seatunnel-spark-new-connector.sh @@ -50,6 +50,8 @@ else args=$@ fi +echo "args: ${args}" + CMD=$(java -cp ${APP_JAR} ${APP_MAIN} ${args} | tail -n 1) && EXIT_CODE=$? || EXIT_CODE=$? if [ ${EXIT_CODE} -eq 234 ]; then # print usage diff --git a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SparkStarter.java b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SparkStarter.java index 19479d0e38d..c87e7478e0f 100644 --- a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SparkStarter.java +++ b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SparkStarter.java @@ -191,7 +191,7 @@ private List getPluginsJarDependencies() throws IOException { return stream .filter(it -> pluginRootDir.relativize(it).getNameCount() == PLUGIN_LIB_DIR_DEPTH) .filter(it -> it.getParent().endsWith("lib")) - .filter(it -> it.getFileName().endsWith("jar")) + .filter(it -> it.getFileName().toString().endsWith("jar")) .collect(Collectors.toList()); } } diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/pom.xml b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/pom.xml index 4da27bcf05a..d3649dbc992 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/pom.xml @@ -22,26 +22,12 @@ ${revision} 4.0.0 - jar + pom + + seatunnel-spark-connector-v2-e2e-2.4 + seatunnel-spark-connector-v2-e2e-3.3 + seatunnel-spark-connector-v2-e2e-common + seatunnel-spark-connector-v2-e2e - - - - org.apache.seatunnel - seatunnel-core-spark - ${project.version} - - - org.apache.seatunnel - seatunnel-connectors-v2-dist - ${project.version} - - - - org.awaitility - awaitility - - - \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/pom.xml b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/pom.xml new file mode 100644 index 00000000000..2f2b62a113b --- /dev/null +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/pom.xml @@ -0,0 +1,41 @@ + + + + + seatunnel-spark-connector-v2-e2e + org.apache.seatunnel + ${revision} + + 4.0.0 + + seatunnel-spark-connector-v2-e2e-2.4 + + + + org.apache.seatunnel + seatunnel-spark-connector-v2-e2e-common + ${project.version} + test-jar + test + + + org.awaitility + awaitility + + + \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java new file mode 100644 index 00000000000..39996462690 --- /dev/null +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.e2e.spark; + +import org.apache.seatunnel.e2e.spark.v2.AbstractSparkContainer; + +import java.nio.file.Paths; + +/** + * This class is the base class of SparkEnvironment test. The before method will create a Spark master, and after method will close the Spark master. + * You can use {@link SparkContainer#executeSeaTunnelSparkJob} to submit a seatunnel conf and a seatunnel spark job. + */ +public abstract class SparkContainer extends AbstractSparkContainer { + + private final String translationJarName = "seatunnel-translation-spark-2.4-dist.jar"; + + @Override + protected String getTranslationJarPath() { + return Paths.get(PROJECT_ROOT_PATH.toString(), + "seatunnel-translation", "seatunnel-translation-spark", + "seatunnel-translation-spark-2.4-dist", "target", + translationJarName).toString(); + } + + @Override + protected String getTranslationJarTargetPath() { + return Paths.get(SEATUNNEL_HOME, "plugins", "translation-spark-2.4", "lib", + translationJarName).toString(); + } + + @Override + protected String getSparkDockerImage() { + return "bitnami/spark:2.4.3"; + } +} diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/fake/FakeSourceToConsoleIT.java b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/java/org/apache/seatunnel/e2e/spark/v2/fake/FakeSourceToConsoleIT.java similarity index 100% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/fake/FakeSourceToConsoleIT.java rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/java/org/apache/seatunnel/e2e/spark/v2/fake/FakeSourceToConsoleIT.java diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/file/FakeSourceToFileIT.java b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/java/org/apache/seatunnel/e2e/spark/v2/file/FakeSourceToFileIT.java similarity index 100% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/file/FakeSourceToFileIT.java rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/java/org/apache/seatunnel/e2e/spark/v2/file/FakeSourceToFileIT.java diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/iotdb/FakeSourceToIoTDBIT.java b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/java/org/apache/seatunnel/e2e/spark/v2/iotdb/FakeSourceToIoTDBIT.java similarity index 100% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/iotdb/FakeSourceToIoTDBIT.java rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/java/org/apache/seatunnel/e2e/spark/v2/iotdb/FakeSourceToIoTDBIT.java diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/jdbc/JdbcGreenplumIT.java b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/java/org/apache/seatunnel/e2e/spark/v2/jdbc/JdbcGreenplumIT.java similarity index 100% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/jdbc/JdbcGreenplumIT.java rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/java/org/apache/seatunnel/e2e/spark/v2/jdbc/JdbcGreenplumIT.java diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/jdbc/JdbcPhoenixIT.java b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/java/org/apache/seatunnel/e2e/spark/v2/jdbc/JdbcPhoenixIT.java similarity index 100% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/jdbc/JdbcPhoenixIT.java rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/java/org/apache/seatunnel/e2e/spark/v2/jdbc/JdbcPhoenixIT.java diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/fake/fakesource_to_console.conf similarity index 100% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/fake/fakesource_to_console.conf diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/file/fakesource_to_hdfs_json.conf similarity index 100% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/file/fakesource_to_hdfs_json.conf diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/file/fakesource_to_hdfs_parquet.conf similarity index 100% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/file/fakesource_to_hdfs_parquet.conf diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/file/fakesource_to_hdfs_text.conf similarity index 100% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/file/fakesource_to_hdfs_text.conf diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/file/fakesource_to_local_json.conf similarity index 100% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/file/fakesource_to_local_json.conf diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/file/fakesource_to_local_parquet.conf similarity index 100% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/file/fakesource_to_local_parquet.conf diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/file/fakesource_to_local_text.conf similarity index 100% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/file/fakesource_to_local_text.conf diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/iotdb/fakesource_to_iotdb.conf similarity index 100% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/iotdb/fakesource_to_iotdb.conf diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/jdbc/jdbc_greenplum_source_and_sink.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/jdbc/jdbc_greenplum_source_and_sink.conf similarity index 100% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/jdbc/jdbc_greenplum_source_and_sink.conf rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/jdbc/jdbc_greenplum_source_and_sink.conf diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/jdbc/jdbc_phoenix_source_and_sink.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/jdbc/jdbc_phoenix_source_and_sink.conf similarity index 100% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/jdbc/jdbc_phoenix_source_and_sink.conf rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/jdbc/jdbc_phoenix_source_and_sink.conf diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/log4j.properties b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/log4j.properties similarity index 100% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/log4j.properties rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-2.4/src/test/resources/log4j.properties diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/pom.xml b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/pom.xml new file mode 100644 index 00000000000..d88a7560e43 --- /dev/null +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/pom.xml @@ -0,0 +1,41 @@ + + + + + seatunnel-spark-connector-v2-e2e + org.apache.seatunnel + ${revision} + + 4.0.0 + + seatunnel-spark-connector-v2-e2e-3.3 + + + + org.apache.seatunnel + seatunnel-spark-connector-v2-e2e-common + ${project.version} + test-jar + test + + + org.awaitility + awaitility + + + \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java new file mode 100644 index 00000000000..de4c734910f --- /dev/null +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.e2e.spark; + +import org.apache.seatunnel.e2e.spark.v2.AbstractSparkContainer; + +import java.nio.file.Paths; + +/** + * This class is the base class of SparkEnvironment test. The before method will create a Spark master, and after method will close the Spark master. + * You can use {@link SparkContainer#executeSeaTunnelSparkJob} to submit a seatunnel conf and a seatunnel spark job. + */ +public abstract class SparkContainer extends AbstractSparkContainer { + + private final String translationJarName = "seatunnel-translation-spark-3.3-dist.jar"; + + @Override + protected String getTranslationJarPath() { + return Paths.get(PROJECT_ROOT_PATH.toString(), + "seatunnel-translation", "seatunnel-translation-spark", + "seatunnel-translation-spark-3.3-dist", "target", + translationJarName).toString(); + } + + @Override + protected String getTranslationJarTargetPath() { + return Paths.get(SEATUNNEL_HOME, "plugins", "translation-spark-3.3", "lib", + translationJarName).toString(); + } + + @Override + protected String getSparkDockerImage() { + return "bitnami/spark:3.3.0"; + } +} diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/src/test/java/org/apache/seatunnel/e2e/spark/fake/FakeSourceToConsoleIT.java b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/src/test/java/org/apache/seatunnel/e2e/spark/fake/FakeSourceToConsoleIT.java new file mode 100644 index 00000000000..fbab18098ec --- /dev/null +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/src/test/java/org/apache/seatunnel/e2e/spark/fake/FakeSourceToConsoleIT.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.e2e.spark.fake; + +import org.apache.seatunnel.e2e.spark.SparkContainer; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.Container; + +import java.io.IOException; + +/** + * This test case is used to verify that the fake source is able to send data to the console. + * Make sure the SeaTunnel job can submit successfully on spark engine. + */ +public class FakeSourceToConsoleIT extends SparkContainer { + + @Test + public void testFakeSourceToConsoleSine() throws IOException, InterruptedException { + Container.ExecResult execResult = executeSeaTunnelSparkJob("/fake/fakesource_to_console.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + } +} diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/src/test/resources/fake/fakesource_to_console.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/src/test/resources/fake/fakesource_to_console.conf new file mode 100644 index 00000000000..a0d2b6d248a --- /dev/null +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/src/test/resources/fake/fakesource_to_console.conf @@ -0,0 +1,81 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + # You can set spark configuration here + # see available properties defined by spark: https://spark.apache.org/docs/latest/configuration.html#available-properties + job.mode = "BATCH" + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + # This is a example input plugin **only for test and demonstrate the feature input plugin** + FakeSource { + result_table_name = "my_dataset" + schema = { + fields { + name = "string" + age = "int" + } + } + } + + # You can also use other input plugins, such as hdfs + # hdfs { + # result_table_name = "accesslog" + # path = "hdfs://hadoop-cluster-01/nginx/accesslog" + # format = "json" + # } + + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource +} + +transform { + # split data by specific delimiter + + # you can also use other transform plugins, such as sql + # sql { + # sql = "select * from accesslog where request_time > 1000" + # } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/spark/configuration/transform-plugins/Split +} + +sink { + # choose stdout output plugin to output data to console + Console { + saveMode = "append" + } + + # you can also you other output plugins, such as sql + # hdfs { + # path = "hdfs://hadoop-cluster-01/nginx/accesslog_processed" + # save_mode = "append" + # } + + # If you would like to get more information about how to configure seatunnel and see full list of output plugins, + # please go to https://seatunnel.apache.org/docs/spark/configuration/sink-plugins/Console +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/src/test/resources/log4j.properties b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/src/test/resources/log4j.properties new file mode 100644 index 00000000000..db5d9e51220 --- /dev/null +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-3.3/src/test/resources/log4j.properties @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Set everything to be logged to the console +log4j.rootCategory=INFO, console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-common/pom.xml b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-common/pom.xml new file mode 100644 index 00000000000..64299f6bb09 --- /dev/null +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-common/pom.xml @@ -0,0 +1,58 @@ + + + + + seatunnel-spark-connector-v2-e2e + org.apache.seatunnel + ${revision} + + 4.0.0 + + seatunnel-spark-connector-v2-e2e-common + + + + org.apache.seatunnel + seatunnel-core-spark + ${project.version} + + + org.apache.seatunnel + seatunnel-connectors-v2-dist + ${project.version} + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + prepare-test-jar + test-compile + + test-jar + + + + + + + \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-common/src/test/java/org/apache/seatunnel/e2e/spark/v2/AbstractSparkContainer.java similarity index 83% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-common/src/test/java/org/apache/seatunnel/e2e/spark/v2/AbstractSparkContainer.java index 13984461286..af9584ee94b 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/seatunnel-spark-connector-v2-e2e-common/src/test/java/org/apache/seatunnel/e2e/spark/v2/AbstractSparkContainer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.seatunnel.e2e.spark; +package org.apache.seatunnel.e2e.spark.v2; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -41,30 +41,37 @@ /** * This class is the base class of SparkEnvironment test. The before method will create a Spark master, and after method will close the Spark master. - * You can use {@link SparkContainer#executeSeaTunnelSparkJob} to submit a seatunnel conf and a seatunnel spark job. + * You can use {@link AbstractSparkContainer#executeSeaTunnelSparkJob} to submit a seatunnel conf and a seatunnel spark job. */ -public abstract class SparkContainer { +public abstract class AbstractSparkContainer { - private static final Logger LOG = LoggerFactory.getLogger(SparkContainer.class); + private static final Logger LOG = LoggerFactory.getLogger(AbstractSparkContainer.class); - private static final String SPARK_DOCKER_IMAGE = "bitnami/spark:2.4.3"; public static final Network NETWORK = Network.newNetwork(); protected GenericContainer master; - private static final Path PROJECT_ROOT_PATH = Paths.get(System.getProperty("user.dir")).getParent().getParent(); - private static final String SEATUNNEL_SPARK_BIN = "start-seatunnel-spark-new-connector.sh"; + protected static final Path PROJECT_ROOT_PATH = Paths.get(System.getProperty("user.dir")).getParent() + .getParent().getParent(); + protected static final String SEATUNNEL_SPARK_BIN = "start-seatunnel-spark-new-connector.sh"; private static final String SEATUNNEL_SPARK_JAR = "seatunnel-spark-starter.jar"; private static final String PLUGIN_MAPPING_FILE = "plugin-mapping.properties"; - private static final String SEATUNNEL_HOME = "/tmp/spark/seatunnel"; + protected static final String SEATUNNEL_HOME = "/tmp/spark/seatunnel"; private static final String SEATUNNEL_BIN = Paths.get(SEATUNNEL_HOME, "bin").toString(); - private static final String SPARK_JAR_PATH = Paths.get(SEATUNNEL_HOME, "lib", SEATUNNEL_SPARK_JAR).toString(); + private static final String STARTER_JAR_PATH = Paths.get(SEATUNNEL_HOME, "lib", + "seatunnel-spark-starter.jar").toString(); private static final String CONNECTORS_PATH = Paths.get(SEATUNNEL_HOME, "connectors").toString(); private static final int WAIT_SPARK_JOB_SUBMIT = 5000; + protected abstract String getTranslationJarPath(); + + protected abstract String getTranslationJarTargetPath(); + + protected abstract String getSparkDockerImage(); + @BeforeEach public void before() { - master = new GenericContainer<>(SPARK_DOCKER_IMAGE) + master = new GenericContainer<>(getSparkDockerImage()) .withNetwork(NETWORK) .withNetworkAliases("spark-master") .withExposedPorts() @@ -116,7 +123,7 @@ protected void copySeaTunnelSparkFile() { // copy lib String seatunnelCoreSparkJarPath = Paths.get(PROJECT_ROOT_PATH.toString(), "seatunnel-core", "seatunnel-spark-starter", "target", SEATUNNEL_SPARK_JAR).toString(); - master.copyFileToContainer(MountableFile.forHostPath(seatunnelCoreSparkJarPath), SPARK_JAR_PATH); + master.copyFileToContainer(MountableFile.forHostPath(seatunnelCoreSparkJarPath), STARTER_JAR_PATH); // copy bin String seatunnelFlinkBinPath = Paths.get(PROJECT_ROOT_PATH.toString(), @@ -132,6 +139,10 @@ protected void copySeaTunnelSparkFile() { MountableFile.forHostPath(jar.getAbsolutePath()), getConnectorPath(jar.getName()))); + // copy lib + String translationSparkJarPath = getTranslationJarPath(); + master.copyFileToContainer(MountableFile.forHostPath(translationSparkJarPath), getTranslationJarTargetPath()); + // copy plugin-mapping.properties master.copyFileToContainer( MountableFile.forHostPath(PROJECT_ROOT_PATH + "/plugin-mapping.properties"), diff --git a/seatunnel-translation/seatunnel-translation-spark/pom.xml b/seatunnel-translation/seatunnel-translation-spark/pom.xml index 25c4bce5b41..ae52c235b60 100644 --- a/seatunnel-translation/seatunnel-translation-spark/pom.xml +++ b/seatunnel-translation/seatunnel-translation-spark/pom.xml @@ -29,5 +29,7 @@ seatunnel-translation-spark-common seatunnel-translation-spark-2.4 seatunnel-translation-spark-3.3 + seatunnel-translation-spark-2.4-dist + seatunnel-translation-spark-3.3-dist diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4-dist/pom.xml b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4-dist/pom.xml new file mode 100644 index 00000000000..72da1c39995 --- /dev/null +++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4-dist/pom.xml @@ -0,0 +1,39 @@ + + + + + seatunnel-translation-spark + org.apache.seatunnel + ${revision} + + 4.0.0 + + seatunnel-translation-spark-2.4-dist + + + + org.apache.seatunnel + seatunnel-translation-spark-2.4 + ${project.version} + + + + + ${project.name} + + \ No newline at end of file diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/pom.xml b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/pom.xml index 850812a1a4c..5d9ea8e4099 100644 --- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/pom.xml +++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/pom.xml @@ -60,5 +60,4 @@ ${spark.scope} - \ No newline at end of file diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3-dist/pom.xml b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3-dist/pom.xml new file mode 100644 index 00000000000..990c31464c1 --- /dev/null +++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3-dist/pom.xml @@ -0,0 +1,39 @@ + + + + + seatunnel-translation-spark + org.apache.seatunnel + ${revision} + + 4.0.0 + + seatunnel-translation-spark-3.3-dist + + + + org.apache.seatunnel + seatunnel-translation-spark-3.3 + ${project.version} + + + + + ${project.name} + + \ No newline at end of file