From fa1098fa7b727f1967f71d1e0cb00e2d08ba0d6d Mon Sep 17 00:00:00 2001 From: Zongwen Li Date: Wed, 28 Sep 2022 10:33:19 +0800 Subject: [PATCH] [Improve][e2e] Remove the un-unified e2e module of Redis --- .../connector-redis-flink-e2e/pom.xml | 53 ----------- .../seatunnel/e2e/flink/v2/redis/RedisIT.java | 92 ------------------- .../src/test/resources/log4j.properties | 22 ----- .../redis/redis_source_and_sink.conf | 68 -------------- .../seatunnel-flink-connector-v2-e2e/pom.xml | 1 - .../connector-redis-spark-e2e/pom.xml | 53 ----------- .../seatunnel/e2e/spark/v2/redis/RedisIT.java | 92 ------------------- .../src/test/resources/log4j.properties | 22 ----- .../redis/redis_source_and_sink.conf | 71 -------------- .../seatunnel-spark-connector-v2-e2e/pom.xml | 1 - 10 files changed, 475 deletions(-) delete mode 100644 seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-redis-flink-e2e/pom.xml delete mode 100644 seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-redis-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/redis/RedisIT.java delete mode 100644 seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-redis-flink-e2e/src/test/resources/log4j.properties delete mode 100644 seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-redis-flink-e2e/src/test/resources/redis/redis_source_and_sink.conf delete mode 100644 seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-redis-spark-e2e/pom.xml delete mode 100644 seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-redis-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/redis/RedisIT.java delete mode 100644 seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-redis-spark-e2e/src/test/resources/log4j.properties delete mode 100644 seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-redis-spark-e2e/src/test/resources/redis/redis_source_and_sink.conf diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-redis-flink-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-redis-flink-e2e/pom.xml deleted file mode 100644 index d92a1b3b55b..00000000000 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-redis-flink-e2e/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - - org.apache.seatunnel - seatunnel-flink-connector-v2-e2e - ${revision} - - 4.0.0 - - connector-redis-flink-e2e - - - - org.apache.seatunnel - connector-flink-e2e-base - ${project.version} - tests - test-jar - test - - - - - org.apache.seatunnel - connector-redis - ${project.version} - test - - - org.apache.seatunnel - connector-assert - ${project.version} - test - - - - \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-redis-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/redis/RedisIT.java b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-redis-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/redis/RedisIT.java deleted file mode 100644 index 93d4361388f..00000000000 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-redis-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/redis/RedisIT.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.e2e.flink.v2.redis; - -import static org.awaitility.Awaitility.given; - -import org.apache.seatunnel.e2e.flink.FlinkContainer; - -import com.google.common.collect.Lists; -import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.Container; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.output.Slf4jLogConsumer; -import org.testcontainers.lifecycle.Startables; -import redis.clients.jedis.Jedis; - -import java.io.IOException; -import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; - -@Slf4j -public class RedisIT extends FlinkContainer { - private static final String REDIS_IMAGE = "redis:latest"; - private static final String REDIS_CONTAINER_HOST = "flink_e2e_redis"; - private static final String REDIS_HOST = "localhost"; - private static final int REDIS_PORT = 6379; - private GenericContainer redisContainer; - private Jedis jedis; - - @BeforeEach - public void startRedisContainer() { - redisContainer = new GenericContainer<>(REDIS_IMAGE) - .withNetwork(NETWORK) - .withNetworkAliases(REDIS_CONTAINER_HOST) - .withLogConsumer(new Slf4jLogConsumer(log)); - redisContainer.setPortBindings(Lists.newArrayList(String.format("%s:%s", REDIS_PORT, REDIS_PORT))); - Startables.deepStart(Stream.of(redisContainer)).join(); - log.info("Redis container started"); - given().ignoreExceptions() - .await() - .atLeast(100, TimeUnit.MILLISECONDS) - .pollInterval(500, TimeUnit.MILLISECONDS) - .atMost(180, TimeUnit.SECONDS) - .untilAsserted(this::initJedis); - this.generateTestData(); - } - - private void initJedis() { - jedis = new Jedis(REDIS_HOST, REDIS_PORT); - jedis.connect(); - } - - private void generateTestData() { - jedis.set("key_test", "test"); - jedis.set("key_test1", "test1"); - jedis.set("key_test2", "test2"); - jedis.set("key_test3", "test3"); - jedis.set("key_test4", "test4"); - } - - @Test - public void testRedisSourceAndSink() throws IOException, InterruptedException { - Container.ExecResult execResult = executeSeaTunnelFlinkJob("/redis/redis_source_and_sink.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - Assertions.assertEquals(5, jedis.llen("key_list")); - } - - @AfterEach - public void close() { - jedis.close(); - redisContainer.close(); - } -} diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-redis-flink-e2e/src/test/resources/log4j.properties b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-redis-flink-e2e/src/test/resources/log4j.properties deleted file mode 100644 index db5d9e51220..00000000000 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-redis-flink-e2e/src/test/resources/log4j.properties +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Set everything to be logged to the console -log4j.rootCategory=INFO, console -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-redis-flink-e2e/src/test/resources/redis/redis_source_and_sink.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-redis-flink-e2e/src/test/resources/redis/redis_source_and_sink.conf deleted file mode 100644 index a7deb67b2ac..00000000000 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-redis-flink-e2e/src/test/resources/redis/redis_source_and_sink.conf +++ /dev/null @@ -1,68 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -###### -###### This config file is a demonstration of streaming processing in seatunnel config -###### - -env { - # You can set flink configuration here - execution.parallelism = 1 - job.mode = "BATCH" - #execution.checkpoint.interval = 10000 - #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint" -} - -source { - # This is a example source plugin **only for test and demonstrate the feature source plugin** - Redis { - host = flink_e2e_redis - port = 6379 - keys = "key_test*" - data_type = key - format = text - } - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Redis -} - -transform { - -} - -sink { - - Redis { - host = flink_e2e_redis - port = 6379 - key = "key_list" - data_type = list - } - - Assert { - rules = [ - { - field_name = content - field_type = string - field_value = [ - { - rule_type = NOT_NULL - } - ] - } - ] - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/pom.xml index 9f9f264ae6c..b492ce56c10 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/pom.xml @@ -28,7 +28,6 @@ connector-flink-e2e-base - connector-redis-flink-e2e connector-file-flink-e2e connector-jdbc-flink-e2e connector-iotdb-flink-e2e diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-redis-spark-e2e/pom.xml b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-redis-spark-e2e/pom.xml deleted file mode 100644 index 25212814ad8..00000000000 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-redis-spark-e2e/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - - org.apache.seatunnel - seatunnel-spark-connector-v2-e2e - ${revision} - - 4.0.0 - - connector-redis-spark-e2e - - - - org.apache.seatunnel - connector-spark-e2e-base - ${project.version} - tests - test-jar - test - - - - - org.apache.seatunnel - connector-assert - ${project.version} - test - - - org.apache.seatunnel - connector-redis - ${project.version} - test - - - - \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-redis-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/redis/RedisIT.java b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-redis-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/redis/RedisIT.java deleted file mode 100644 index 065c4e680f2..00000000000 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-redis-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/redis/RedisIT.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.e2e.spark.v2.redis; - -import static org.awaitility.Awaitility.given; - -import org.apache.seatunnel.e2e.spark.SparkContainer; - -import com.google.common.collect.Lists; -import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.Container; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.output.Slf4jLogConsumer; -import org.testcontainers.lifecycle.Startables; -import redis.clients.jedis.Jedis; - -import java.io.IOException; -import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; - -@Slf4j -public class RedisIT extends SparkContainer { - private static final String REDIS_IMAGE = "redis:latest"; - private static final String REDIS_CONTAINER_HOST = "spark_e2e_redis"; - private static final String REDIS_HOST = "localhost"; - private static final int REDIS_PORT = 6379; - private GenericContainer redisContainer; - private Jedis jedis; - - @BeforeEach - public void startRedisContainer() { - redisContainer = new GenericContainer<>(REDIS_IMAGE) - .withNetwork(NETWORK) - .withNetworkAliases(REDIS_CONTAINER_HOST) - .withLogConsumer(new Slf4jLogConsumer(log)); - redisContainer.setPortBindings(Lists.newArrayList(String.format("%s:%s", REDIS_PORT, REDIS_PORT))); - Startables.deepStart(Stream.of(redisContainer)).join(); - log.info("Redis container started"); - given().ignoreExceptions() - .await() - .atLeast(100, TimeUnit.MILLISECONDS) - .pollInterval(500, TimeUnit.MILLISECONDS) - .atMost(180, TimeUnit.SECONDS) - .untilAsserted(this::initJedis); - this.generateTestData(); - } - - private void initJedis() { - jedis = new Jedis(REDIS_HOST, REDIS_PORT); - jedis.connect(); - } - - private void generateTestData() { - jedis.set("key_test", "test"); - jedis.set("key_test1", "test1"); - jedis.set("key_test2", "test2"); - jedis.set("key_test3", "test3"); - jedis.set("key_test4", "test4"); - } - - @Test - public void testRedisSourceAndSink() throws IOException, InterruptedException { - Container.ExecResult execResult = executeSeaTunnelSparkJob("/redis/redis_source_and_sink.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - Assertions.assertEquals(5, jedis.llen("key_list")); - } - - @AfterEach - public void close() { - jedis.close(); - redisContainer.close(); - } -} diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-redis-spark-e2e/src/test/resources/log4j.properties b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-redis-spark-e2e/src/test/resources/log4j.properties deleted file mode 100644 index db5d9e51220..00000000000 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-redis-spark-e2e/src/test/resources/log4j.properties +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Set everything to be logged to the console -log4j.rootCategory=INFO, console -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-redis-spark-e2e/src/test/resources/redis/redis_source_and_sink.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-redis-spark-e2e/src/test/resources/redis/redis_source_and_sink.conf deleted file mode 100644 index 051e2f61d4d..00000000000 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-redis-spark-e2e/src/test/resources/redis/redis_source_and_sink.conf +++ /dev/null @@ -1,71 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - # You can set spark configuration here - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local - job.mode = "BATCH" -} - -source { - # This is a example source plugin **only for test and demonstrate the feature source plugin** - Redis { - host = spark_e2e_redis - port = 6379 - keys = "key_test*" - data_type = key - format = text - } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Redis -} - -transform { - -} - -sink { - - Redis { - host = spark_e2e_redis - port = 6379 - key = "key_list" - data_type = list - } - - Assert { - rules = [ - { - field_name = content - field_type = string - field_value = [ - { - rule_type = NOT_NULL - } - ] - } - ] - } - - # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Console -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/pom.xml b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/pom.xml index cc5cc2bafa3..08bfa545fc8 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/pom.xml @@ -33,7 +33,6 @@ connector-file-spark-e2e connector-iotdb-spark-e2e connector-jdbc-spark-e2e - connector-redis-spark-e2e connector-mongodb-spark-e2e