Skip to content

Commit

Permalink
[FLINK-30430][Connector/Pulsar] Remove all the useless dependencies, …
Browse files Browse the repository at this point in the history 
…drop mock runtime.
  • Loading branch information
@syhily
syhily committed Dec 29, 2022
1 parent 6fe0a44 commit b90119e
Show file tree
Hide file tree
Showing 16 changed files with 159 additions and 1,071 deletions.
29 changes: 5 additions & 24 deletions flink-connector-pulsar-e2e-tests/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,44 +27,29 @@ under the License.
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>flink-end-to-end-tests-pulsar</artifactId>
<artifactId>flink-connector-pulsar-e2e-tests</artifactId>
<name>Flink : E2E Tests : Pulsar</name>

<properties>
<jaxb-api.version>2.3.1</jaxb-api.version>
<japicmp.skip>true</japicmp.skip>
</properties>

<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-test-utils</artifactId>
<version>${flink.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-pulsar</artifactId>
<version>${project.version}</version>
</dependency>
<!-- pulsar-client-all requires jaxb-api for javax.xml.bind.annotation.XmlElement -->
<!-- packaged in flink-dist but not provided in e2e environment. -->
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>${jaxb-api.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-pulsar</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-test-utils</artifactId>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>pulsar</artifactId>
Expand Down Expand Up @@ -114,10 +99,6 @@ under the License.

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
Expand Down
142 changes: 8 additions & 134 deletions flink-connector-pulsar/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,33 +34,20 @@ under the License.

<packaging>jar</packaging>

<properties>
<protoc.version>3.21.7</protoc.version>

<!-- Test Libraries -->
<protobuf-maven-plugin.version>0.6.1</protobuf-maven-plugin.version>
<os-maven-plugin.version>1.7.0</os-maven-plugin.version>
<pulsar-netty.version>4.1.77.Final</pulsar-netty.version>
<pulsar-grpc.version>1.45.1</pulsar-grpc.version>
<pulsar-caffeine.version>2.9.1</pulsar-caffeine.version>
</properties>

<dependencies>

<!-- Core -->

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-base</artifactId>
<version>${flink.version}</version>
</dependency>

<!-- Connectors -->

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>

Expand All @@ -69,11 +56,17 @@ under the License.
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${protoc.version}</version>
<scope>provided</scope>
<optional>true</optional>
</dependency>

<!-- Pulsar Client -->

<dependency>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-client-all</artifactId>
</dependency>

<!-- Tests -->

<dependency>
Expand All @@ -95,7 +88,6 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-test-utils</artifactId>
<version>${flink.version}</version>
<scope>test</scope>
</dependency>

Expand All @@ -106,131 +98,13 @@ under the License.
<artifactId>pulsar</artifactId>
<scope>test</scope>
</dependency>

<!-- Pulsar bundles the latest bookkeeper -->
<!-- we don't override the version here. -->
<dependency>
<groupId>org.apache.pulsar</groupId>
<artifactId>testmocks</artifactId>
<version>${pulsar.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</exclusion>
<exclusion>
<groupId>org.powermock</groupId>
<artifactId>powermock-module-testng</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-broker</artifactId>
<version>${pulsar.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
<version>${pulsar-caffeine.version}</version>
<scope>test</scope>
</dependency>

<!-- Add Pulsar 2.x as a dependency. -->
<!-- Move this to button for avoiding class conflicts with pulsar-broker. -->

<dependency>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-client-all</artifactId>
<version>${pulsar.version}</version>
<exclusions>
<exclusion>
<groupId>com.sun.activation</groupId>
<artifactId>javax.activation</artifactId>
</exclusion>
<exclusion>
<groupId>jakarta.activation</groupId>
<artifactId>jakarta.activation-api</artifactId>
</exclusion>
<exclusion>
<groupId>jakarta.ws.rs</groupId>
<artifactId>jakarta.ws.rs-api</artifactId>
</exclusion>
<exclusion>
<groupId>jakarta.xml.bind</groupId>
<artifactId>jakarta.xml.bind-api</artifactId>
</exclusion>
<exclusion>
<groupId>javax.validation</groupId>
<artifactId>validation-api</artifactId>
</exclusion>
<exclusion>
<groupId>net.jcip</groupId>
<artifactId>jcip-annotations</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-package-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.beust</groupId>
<artifactId>jcommander</artifactId>
</exclusion>
</exclusions>
</dependency>

<!-- ArchUit test dependencies -->

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-architecture-tests-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>


<dependencyManagement>
<dependencies>
<!-- Pulsar use higher gRPC version. -->
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-bom</artifactId>
<version>${pulsar-grpc.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>

<!-- Pulsar use higher netty version. -->
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-bom</artifactId>
<version>${pulsar-netty.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>

<build>
<extensions>
<extension>
<groupId>kr.motd.maven</groupId>
<artifactId>os-maven-plugin</artifactId>
<version>${os-maven-plugin.version}</version>
</extension>
</extensions>

<plugins>
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
<version>${protobuf-maven-plugin.version}</version>
<extensions>true</extensions>
<configuration>
<!-- Currently Flink azure test pipeline would first pre-compile and then upload the compiled
Expand All @@ -247,7 +121,7 @@ under the License.
${project.build.directory}/generated-test-sources/protobuf/java
</outputDirectory>
<protocArtifact>
com.google.protobuf:protoc:${protoc.version}:exe:${os.detected.classifier}
com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
</protocArtifact>
</configuration>
<executions>
Expand Down
2 changes: 1 addition & 1 deletion ...nnector-pulsar/src/test/java/org/apache/flink/connector/pulsar/sink/PulsarSinkITCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ class IntegrationTest extends SinkTestSuiteBase<String> {
@TestEnv MiniClusterTestEnvironment flink = new MiniClusterTestEnvironment();

@TestExternalSystem
PulsarTestEnvironment pulsar = new PulsarTestEnvironment(PulsarRuntime.mock());
PulsarTestEnvironment pulsar = new PulsarTestEnvironment(PulsarRuntime.container());

@TestSemantics
CheckpointingMode[] semantics =
Expand Down
2 changes: 1 addition & 1 deletion ...tor-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/PulsarSourceITCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ class PulsarSourceITCase extends SourceTestSuiteBase<String> {

// Defines pulsar running environment
@TestExternalSystem
PulsarTestEnvironment pulsar = new PulsarTestEnvironment(PulsarRuntime.mock());
PulsarTestEnvironment pulsar = new PulsarTestEnvironment(PulsarRuntime.container());

// This field is preserved, we don't support the semantics in source currently.
@TestSemantics
Expand Down
2 changes: 1 addition & 1 deletion ...r/src/test/java/org/apache/flink/connector/pulsar/source/PulsarUnorderedSourceITCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ public class PulsarUnorderedSourceITCase extends UnorderedSourceTestSuiteBase<St

// Defines pulsar running environment
@TestExternalSystem
PulsarTestEnvironment pulsar = new PulsarTestEnvironment(PulsarRuntime.mock());
PulsarTestEnvironment pulsar = new PulsarTestEnvironment(PulsarRuntime.container());

@TestSemantics
CheckpointingMode[] semantics = new CheckpointingMode[] {CheckpointingMode.EXACTLY_ONCE};
Expand Down
2 changes: 1 addition & 1 deletion ...pulsar/src/test/java/org/apache/flink/connector/pulsar/testutils/PulsarTestSuiteBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public abstract class PulsarTestSuiteBase {
* pulsar broker. Override this method when needs.
*/
protected PulsarRuntime runtime() {
return PulsarRuntime.mock();
return PulsarRuntime.container();
}

/** Operate pulsar by acquiring a runtime operator. */
Expand Down
16 changes: 4 additions & 12 deletions ...lsar/src/test/java/org/apache/flink/connector/pulsar/testutils/runtime/PulsarRuntime.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@

import org.apache.flink.connector.pulsar.testutils.PulsarTestEnvironment;
import org.apache.flink.connector.pulsar.testutils.runtime.container.PulsarContainerRuntime;
import org.apache.flink.connector.pulsar.testutils.runtime.mock.PulsarMockRuntime;

import org.testcontainers.containers.GenericContainer;

Expand All @@ -44,26 +43,19 @@ public interface PulsarRuntime {
*/
PulsarRuntimeOperator operator();

/** Create a Pulsar instance which would mock all the backends. */
static PulsarRuntime mock() {
return new PulsarMockRuntime();
}

/**
* Create a Pulsar instance in docker. We would start a standalone Pulsar in TestContainers.
* This runtime is often used in end-to-end tests. The performance may be a bit of slower than
* {@link #mock()}. The stream storage for bookkeeper is disabled. The function worker is
* disabled on Pulsar broker.
* This runtime is often used in end-to-end tests. The stream storage for bookkeeper is
* disabled. The function worker is disabled on Pulsar broker.
*/
static PulsarRuntime container() {
return new PulsarContainerRuntime();
}

/**
* Create a Pulsar instance in docker. We would start a standalone Pulsar in TestContainers.
* This runtime is often used in end-to-end tests. The performance may be a bit of slower than
* {@link #mock()}. The stream storage for bookkeeper is disabled. The function worker is
* disabled on Pulsar broker.
* This runtime is often used in end-to-end tests. The stream storage for bookkeeper is
* disabled. The function worker is disabled on Pulsar broker.
*
* <p>We would link the created Pulsar docker instance with the given flink instance. This would
* enable the connection for Pulsar and Flink in docker environment.
Expand Down

0 comments on commit b90119e

Please sign in to comment.