Skip to content
Permalink
Browse files
[IOTDB-1310] Enable docker, docker-compose and testcontainer for End …
…to end test (#3024)

* enable TestCongtainer for E2E test for (singleNode and cluster)

* remove duplicated operations in integration-test phase

* move spotless:apply to a profile `spotless`, which is enabled by default.

Co-authored-by: xiangdong huang <sainthxd@gmail.com>
  • Loading branch information
jixuan1989 and xiangdong huang committed Apr 25, 2021
1 parent f2e53d7 commit 7458f4d342b3a8002f12127c98c9552b1a7989e6
Show file tree
Hide file tree
Showing 35 changed files with 962 additions and 24 deletions.
@@ -39,11 +39,11 @@ jobs:
restore-keys: ${{ runner.os }}-m2

- name: Build Distribution Zip
run: ./mvnw.sh -B -DskipTests clean package
run: ./mvnw.sh -B -DskipTests clean install

- name: Build Docker Image
run: |
docker build . -f docker/src/main/Dockerfile -t "iotdb:$GITHUB_SHA"
docker build . -f docker/src/main/Dockerfile-single -t "iotdb:$GITHUB_SHA"
docker images
- name: Run Test Case ${{ matrix.case }}
@@ -52,3 +52,7 @@ jobs:
- name: Clean Up
if: ${{ always() }}
run: bash test/e2e/cases/${{ matrix.case }}/cleanup.sh

- name: TestContainer
run: |
mvn -B integration-test -pl testcontainer
@@ -96,7 +96,6 @@
<phase>integration-test</phase>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
@@ -202,7 +202,6 @@
<phase>integration-test</phase>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
@@ -29,6 +29,7 @@
#-------------------------------------------IMPORTANT---------------------------------------------#

# used for communication between cluster nodes, eg heartbeat、raft logs and snapshots etc.
# if this parameter is commented, then the IP that binded by the hostname will be used.
internal_ip=127.0.0.1

# port for metadata service
@@ -51,7 +52,8 @@ internal_data_port=40010
# nodes that already in the cluster, unnecessary to be the nodes that were used to build the
# initial cluster by start-node.sh(.bat). Several nodes will be picked randomly to send the
# request, the number of nodes picked depends on the number of retries.
seed_nodes=127.0.0.1:9003,127.0.0.1:9005,127.0.0.1:9007
#seed_nodes=127.0.0.1:9003,127.0.0.1:9005,127.0.0.1:9007
seed_nodes=127.0.0.1:9003

# whether to use thrift compressed protocol for internal communications. If you want to change
# compression settings for external clients, please modify 'rpc_thrift_compression_enable' in
@@ -157,6 +157,7 @@ private static void startServerCheck() throws StartupException {
config.getSeedNodeUrls().size(), quorum);
throw new StartupException(metaServer.getMember().getName(), message);
}

// assert not duplicated nodes
Set<Node> seedNodes = new HashSet<>();
for (String url : config.getSeedNodeUrls()) {
@@ -21,22 +21,26 @@
import org.apache.iotdb.cluster.utils.ClusterConsistent;
import org.apache.iotdb.db.conf.IoTDBDescriptor;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.TimeUnit;

public class ClusterConfig {

private static Logger logger = LoggerFactory.getLogger(ClusterConfig.class);
static final String CONFIG_NAME = "iotdb-cluster.properties";

private String internalIp = "127.0.0.1";
private String internalIp;
private int internalMetaPort = 9003;
private int internalDataPort = 40010;
private int clusterRpcPort = IoTDBDescriptor.getInstance().getConfig().getRpcPort();

/** each one is a {internalIp | domain name}:{meta port} string tuple. */
private List<String> seedNodeUrls =
Arrays.asList(String.format("%s:%d", internalIp, internalMetaPort));
private List<String> seedNodeUrls;

@ClusterConsistent private boolean isRpcThriftCompressionEnabled = false;
private int maxConcurrentClientNum = 10000;
@@ -164,6 +168,21 @@ public class ClusterConfig {

private boolean openServerRpcPort = false;

/**
* create a clusterConfig class. The internalIP will be set according to the server's hostname. If
* there is something error for getting the ip of the hostname, then set the internalIp as
* localhost.
*/
public ClusterConfig() {
try {
internalIp = InetAddress.getLocalHost().getHostAddress();
} catch (UnknownHostException e) {
logger.error(e.getMessage());
internalIp = "127.0.0.1";
}
seedNodeUrls = Arrays.asList(String.format("%s:%d", internalIp, internalMetaPort));
}

public int getSelectorNumOfClientPool() {
return selectorNumOfClientPool;
}
@@ -73,7 +73,6 @@
<phase>integration-test</phase>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
@@ -0,0 +1,43 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#

# docker build context is the root path of the repository

FROM openjdk:11-jre-slim

ADD distribution/target/apache-iotdb-*-cluster-bin.zip /

RUN apt update \
&& apt install lsof procps unzip -y \
&& unzip /apache-iotdb-*-bin.zip -d / \
&& rm /apache-iotdb-*-bin.zip \
&& mv /apache-iotdb-* /iotdb \
&& apt remove unzip -y \
&& apt autoremove -y \
&& apt purge --auto-remove -y \
&& apt clean -y

EXPOSE 6667
EXPOSE 31999
EXPOSE 5555
EXPOSE 8181
VOLUME /iotdb/data
VOLUME /iotdb/logs
ENV PATH="/iotdb/sbin/:/iotdb/tools/:${PATH}"
ENTRYPOINT ["/iotdb/sbin/start-node.sh"]
@@ -21,7 +21,7 @@

FROM openjdk:11-jre-slim

ADD distribution/target/apache-iotdb-*-all-bin.zip /
ADD distribution/target/apache-iotdb-*-server-bin.zip /

RUN apt update \
&& apt install lsof procps unzip -y \
@@ -197,7 +197,6 @@
<phase>integration-test</phase>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
@@ -116,7 +116,6 @@
<phase>integration-test</phase>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
@@ -159,7 +159,6 @@
<phase>integration-test</phase>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
56 pom.xml
@@ -509,6 +509,12 @@
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>1.15.2</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<pluginManagement>
@@ -913,6 +919,10 @@
<!-- Include integration tests within integration-test phase. -->
<include>src/test/**/*IT.java</include>
</includes>
<excludes>
<!-- Exclude unit tests within (unit) test phase. -->
<exclude>src/test/**/*Test.java</exclude>
</excludes>
</configuration>
</execution>
</executions>
@@ -932,13 +942,27 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.diffplug.spotless</groupId>
<artifactId>spotless-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
<profiles>
<!-- spotless is too slow, so we put it into a profile to skip it if needed -->
<profile>
<id>spotless</id>
<activation>
<!-- activeByDefault does not take effect-->
<file>
<exists>.</exists>
</file>
</activation>
<build>
<plugins>
<plugin>
<groupId>com.diffplug.spotless</groupId>
<artifactId>spotless-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</profile>
<!--
A set of profiles defining the different properties needed to download and run thrift
They are automatically activated depending on the OS you are using.
@@ -992,6 +1016,30 @@
<thrift.exec-cmd.args>+x ${project.build.directory}/tools/${thrift.executable}</thrift.exec-cmd.args>
</properties>
</profile>
<!-- for TestContainer. As it requires docker, we have to detect whether docker exists.-->
<profile>
<!-- Mac and Unix-->
<id>unixDockerCheck</id>
<activation>
<file>
<exists>/var/run/docker.sock</exists>
</file>
</activation>
<modules>
<module>testcontainer</module>
</modules>
</profile>
<profile>
<id>WinDockerCheck</id>
<activation>
<file>
<exists>C:\Program Files\Docker\Docker\resources\bin\docker.exe</exists>
</file>
</activation>
<modules>
<module>testcontainer</module>
</modules>
</profile>
<!-- Some APIs were removed in Java 11, so we need to add replacements -->
<profile>
<id>java-11-and-above</id>
@@ -231,7 +231,6 @@
<phase>integration-test</phase>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
@@ -0,0 +1,17 @@
#Licensed to the Apache Software Foundation (ASF) under one
#or more contributor license agreements. See the NOTICE file
#distributed with this work for additional information
#regarding copyright ownership. The ASF licenses this file
#to you under the Apache License, Version 2.0 (the
#"License"); you may not use this file except in compliance
#with the License. You may obtain a copy of the License at
#
#http://www.apache.org/licenses/LICENSE-2.0
#
#Unless required by applicable law or agreed to in writing,
#software distributed under the License is distributed on an
#"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#KIND, either express or implied. See the License for the
#specific language governing permissions and limitations
#under the License.

@@ -114,7 +114,6 @@
<phase>integration-test</phase>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
@@ -54,7 +54,6 @@
<phase>integration-test</phase>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
@@ -23,7 +23,7 @@ services:
server-prototype:
build:
context: ../../..
dockerfile: docker/src/main/Dockerfile
dockerfile: docker/src/main/Dockerfile-single
ports:
- 6667:6667
networks:
@@ -37,7 +37,7 @@ services:
initializer:
build:
context: ../../..
dockerfile: docker/src/main/Dockerfile
dockerfile: docker/src/main/Dockerfile-single
networks:
iotdb:
entrypoint:
0 test/e2e/cases/cli/cleanup.sh 100644 → 100755
Empty file.
0 test/e2e/cases/cli/run.sh 100644 → 100755
Empty file.

0 comments on commit 7458f4d

Please sign in to comment.