Skip to content
Permalink
Browse files

Add integration tests

These tests are small Spring Cloud Kubernetes application that are
deployed inside a Kubernetes cluster launched with microk8s.
The deployment of the applications is done with the Fabric8 Maven Plugin
  • Loading branch information...
geoand authored and iocanel committed Oct 24, 2018
1 parent 9504dc5 commit 9f02f7e7533b36e4f3e03b8918acc4dbf4a95444
Showing with 791 additions and 5 deletions.
  1. +46 −4 .circleci/config.yml
  2. +3 −1 pom.xml
  3. +108 −0 spring-cloud-kubernetes-integration-tests/README.md
  4. +13 −0 spring-cloud-kubernetes-integration-tests/deploy_test_undeploy_all.sh
  5. +27 −0 spring-cloud-kubernetes-integration-tests/deploy_test_undeploy_single.sh
  6. +140 −0 spring-cloud-kubernetes-integration-tests/pom.xml
  7. +64 −0 spring-cloud-kubernetes-integration-tests/simple-configmap/pom.xml
  8. +10 −0 spring-cloud-kubernetes-integration-tests/simple-configmap/src/main/fabric8/deployment.yml
  9. +7 −0 spring-cloud-kubernetes-integration-tests/simple-configmap/src/main/fabric8/svc.yml
  10. +34 −0 ...ion-tests/simple-configmap/src/main/java/org/springframework/cloud/kubernetes/it/DummyConfig.java
  11. +40 −0 ...e-configmap/src/main/java/org/springframework/cloud/kubernetes/it/SimpleConfigMapApplication.java
  12. +12 −0 spring-cloud-kubernetes-integration-tests/simple-configmap/src/main/resources/application.yml
  13. +14 −0 spring-cloud-kubernetes-integration-tests/simple-configmap/src/main/resources/bootstrap.yaml
  14. +52 −0 ...tion-tests/simple-configmap/src/test/java/org/springframework/cloud/kubernetes/it/GreetingIT.java
  15. +13 −0 spring-cloud-kubernetes-integration-tests/simple-configmap/src/test/resources/arquillian.xml
  16. +6 −0 spring-cloud-kubernetes-integration-tests/simple-configmap/src/test/resources/bootstrap.yaml
  17. +8 −0 spring-cloud-kubernetes-integration-tests/simple-configmap/src/test/resources/config-map.yml
  18. +5 −0 spring-cloud-kubernetes-integration-tests/simple-configmap/src/test/resources/logback-test.xml
  19. +64 −0 spring-cloud-kubernetes-integration-tests/simple-core/pom.xml
  20. +10 −0 spring-cloud-kubernetes-integration-tests/simple-core/src/main/fabric8/deployment.yml
  21. +15 −0 spring-cloud-kubernetes-integration-tests/simple-core/src/main/fabric8/svc.yml
  22. +34 −0 ...ests/simple-core/src/main/java/org/springframework/cloud/kubernetes/it/SimpleCoreApplication.java
  23. +9 −0 spring-cloud-kubernetes-integration-tests/simple-core/src/main/resources/application.yml
  24. +39 −0 ...-tests/simple-core/src/test/java/org/springframework/cloud/kubernetes/it/GreetingAndHealthIT.java
  25. +13 −0 spring-cloud-kubernetes-integration-tests/simple-core/src/test/resources/arquillian.xml
  26. +5 −0 spring-cloud-kubernetes-integration-tests/simple-core/src/test/resources/logback-test.xml
@@ -20,8 +20,7 @@ jobs:
branches:
ignore:
- gh-pages
docker:
- image: circleci/openjdk:8-jdk-node-browsers
machine: true
environment:
_JAVA_OPTIONS: "-Xms1024m -Xmx2048m"
_SERVICE_OCCURENCE: 5
@@ -41,18 +40,61 @@ jobs:
- ~/.m2
key: spring-cloud-kubernetes-{{ .Branch }}-{{ checksum "pom.xml" }}
- run:
name: test
name: Run regular tests
command: |
./mvnw -s .settings.xml clean install -Dservice.occurence=${_SERVICE_OCCURENCE} #org.jacoco:jacoco-maven-plugin:prepare-agent install -U -P sonar -nsu --batch-mode -Dmaven.test.redirectTestOutputToFile=true -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
mkdir -p $HOME/artifacts/junit/
find . -type f -regex ".*/spring-cloud-*.*/target/*.*" -exec cp {} $HOME/artifacts/ \;
find . -type f -regex ".*/target/.*-reports/.*" -exec cp {} $HOME/artifacts/junit/ \;
bash <(curl -s https://codecov.io/bash)
- run:
name: Install Snap
command: |
sudo apt update
sudo apt install snapd
- run:
name: Launch Kubernetes with microk8s
command: |
sudo snap install microk8s --classic --channel=1.11/stable
# wait until a k8s node is ready
sleep 10
n=0
until [ $n -ge 10 ]
do
(/snap/bin/microk8s.kubectl get no | grep -z "Ready") && break
n=$[$n+1]
sleep 20
done
echo "Kubernetes cluster launched"
# Allow intra-pod communication
sudo iptables -P FORWARD ACCEPT
/snap/bin/microk8s.enable dns registry
# wait until the registry is up and running
sleep 10
n=0
until [ $n -ge 10 ]
do
(/snap/bin/microk8s.kubectl get pod --namespace=container-registry | grep -z "Running") && break
n=$[$n+1]
sleep 10
done
echo "Kubernetes Container Registry enabled"
- run:
name: Run integration tests
command: |
cd spring-cloud-kubernetes-integration-tests
/snap/bin/microk8s.kubectl config view --raw > /tmp/kubeconfig
./deploy_test_undeploy_all.sh
- store_test_results:
path: $HOME/artifacts/junit/
- store_artifacts:
path: $HOME/artifacts/
notify:
webhooks:
- url: https://webhooks.gitter.im/e/22e6bb4eb945dd61ba54

@@ -71,6 +71,7 @@
<fabric8.maven.plugin.version>3.5.37</fabric8.maven.plugin.version>
<gmavenplus-plugin.version>1.6</gmavenplus-plugin.version>
<groovy.version>2.4.12</groovy.version>
<restassured.version>3.0.2</restassured.version>
<spock-spring.version>1.1-groovy-2.4</spock-spring.version>
</properties>

@@ -86,12 +87,13 @@
<module>spring-cloud-starter-kubernetes-all</module>
<module>spring-cloud-kubernetes-examples</module>
<module>spring-cloud-kubernetes-leader</module>
<module>spring-cloud-kubernetes-integration-tests</module>
<module>docs</module>
</modules>

<dependencyManagement>
<dependencies>

<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-kubernetes-dependencies</artifactId>
@@ -0,0 +1,108 @@
# Purpose

Demonstrate how integration tests can be run against a local cluster setup with [microk8s](https://microk8s.io/)
The Kubernetes resources necessary to get the test application
onto the cluster are created using the [Fabric8 Maven Plugin](https://maven.fabric8.io/)

# Instructions

## Install microk8s

```bash
sudo snap install microk8s --classic --channel=1.11/stable
sleep 10
microk8s.enable dns registry
```

Ensure everything is running by inspecting the output of:

```bash
microk8s.kubectl get all --all-namespaces
```

It should look something like:

```
NAMESPACE NAME READY STATUS RESTARTS AGE
container-registry pod/registry-6bc95dfd76-274lc 1/1 Running 0 40s
kube-system pod/hostpath-provisioner-9979c7f64-f96tw 1/1 Running 0 41s
kube-system pod/kube-dns-864b8bdc77-68kmn 2/3 Running 0 41s
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
container-registry service/registry NodePort 10.152.183.175 <none> 5000:32000/TCP 50s
default service/kubernetes ClusterIP 10.152.183.1 <none> 443/TCP 1m
kube-system service/kube-dns ClusterIP 10.152.183.10 <none> 53/UDP,53/TCP 56s
NAMESPACE NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE
container-registry deployment.apps/registry 1 1 1 1 50s
kube-system deployment.apps/hostpath-provisioner 1 1 1 1 51s
kube-system deployment.apps/kube-dns 1 1 1 0 56s
NAMESPACE NAME DESIRED CURRENT READY AGE
container-registry replicaset.apps/registry-6bc95dfd76 1 1 1 41s
kube-system replicaset.apps/hostpath-provisioner-9979c7f64 1 1 1 41s
kube-system replicaset.apps/kube-dns-864b8bdc77 1 1 0 41s
```

## Setup environment for FMP to work

One premise for the Fabric8 Maven Plugin to work is that is can read the proper Kubernetes Config file.
We export the corresponding config file for the cluster microk8s sets up to temp file which will
be used later

```bash
microk8s.kubectl config view --raw > /tmp/kubeconfig
```

## Deploy application

**The following commands assume that they are executed inside the maven module of each specific test**

Since FMP is based on the Fabric8 Kubernetes Client, we can leverage the `KUBECONFIG` environment variable
to make FMP aware of the Kubernetes Config file we created above

```bash
export KUBECONFIG=/tmp/kubeconfig
../../mvnw clean package fabric8:build fabric8:push fabric8:deploy -Pfmp
```

Make sure the application was deployed be executing:

```bash
microk8s.kubectl get pod -l app=sb-fmp-microk8s
```

It should look something like:

```
NAME READY STATUS RESTARTS AGE
simple-core-5fbb7646dc-66t7b 1/1 Running 0 57s
```

The integration tests can be run against the service which is deployed inside the cluster by executing:

```bash
../../mvnw verify -Pfmp,it -Dfabric8.skip
```

This integration tests runs locally (making it easily debuggable), and interacts with the deployed service
using the exposed port.
By leveraging [Aquillian Cube Kubernetes](http://arquillian.org/arquillian-cube/#_kubernetes), it's able to setup and teardown resources needed for each test

To undeploy the application from the cluster simply execute:

```bash
../../mvnw fabric8:undeploy -Pfmp
```

## Run all tests

By executing

```bash
deploy_test_undeploy_all.sh
```

each one of the test applications will be deployed to the cluster and the integration tests will be executed


@@ -0,0 +1,13 @@
#!/usr/bin/env bash

set -e

# The script launches and tests each test application - one at a time

for D in $(find . -maxdepth 1 -mindepth 1 -not -path '*/\.*' -type d); do
pushd ${D} > /dev/null
../deploy_test_undeploy_single.sh
/snap/bin/microk8s.kubectl wait --for=delete pod -l group=org.springframework.cloud --timeout=60s > /dev/null
popd > /dev/null
done

@@ -0,0 +1,27 @@
#!/usr/bin/env bash

set -e

# The script assumes that there is only one application deployed by FMP running at a time

SCRIPT_ABSOLUTE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
mvnCmd="${SCRIPT_ABSOLUTE_DIR}"/../mvnw
kubectlCmd=/snap/bin/microk8s.kubectl

export KUBECONFIG=/tmp/kubeconfig

echo "Starting application deployment to cluster"
$mvnCmd clean package fabric8:build fabric8:push fabric8:deploy -Pfmp
echo "Finished deployment"

echo "Waiting for pod to become Ready"
# We are leveraging the fact the fact that FMP adds the group label and the fact that only one
# application can be running at a time
$kubectlCmd wait --for=condition=Ready pod -l group=org.springframework.cloud --timeout=60s > /dev/null

echo "Starting the integration tests"
$mvnCmd verify -Pfmp,it -Dfabric8.skip

echo "Successfully executed integration tests"
echo "Undeploying application"
$mvnCmd fabric8:undeploy -Pfmp
@@ -0,0 +1,140 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-kubernetes</artifactId>
<version>1.0.0.BUILD-SNAPSHOT</version>
</parent>

<artifactId>spring-cloud-kubernetes-integration-tests</artifactId>
<packaging>pom</packaging>

<name>Spring Cloud Kubernetes :: Integration Tests</name>
<description>Integration tests where SCK applications are run inside a Kubernetes cluster
</description>

<properties>
<java.version>1.8</java.version>
<fmp.version>3.5.42</fmp.version>
<arquillian-cube.version>1.18.2</arquillian-cube.version>
<arquillian.version>1.4.0.Final</arquillian.version>
<rest-assured.version>3.2.0</rest-assured.version>

<!-- We use the unix socket where microk8s' docker daemon listens -->
<docker.host>unix:///var/snap/microk8s/current/docker.sock</docker.host>

<!-- This is the default location of the container registry that comes with microk8s -->
<microk8s.registry>localhost:32000</microk8s.registry>

<!--
The port on localhost where the application will listen to from outside the cluster
Will be used to construct a NodePort
-->
<nodeport.value>32222</nodeport.value>
</properties>

<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>${maven-deploy-plugin.version}</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>

</plugins>
</build>

<profiles>
<profile>
<id>fmp</id>
<build>
<plugins>
<plugin>
<groupId>io.fabric8</groupId>
<artifactId>fabric8-maven-plugin</artifactId>
<version>${fmp.version}</version>
<configuration>
<!--
FMP can work with Openshift as well, so here we explicitly force it
to work with vanilla Kubernetes and Docker builds
-->
<buildStrategy>docker</buildStrategy>
<mode>kubernetes</mode>
<pushRegistry>${microk8s.registry}</pushRegistry>
</configuration>
<executions>
<execution>
<id>fabric8</id>
<goals>
<goal>resource</goal>
<goal>build</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>it</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<systemPropertyVariables>
<service.host>localhost</service.host>
<service.port>${nodeport.value}</service.port>
</systemPropertyVariables>
<classesDirectory>${project.build.outputDirectory}</classesDirectory>
</configuration>
<executions>
<execution>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>

<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.jboss.arquillian</groupId>
<artifactId>arquillian-bom</artifactId>
<version>${arquillian.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>io.rest-assured</groupId>
<artifactId>rest-assured</artifactId>
<version>${rest-assured.version}</version>
</dependency>
</dependencies>
</dependencyManagement>

<modules>
<module>simple-core</module>
<module>simple-configmap</module>
</modules>


</project>
Oops, something went wrong.

0 comments on commit 9f02f7e

Please sign in to comment.
You can’t perform that action at this time.