CrossDC functional test suite automation (#589)

* initial work Signed-off-by: Martin Kanis <mkanis@redhat.com> * Stretch attempt to add an GHA to run the CrossDC functional tests Signed-off-by: Kamesh Akella <kamesh.asp@gmail.com> --------- Signed-off-by: Martin Kanis <mkanis@redhat.com> Signed-off-by: Kamesh Akella <kamesh.asp@gmail.com> Co-authored-by: Kamesh Akella <kakella@redhat.com> Co-authored-by: TTomáš Kyjovský <tkyjovsk@redhat.com@redhat.com>
keycloak · Nov 28, 2023 · d6895df · d6895df
1 parent 06f22fa
commit d6895df
Show file tree

Hide file tree

Showing 12 changed files with 735 additions and 29 deletions.
diff --git a/.github/workflows/rosa-run-crossdc-func-tests.yml b/.github/workflows/rosa-run-crossdc-func-tests.yml
@@ -0,0 +1,68 @@
+name: Run CrossDC functional tests on ROSA cluster
+
+on:
+  workflow_dispatch:
+    inputs:
+      clusterPrefix:
+        description: 'The prefix used when creating the Cross DC clusters'
+        type: string
+
+concurrency:
+  # Only run once for the latest commit per ref and cancel other (previous) runs.
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '21'
+          cache: 'maven'
+      - name: Cache Maven Wrapper
+        uses: actions/cache@v3
+        with:
+          path: |
+            .mvn/wrapper/maven-wrapper.jar
+          key: ${{ runner.os }}-maven-wrapper-${{ hashFiles('**/maven-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-wrapper-
+      - name: Setup ROSA CLI
+        uses: ./.github/actions/rosa-cli-setup
+        with:
+          aws-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-default-region: ${{ vars.AWS_DEFAULT_REGION }}
+          rosa-token: ${{ secrets.ROSA_TOKEN }}
+      - name: Login to OpenShift cluster A
+        uses: ./.github/actions/oc-keycloak-login
+        with:
+          clusterName: ${{ inputs.clusterPrefix }}-a
+      - name: Get DC1 URLs
+        shell: bash
+        run: |
+          KEYCLOAK_DC1_URL=https://$(kubectl get routes -n "${{ inputs.project }}" -l app=keycloak -o jsonpath='{.items[*].spec.host}')
+          echo "KEYCLOAK_DC1_URL=$KEYCLOAK_DC1_URL" >> "$GITHUB_ENV"
+          ISPN_DC1_URL=https://$(kubectl get routes -n "${{ inputs.project }}" -l app=infinispan-service-external -o jsonpath='{.items[*].spec.host}')
+          echo "ISPN_DC1_URL=$ISPN_DC1_URL" >> "$GITHUB_ENV"
+      - name: Login to OpenShift cluster B
+        uses: ./.github/actions/oc-keycloak-login
+        with:
+          clusterName: ${{ inputs.clusterPrefix }}-b
+      - name: Get DC2 URLs
+        shell: bash
+        run: |
+          KEYCLOAK_DC2_URL=https://$(kubectl get routes -n "${{ inputs.project }}" -l app=keycloak -o jsonpath='{.items[*].spec.host}')
+          echo "KEYCLOAK_DC2_URL=$KEYCLOAK_DC1_URL" >> "$GITHUB_ENV"
+          ISPN_DC2_URL=https://$(kubectl get routes -n "${{ inputs.project }}" -l app=infinispan-service-external -o jsonpath='{.items[*].spec.host}')
+          echo "ISPN_DC2_URL=$ISPN_DC1_URL" >> "$GITHUB_ENV"
+      - name: Run CrossDC functional tests
+        working-directory: provision/rosa-cross-dc/keycloak-benchmark-crossdc-tests
+        run: |
+          ./run-crossdc-tests.sh
diff --git a/dataset/pom.xml b/dataset/pom.xml
@@ -40,8 +40,8 @@
             <scope>provided</scope>
         </dependency>
         <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
             <scope>test</scope>
         </dependency>
         <dependency>

diff --git a/dataset/src/test/java/org/keycloak/benchmark/it/DeploymentIT.java b/dataset/src/test/java/org/keycloak/benchmark/it/DeploymentIT.java
@@ -20,10 +20,8 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.commons.io.FileUtils;
 import org.awaitility.Awaitility;
-import org.junit.Assert;
-import org.junit.Assume;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 import org.keycloak.benchmark.dataset.TaskResponse;
 
 import java.io.IOException;
@@ -49,6 +47,11 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
+
 /**
  * This tests the deployment of the dataset provider in an integration test.
  * To do this, it expects Keycloak to be extracted in the target folder by the maven-dependency-plugin.
@@ -74,7 +77,7 @@ public DeploymentIT() {
         keycloak = "http://127.0.0.1:" + port + "/";
     }
 
-    @Before
+    @BeforeEach
     public void setup() throws IOException {
         // use a random free port to allow running the tests while other instances of Keycloak run at the same time
         ServerSocket s = new ServerSocket(0);
@@ -111,14 +114,14 @@ private void executeDatasetCommand(String command) throws URISyntaxException, IO
                 .timeout(Duration.of(10, ChronoUnit.SECONDS))
                 .build();
         HttpResponse<String> clearStatusResponse = httpClient.send(clearStatus, HttpResponse.BodyHandlers.ofString());
-        Assert.assertTrue(clearStatusResponse.statusCode() == 204 || clearStatusResponse.statusCode() == 404);
+        assertTrue(clearStatusResponse.statusCode() == 204 || clearStatusResponse.statusCode() == 404);
 
         HttpRequest masterStatus = HttpRequest.newBuilder(new URI(keycloak + "realms/master/dataset/" + command))
                 .GET()
                 .timeout(Duration.of(10, ChronoUnit.SECONDS))
                 .build();
         HttpResponse<String> healthStatus = httpClient.send(masterStatus, HttpResponse.BodyHandlers.ofString());
-        Assert.assertEquals(healthStatus.statusCode(), 200);
+        assertEquals(healthStatus.statusCode(), 200);
     }
 
     private void waitForDatasetCompleted() throws URISyntaxException {
@@ -158,7 +161,7 @@ private void stopKeycloak(Process process) throws ExecutionException, Interrupte
             ProcessBuilder processBuilder = new ProcessBuilder("taskkill", "/F", "/T", "/PID", String.valueOf(process.pid()));
             Process killProcess = processBuilder.start();
             int exitCodeKill = killProcess.waitFor();
-            Assert.assertEquals(0, exitCodeKill);
+            assertEquals(0, exitCodeKill);
         }
         process.destroy();
         processCompletableFuture.get();
@@ -173,13 +176,13 @@ private Process startKeycloak(Path keycloakProvidersFolder, String[] args) throw
         List<String> cli = new ArrayList<>();
         if (isWindows()) {
             Path executable = keycloakProvidersFolder.resolve("..").resolve("bin").resolve("kc.bat").normalize();
-            Assert.assertTrue(executable.toFile().exists());
+            assertTrue(executable.toFile().exists());
             cli.add("cmd.exe");
             cli.add("/c");
             cli.add(executable.toString().replaceAll("/", "\\"));
         } else {
             Path executable = keycloakProvidersFolder.resolve("..").resolve("bin").resolve("kc.sh").normalize();
-            Assert.assertTrue(executable.toFile().exists());
+            assertTrue(executable.toFile().exists());
             cli.add(executable.toString());
         }
         cli.addAll(Arrays.asList("--verbose", "start-dev", "--http-port", Integer.toString(port)));
@@ -191,7 +194,7 @@ private Process startKeycloak(Path keycloakProvidersFolder, String[] args) throw
                 .redirectErrorStream(true) // redirect error stream to output stream
                 .redirectOutput(ProcessBuilder.Redirect.INHERIT);
         Process keycloak = processBuilder.start();
-        Assert.assertTrue("keycloak should be running", keycloak.isAlive());
+        assertTrue(keycloak.isAlive(), "keycloak should be running");
         return keycloak;
     }
 
@@ -213,7 +216,7 @@ private Path getKeycloakDatasetProviderJar() throws IOException {
         try (Stream<Path> files = Files.list(target).filter(path -> path.getFileName().toString().startsWith("keycloak-benchmark-dataset") && path.getFileName().toString().endsWith(".jar"))) {
             Iterator<Path> iterator = files.iterator();
             Path provider = iterator.next();
-            Assume.assumeFalse("should only have one sibling", iterator.hasNext());
+            assumeFalse(iterator.hasNext(), "should only have one sibling");
             return provider;
         }
     }
@@ -223,9 +226,9 @@ private Path getKeycloakProvidersFolder() throws IOException {
         try (Stream<Path> files = Files.list(keycloakExtracted)) {
             Iterator<Path> iterator = files.iterator();
             Path keycloak = iterator.next();
-            Assume.assumeFalse("should only have one sibling", iterator.hasNext());
+            assumeFalse(iterator.hasNext(), "should only have one sibling");
             Path providers = keycloak.resolve("providers");
-            Assume.assumeNotNull(providers);
+            assertNotNull(providers);
             return providers;
         }
     }
@@ -235,7 +238,7 @@ private Path getKeycloakProvidersFolder() throws IOException {
      * Only used when running it via the main method / from the IDE.
      */
     private static void packageProvider() throws IOException, InterruptedException {
-        Assert.assertTrue("should be run from the module path 'dataset'", Path.of("../mvnw").toFile().exists());
+        assertTrue(Path.of("../mvnw").toFile().exists(), "should be run from the module path 'dataset'");
         List<String> cli = new ArrayList<>();
         if (isWindows()) {
             cli.addAll(Arrays.asList("cmd", "/c", "mvnw.cmd"));
@@ -250,7 +253,7 @@ private static void packageProvider() throws IOException, InterruptedException {
                 .redirectOutput(ProcessBuilder.Redirect.INHERIT);
         Process keycloak = processBuilder.start();
         keycloak.waitFor();
-        Assert.assertEquals(keycloak.exitValue(), 0);
+        assertEquals( 0, keycloak.exitValue());
     }
 
     private void startService(String[] args) throws IOException, ExecutionException, InterruptedException, URISyntaxException {

diff --git a/dataset/src/test/java/org/keycloak/benchmark/test/ExecutorTest.java b/dataset/src/test/java/org/keycloak/benchmark/test/ExecutorTest.java
@@ -18,23 +18,23 @@
 
 package org.keycloak.benchmark.test;
 
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
-import org.junit.Ignore;
-import org.junit.Test;
-
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
 public class ExecutorTest {
 
     @Test
-    @Ignore
+    @Disabled
     public void testExecutor() throws Exception {
         ExecutorService executor = Executors.newFixedThreadPool(5);
 

diff --git a/dataset/src/test/java/org/keycloak/benchmark/test/FinderTest.java b/dataset/src/test/java/org/keycloak/benchmark/test/FinderTest.java
@@ -20,10 +20,11 @@
 
 import java.util.function.Function;
 
-import org.junit.Assert;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 import org.keycloak.benchmark.dataset.config.ConfigUtil;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
@@ -41,7 +42,7 @@ public void testFinder() {
 
     private void assertFinder(int index) {
         SimpleFinder simpleFinder = new SimpleFinder(index);
-        Assert.assertEquals(index + 1, ConfigUtil.findFreeEntityIndex(simpleFinder));
+        assertEquals(index + 1, ConfigUtil.findFreeEntityIndex(simpleFinder));
     }
 
 

diff --git a/pom.xml b/pom.xml
@@ -11,7 +11,9 @@
 
   <properties>
     <keycloak.version>999.0.0-SNAPSHOT</keycloak.version>
-    <junit.version>4.13.2</junit.version>
+    <junit5.version>5.9.3</junit5.version>
+    <httpclient.version>4.5.14</httpclient.version>
+    <maven.surefire.plugin.version>3.1.2</maven.surefire.plugin.version>
     <maven.compiler.source>11</maven.compiler.source>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <maven.compiler.target>11</maven.compiler.target>
@@ -23,6 +25,7 @@
     <module>dataset</module>
     <module>benchmark</module>
     <module>provision/tlsdisableagent/java-instrumentation-tool</module>
+    <module>provision/rosa-cross-dc/keycloak-benchmark-crossdc-tests</module>
   </modules>
 
   <dependencyManagement>
@@ -58,9 +61,20 @@
         <version>${keycloak.version}</version>
       </dependency>
       <dependency>
-        <groupId>junit</groupId>
-        <artifactId>junit</artifactId>
-        <version>${junit.version}</version>
+        <groupId>org.keycloak</groupId>
+        <artifactId>keycloak-cli</artifactId>
+        <version>${keycloak.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.junit.jupiter</groupId>
+        <artifactId>junit-jupiter-engine</artifactId>
+        <version>${junit5.version}</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.httpcomponents</groupId>
+        <artifactId>httpclient</artifactId>
+        <version>${httpclient.version}</version>
         <scope>test</scope>
       </dependency>
     </dependencies>
@@ -97,4 +111,14 @@
     </repository>
   </repositories>
 
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>${maven.surefire.plugin.version}</version>
+      </plugin>
+    </plugins>
+  </build>
+
 </project>
diff --git a/provision/infinispan/ispn-helm/templates/infinispan.yaml b/provision/infinispan/ispn-helm/templates/infinispan.yaml
@@ -24,6 +24,17 @@ data:
           namesAsTags: true
           gauges: true
           histograms: {{ .Values.metrics.histograms }}
+      server:
+        endpoints:
+          - securityRealm: default
+            socketBinding: default
+            connectors:
+              rest:
+                restConnector:
+                  authentication:
+                    mechanisms: BASIC
+              hotrod:
+                hotrodConnector: null
 ---
 # tag::infinispan-crossdc[]
 # tag::infinispan-single[]

diff --git a/provision/rosa-cross-dc/keycloak-benchmark-crossdc-tests/README.md b/provision/rosa-cross-dc/keycloak-benchmark-crossdc-tests/README.md
@@ -0,0 +1,35 @@
+### Motivation
+
+We wanted to validate the Cross Data Center functionality of Keycloak in an actual cross data center setup for Keycloak application. As part of that we created a test suite that is currently hosted in the keycloak-benchmark and in future would move to a more appropriate place.
+
+### CrossDC Test Framework
+
+The current framework is made up of the below components
+
+- **Testsuite root directory**: keycloak-benchmark/provision/rosa-cross-dc
+- **Test Runner**: JUnit5.
+- **Test Data**: Use [Keycloak Dataset provider](https://www.keycloak.org/keycloak-benchmark/dataset-guide/latest/).
+- **Cache Metrics**: [ISPN Http REST client](https://infinispan.org/docs/stable/titles/rest/rest.html) to access Cache stats for external ISPN server. And for embedded infinispan cache we are relying on [Dataset provider](https://www.keycloak.org/keycloak-benchmark/dataset-guide/latest/).
+- **Execution Target**: A CrossDC cluster with access to two Keycloak and Infinispan datacenter urls.
+
+<br/> Note: We will use the existing ROSA OCP cluster based deployment during development to bring up the cross-dc cluster.
+
+### How to run
+
+From the Testsuite root directory run the below command to run the tests
+
+```
+mvn clean install -DcrossDCTests \
+-Dinfinispan.dc1.url=<ISPN_DC1_URL> -Dkeycloak.dc1.url=<KEYCLOAK_DC1_URL> \
+-Dinfinispan.dc2.url=<ISPN_DC2_URL> -Dkeycloak.dc2.url=<KEYCLOAK_DC2_URL>\
+-Dinfinispan.password=<ISPN_PASSWORD>
+```
+
+Alternatively could use the `run-crossdc-tests.sh` (located in the Testsuite root) directory to execute the tests when using a ROSA style provisioning setup to fetch the `ISPN_PASSWORD` on the fly, or by setting it manually.
+
+Example usage:
+```
+ISPN_DC1_URL=<ISPN_DC1_URL> ISPN_DC2_URL=<ISPN_DC2_URL> \
+KEYCLOAK_DC1_URL=<KEYCLOAK_DC1_URL> KEYCLOAK_DC2_URL=<KEYCLOAK_DC2_URL> \
+./run-crossdc-tests.sh
+```