HTTP Management Proxy Implementation (#1356)

* New integration tests for HTTP management proxy. Add GHA workflow for new tests. Add Management API agent for http tests (#1347) * Separate JmxProxy interface from the JMX implementation (#1314) * Add new Management API client to Reaper (#1353) * Implement the HttpManagementProxy (#1326) --------- Co-authored-by: Erik Merkle <erik.merkle@datastax.com> Co-authored-by: Michael Burman <michael.burman@datastax.com> Co-authored-by: Olivier Michallat <olim7t@users.noreply.github.com> Co-authored-by: Alexander Dejanovski <adejanovski@gmail.com>
thelastpickle · Oct 16, 2023 · 8a26f9d · 8a26f9d
1 parent 7d4c0e3
commit 8a26f9d
Show file tree

Hide file tree

Showing 46 changed files with 29,279 additions and 876 deletions.
diff --git a/.github/scripts/configure-ccm.sh b/.github/scripts/configure-ccm.sh
@@ -67,11 +67,10 @@ case "${TEST_TYPE}" in
         echo "ERROR: Environment variable TEST_TYPE is unspecified."
         exit 1
         ;;
-    "ccm"|"upgrade"|"elassandra")
+    "ccm"|"upgrade"|"elassandra"|"http-api")
         mkdir -p ~/.local
         cp ./.github/files/jmxremote.password ~/.local/jmxremote.password
         chmod 400 ~/.local/jmxremote.password
-        ls -lrt /opt/hostedtoolcache/
         for jdk in /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk/8*/; 
         do 
           sudo chmod 777 "${jdk/}"x64/jre/lib/management/jmxremote.access
@@ -116,5 +115,3 @@ case "${TEST_TYPE}" in
     *)
         echo "Skipping, no actions for TEST_TYPE=${TEST_TYPE}."
 esac
-
-
diff --git a/.github/scripts/run-tests.sh b/.github/scripts/run-tests.sh
@@ -29,6 +29,27 @@ function set_java_home() {
     done
 }
 
+add_management_api () {
+   if [[ ! -L /tmp/datastax-mgmtapi-agent.jar ]]; then
+     # Do some fancy pom.xml parsing to figure out which version of the Management API client we are using
+     MGMT_API_VERSION=`mvn dependency:tree -f src/server/pom.xml |grep datastax-mgmtapi-client-openapi|cut -d ":" -f 4`
+     # Download the Management API bundle
+     mvn dependency:copy -Dartifact=io.k8ssandra:datastax-mgmtapi-server:$MGMT_API_VERSION -f src/server/pom.xml -DoutputDirectory=/tmp -Dmdep.stripVersion=true -Dmdep.overWriteReleases=true
+     # Unzip the agent for the version of Cassandra
+     if [[ "$CASSANDRA_VERSION" == *"3.11"* ]]; then
+        mvn dependency:copy -Dartifact=io.k8ssandra:datastax-mgmtapi-agent-3.x:$MGMT_API_VERSION -f src/server/pom.xml -DoutputDirectory=/tmp -Dmdep.stripVersion=true -Dmdep.overWriteReleases=true
+        ln -s /tmp/datastax-mgmtapi-agent-3.x.jar /tmp/datastax-mgmtapi-agent.jar
+     elif [[ "$CASSANDRA_VERSION" == *"4.0"* ]]; then
+        mvn dependency:copy -Dartifact=io.k8ssandra:datastax-mgmtapi-agent-4.x:$MGMT_API_VERSION -f src/server/pom.xml -DoutputDirectory=/tmp -Dmdep.stripVersion=true -Dmdep.overWriteReleases=true
+        ln -s /tmp/datastax-mgmtapi-agent-4.x.jar /tmp/datastax-mgmtapi-agent.jar
+     elif [[ "$CASSANDRA_VERSION" == *"4.1"* ]]; then
+        mvn dependency:copy -Dartifact=io.k8ssandra:datastax-mgmtapi-agent-4.1.x:$MGMT_API_VERSION -f src/server/pom.xml -DoutputDirectory=/tmp -Dmdep.stripVersion=true -Dmdep.overWriteReleases=true
+        ln -s /tmp/datastax-mgmtapi-agent-4.1.x.jar /tmp/datastax-mgmtapi-agent.jar
+     fi
+   fi
+  echo "JVM_OPTS=\"\$JVM_OPTS -javaagent:/tmp/datastax-mgmtapi-agent.jar\"" >> ~/.ccm/test/node$1/conf/cassandra-env.sh
+}
+
 case "${TEST_TYPE}" in
     "")
         echo "ERROR: Environment variable TEST_TYPE is unspecified."
@@ -80,6 +101,57 @@ case "${TEST_TYPE}" in
         esac
 
         ;;
+    "http-api")
+            mvn --version -B
+            ps uax | grep cass
+            # dependending on the version of cassandra, we may need to use a different jdk
+            set_java_home ${JDK_VERSION}
+            # Add in  Management API agent jarfile
+            for i in `seq 1 2` ; do
+              add_management_api $i
+              mkdir -p /tmp/log/cassandra$i/ && touch /tmp/log/cassandra$i/stdout.log
+            done
+            ccm start -v --no-wait --skip-wait-other-notice || true
+            echo "${TEST_TYPE}" | grep -q ccm && sleep 30 || sleep 120
+            ccm status
+            ccm node1 nodetool -- -u cassandra -pw cassandrapassword status
+            # Stop CCM now so we can restart it with Management API
+            ccm stop
+            # Start Management API
+            MGMT_API_LOG_DIR=/tmp/log/cassandra1 bash -c 'nohup java -jar /tmp/datastax-mgmtapi-server.jar --db-socket=/tmp/db1.sock --host=unix:///tmp/mgmtapi1.sock --host=http://127.0.0.1:8080 --db-home=`dirname ~/.ccm/test/node1`/node1 &'
+            MGMT_API_LOG_DIR=/tmp/log/cassandra2 bash -c 'nohup java -jar /tmp/datastax-mgmtapi-server.jar --db-socket=/tmp/db2.sock --host=unix:///tmp/mgmtapi2.sock --host=http://127.0.0.2:8080 --db-home=`dirname ~/.ccm/test/node2`/node2 &'
+            # wait for Cassandra to be ready
+            for i in `seq 1 30` ; do
+                # keep curl from exiting with non-zero
+                HTTPCODE1=`curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8080/api/v0/probes/readiness` || true
+                HTTPCODE2=`curl -s -o /dev/null -w "%{http_code}" http://127.0.0.2:8080/api/v0/probes/readiness` || true
+                if [ "${HTTPCODE1}" != "200" -o "${HTTPCODE2}" != "200" ]
+                then
+                    echo "Cassandra not ready yet. Sleeping.... $i"
+                else
+                    echo "Cassandra started via Management API successfully"
+                    break
+                fi
+                sleep 5
+            done
+            # Reaper requires JDK11 for compilation
+            set_java_home 11
+            case "${STORAGE_TYPE}" in
+                "")
+                    echo "ERROR: Environment variable STORAGE_TYPE is unspecified."
+                    exit 1
+                    ;;
+                "ccm")
+                    mvn -B package -DskipTests
+                    ccm node1 cqlsh -e "DROP KEYSPACE reaper_db" || true
+                    mvn -B org.jacoco:jacoco-maven-plugin:${JACOCO_VERSION}:prepare-agent surefire:test -DsurefireArgLine="-Xmx256m"  -Dtest=ReaperHttpIT -Dcucumber.options="$CUCUMBER_OPTIONS" org.jacoco:jacoco-maven-plugin:${JACOCO_VERSION}:report
+                    ;;
+                *)
+                    echo "Skipping, no actions for STORAGE_TYPE=${STORAGE_TYPE}."
+                    ;;
+            esac
+
+            ;;
     "sidecar")
         mvn --version -B
         mvn -B package -DskipTests

diff --git a/.github/workflows/check_pr_linked_issue.yaml b/.github/workflows/check_pr_linked_issue.yaml
@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Check linked issues
     steps:
-      - uses: nearform/github-action-check-linked-issues@v1
+      - uses: nearform-actions/github-action-check-linked-issues@v1
         continue-on-error: true
         id: check-linked-issues
         with:

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -78,36 +78,20 @@ jobs:
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       matrix:
-        cassandra-version: [ "binary:1.2.19", "binary:2.0.17", 'binary:2.1.22', 'binary:2.2.19', 'binary:3.0.25', 'binary:3.11.11', 'binary:4.0.1' ]
+        cassandra-version: [ 'binary:3.0.25', 'binary:3.11.16', 'binary:4.0.6' ]
         storage-type: [local]
         test-type: [ccm]
         include:
           # don't run against the following C* versions when using cassandra storage type
-          - cassandra-version: "binary:1.2.19"
-            cucumber-options: '--tags ~@cassandra_2_1_onwards --tags ~@cassandra_4_0_onwards --tags ~@cassandra_3_11_onwards'
-            experimental: false
-            jdk-version: '8'
-          - cassandra-version: "binary:2.0.17"
-            cucumber-options: '--tags ~@cassandra_2_1_onwards --tags ~@cassandra_4_0_onwards --tags ~@cassandra_3_11_onwards'
-            experimental: false
-            jdk-version: '8'
-          - cassandra-version: 'binary:2.1.22'
-            cucumber-options: '--tags ~@cassandra_4_0_onwards --tags ~@cassandra_3_11_onwards'
-            experimental: false
-            jdk-version: '8'
-          - cassandra-version: 'binary:2.2.19'
-            cucumber-options: '--tags ~@cassandra_4_0_onwards --tags ~@cassandra_3_11_onwards'
-            experimental: false
-            jdk-version: '8'
           - cassandra-version: 'binary:3.0.25'
             cucumber-options: '--tags ~@cassandra_4_0_onwards --tags ~@cassandra_3_11_onwards'
             experimental: false
             jdk-version: '8'
-          - cassandra-version: 'binary:3.11.11'
+          - cassandra-version: 'binary:3.11.16'
             cucumber-options: '--tags ~@cassandra_4_0_onwards'
             experimental: false
             jdk-version: '8'
-          - cassandra-version: 'binary:4.0.1'
+          - cassandra-version: 'binary:4.0.6'
             experimental: false
             jdk-version: '11'
           #- cassandra-version: 'github:apache/trunk'
@@ -156,19 +140,19 @@ jobs:
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       matrix:
-        cassandra-version: [ 'binary:3.0.25', 'binary:3.11.11', 'binary:4.0.1']
+        cassandra-version: [ 'binary:3.0.25', 'binary:3.11.16', 'binary:4.0.6']
         storage-type: [cassandra]
         test-type: [ccm]
         include:
           - cassandra-version: 'binary:3.0.25'
             cucumber-options: '--tags ~@cassandra_4_0_onwards --tags ~@cassandra_3_11_onwards'
             experimental: false
             jdk-version: '8'
-          - cassandra-version: 'binary:3.11.11'
+          - cassandra-version: 'binary:3.11.16'
             cucumber-options: '--tags ~@cassandra_4_0_onwards'
             experimental: false
             jdk-version: '8'
-          - cassandra-version: 'binary:4.0.1'
+          - cassandra-version: 'binary:4.0.6'
             experimental: false
             jdk-version: '11'
           #- cassandra-version: 'github:apache/trunk'
@@ -217,7 +201,7 @@ jobs:
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       matrix:
-        cassandra-version: ['binary:3.0.25', 'binary:3.11.11', 'binary:4.0.1']
+        cassandra-version: ['binary:3.0.25', 'binary:3.11.16', 'binary:4.0.6']
         storage-type: [cassandra]
         test-type: [sidecar]
         grim-max: [1]
@@ -228,11 +212,11 @@ jobs:
             cucumber-options: '--tags ~@cassandra_4_0_onwards --tags @sidecar --tags ~@cassandra_3_11_onwards'
             experimental: false
             jdk-version: '8'
-          - cassandra-version: 'binary:3.11.11'
+          - cassandra-version: 'binary:3.11.16'
             cucumber-options: '--tags ~@cassandra_4_0_onwards --tags @sidecar'
             experimental: false
             jdk-version: '8'
-          - cassandra-version: 'binary:4.0.1'
+          - cassandra-version: 'binary:4.0.6'
             cucumber-options: '--tags @sidecar'
             experimental: false
             jdk-version: '11'
@@ -273,21 +257,86 @@ jobs:
 
       - uses: codecov/codecov-action@v1
 
+  its-http-management:
+    needs: [ build ]
+    name: HTTP Management
+    runs-on: ubuntu-20.04
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      matrix:
+        cassandra-version: [ 'binary:4.0.6', 'binary:3.11.16' ]
+        storage-type: [ "ccm" ]
+        test-type: [ "http-api" ]
+        grim-max: [ 1 ]
+        grim-min: [ 1 ]
+        # all versions but trunk have the same cucumber options, but we can't declare that more effectively (yet)
+        include:
+          - cassandra-version: 'binary:3.11.16'
+            cucumber-options: '--tags ~@cassandra_4_0_onwards --tags @http_management'
+            experimental: false
+            jdk-version: '8'
+          - cassandra-version: 'binary:4.0.6'
+            cucumber-options: '--tags @http_management'
+            experimental: false
+            jdk-version: '11'
+          - cassandra-version: 'binary:3.11.16'
+            cucumber-options: '--tags ~@cassandra_4_0_onwards'
+            experimental: false
+            jdk-version: '8'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup
+        uses: ./.github/actions/setup
+
+      - name: Install CCM
+        run: |
+          pip install pyyaml
+          pip install ccm
+
+      - name: Setup CCM Cluster
+        run: ./.github/scripts/configure-ccm.sh
+        env:
+          TEST_TYPE: ${{ matrix.test-type }}
+          CASSANDRA_VERSION: ${{ matrix.cassandra-version }}
+          STORAGE_TYPE: ${{ matrix.storage-type }}
+          JDK_VERSION: ${{ matrix.jdk-version }}
+
+      - name: Run Tests
+        run: ./.github/scripts/run-tests.sh
+        env:
+          TEST_TYPE: ${{ matrix.test-type }}
+          CASSANDRA_VERSION: ${{ matrix.cassandra-version }}
+          STORAGE_TYPE: ${{ matrix.storage-type }}
+          GRIM_MAX: ${{ matrix.grim-max }}
+          GRIM_MIN: ${{ matrix.grim-min }}
+          CUCUMBER_OPTIONS: ${{ matrix.cucumber-options }}
+          JDK_VERSION: ${{ matrix.jdk-version }}
+
+      # Add a tmate action to debug failures. 
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
+        timeout-minutes: 30
+
+      - uses: codecov/codecov-action@v1
+
   its-each:
     needs: [docker-tests, its-ccm-local, its-ccm-cass]
     name: Each DC Availability
     runs-on: ubuntu-20.04
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       matrix:
-        cassandra-version: ['binary:4.0.1']
+        cassandra-version: ['binary:4.0.6']
         storage-type: [cassandra]
         test-type: [each]
         grim-max: [1]
         grim-min: [1]
         # Reduced scope to shorten the integration test times
         include:
-          - cassandra-version: 'binary:4.0.1'
+          - cassandra-version: 'binary:4.0.6'
             cucumber-options: '--tags @sidecar'
             experimental: false
           #- cassandra-version: 'github:apache/trunk'
@@ -333,7 +382,7 @@ jobs:
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       matrix:
-        cassandra-version: ['binary:3.0.25', 'binary:3.11.11', 'binary:4.0.1']
+        cassandra-version: ['binary:3.0.25', 'binary:3.11.16', 'binary:4.0.6']
         storage-type: [cassandra]
         test-type: [ccm]
         grim-max: [2]
@@ -343,11 +392,11 @@ jobs:
             cucumber-options: '--tags ~@cassandra_4_0_onwards --tags ~@cassandra_3_11_onwards'
             experimental: false
             jdk-version: '8'
-          - cassandra-version: 'binary:3.11.11'
+          - cassandra-version: 'binary:3.11.16'
             cucumber-options: '--tags ~@cassandra_4_0_onwards'
             experimental: false
             jdk-version: '8'
-          - cassandra-version: 'binary:4.0.1'
+          - cassandra-version: 'binary:4.0.6'
             experimental: false
             jdk-version: '11'
           #- cassandra-version: 'github:apache/trunk'
@@ -386,23 +435,22 @@ jobs:
 
       - uses: codecov/codecov-action@v1
 
-
   its-flapping:
     needs: [docker-tests, its-ccm-local, its-ccm-cass]
     name: Flapping reapers
     runs-on: ubuntu-20.04
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       matrix:
-        cassandra-version: ['binary:4.0.1']
+        cassandra-version: ['binary:4.0.6']
         storage-type: [cassandra]
         test-type: [ccm]
         grim-max: [4]
         grim-min: [2]
         cucumber-options: ['--tags ~@diag_event_subscriptions']
         include:
           # all versions but trunk need to exclude trunk tests
-          - cassandra-version: 'binary:4.0.1'
+          - cassandra-version: 'binary:4.0.6'
             experimental: true
             jdk-version: '11'
           #- cassandra-version: 'github:apache/trunk'
@@ -585,4 +633,4 @@ jobs:
           distro: 'any-distro'
           release: 'any-version'
           republish: 'true'
-          file: ${{ env.RPM_RELEASE_FILE }}
+          file: ${{ env.RPM_RELEASE_FILE }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,9 @@
 ## Change Log
 
+
+### Unreleased
+- [#1326] Epic to cover implementation of a new Reaper HTTP management proxy, to be used as an alternative to the current JMX implementation.
+
 ### 3.3.4 (2023/10/05 15:30 +00:00)
 - [#1418](https://github.com/thelastpickle/cassandra-reaper/pull/1418) Build arm64 images (#1418) (@adejanovski)