Skip to content

Subrange incremental repair #1609

Subrange incremental repair

Subrange incremental repair #1609

Workflow file for this run

# Copyright 2014-2016 Spotify AB
# Copyright 2016-2019 The Last Pickle Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: CI
on:
push:
branches:
- master
pull_request:
release:
types:
- published
jobs:
build:
name: Build Reaper
runs-on: ubuntu-20.04
env:
JACOCO_VERSION: 0.8.6
REAPER_ENCRYPTION_KEY: SECRET_KEY
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup
uses: ./.github/actions/setup
- name: Build Reaper
run: |
MAVEN_OPTS="-Xmx384m" mvn -B org.jacoco:jacoco-maven-plugin:${{ env.JACOCO_VERSION }}:prepare-agent install org.jacoco:jacoco-maven-plugin:${{ env.JACOCO_VERSION }}:report
mv src/server/target/site/jacoco src/server/target/site/jacoco_ut
mvn -B org.jacoco:jacoco-maven-plugin:${{ env.JACOCO_VERSION }}:prepare-agent surefire:test org.jacoco:jacoco-maven-plugin:${{ env.JACOCO_VERSION }}:report -Dtest=ReaperShiroIT
- uses: codecov/codecov-action@v1
docker-tests:
name: Docker Image Tests
needs: build
runs-on: ubuntu-20.04
strategy:
matrix:
storage-type: [cassandra]
test-type: [docker]
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup
uses: ./.github/actions/setup
- name: Run Tests
run: ./.github/scripts/run-tests.sh
env:
TEST_TYPE: ${{ matrix.test-type }}
- name: Setup tmate session
if: "failure()"
uses: mxschmitt/action-tmate@v3
timeout-minutes: 30
its-ccm-local:
name: Memory backends
needs: build
runs-on: ubuntu-20.04
continue-on-error: ${{ matrix.experimental }}
strategy:
matrix:
cassandra-version: [ 'binary:3.0.25', 'binary:3.11.16', 'binary:4.0.6' ]
storage-type: [local]
test-type: [ccm]
include:
# don't run against the following C* versions when using cassandra storage type
- cassandra-version: 'binary:3.0.25'
cucumber-options: '--tags ~@cassandra_4_0_onwards --tags ~@cassandra_3_11_onwards'
experimental: false
jdk-version: '8'
- cassandra-version: 'binary:3.11.16'
cucumber-options: '--tags ~@cassandra_4_0_onwards'
experimental: false
jdk-version: '8'
- cassandra-version: 'binary:4.0.6'
experimental: false
jdk-version: '11'
#- cassandra-version: 'github:apache/trunk'
# experimental: true
# jdk-version: '11'
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup
uses: ./.github/actions/setup
- name: Install CCM
run: |
pip install pyyaml
pip install ccm
pip install setuptools
- name: Setup CCM Cluster
run: ./.github/scripts/configure-ccm.sh
env:
TEST_TYPE: ${{ matrix.test-type }}
CASSANDRA_VERSION: ${{ matrix.cassandra-version }}
STORAGE_TYPE: ${{ matrix.storage-type }}
JDK_VERSION: ${{ matrix.jdk-version }}
- name: Run Tests
run: ./.github/scripts/run-tests.sh
env:
TEST_TYPE: ${{ matrix.test-type }}
CASSANDRA_VERSION: ${{ matrix.cassandra-version }}
STORAGE_TYPE: ${{ matrix.storage-type }}
CUCUMBER_OPTIONS: ${{ matrix.cucumber-options }}
JDK_VERSION: ${{ matrix.jdk-version }}
- uses: codecov/codecov-action@v1
- name: Setup tmate session
if: "failure()"
uses: mxschmitt/action-tmate@v3
timeout-minutes: 30
its-ccm-cass:
needs: build
name: Cassandra backend
runs-on: ubuntu-20.04
continue-on-error: ${{ matrix.experimental }}
strategy:
matrix:
cassandra-version: [ 'binary:3.0.25', 'binary:3.11.16', 'binary:4.0.6']
storage-type: [cassandra]
test-type: [ccm]
include:
- cassandra-version: 'binary:3.0.25'
cucumber-options: '--tags ~@cassandra_4_0_onwards --tags ~@cassandra_3_11_onwards'
experimental: false
jdk-version: '8'
- cassandra-version: 'binary:3.11.16'
cucumber-options: '--tags ~@cassandra_4_0_onwards'
experimental: false
jdk-version: '8'
- cassandra-version: 'binary:4.0.6'
experimental: false
jdk-version: '11'
#- cassandra-version: 'github:apache/trunk'
# experimental: true
# jdk-version: '11'
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup
uses: ./.github/actions/setup
- name: Install CCM
run: |
pip install pyyaml
pip install ccm
- name: Setup CCM Cluster
run: ./.github/scripts/configure-ccm.sh
env:
TEST_TYPE: ${{ matrix.test-type }}
CASSANDRA_VERSION: ${{ matrix.cassandra-version }}
STORAGE_TYPE: ${{ matrix.storage-type }}
JDK_VERSION: ${{ matrix.jdk-version }}
- name: Run Tests
run: ./.github/scripts/run-tests.sh
env:
TEST_TYPE: ${{ matrix.test-type }}
CASSANDRA_VERSION: ${{ matrix.cassandra-version }}
STORAGE_TYPE: ${{ matrix.storage-type }}
CUCUMBER_OPTIONS: ${{ matrix.cucumber-options }}
JDK_VERSION: ${{ matrix.jdk-version }}
- uses: codecov/codecov-action@v1
- name: Setup tmate session
if: "failure()"
uses: mxschmitt/action-tmate@v3
timeout-minutes: 30
its-sidecar:
needs: [docker-tests, its-ccm-local, its-ccm-cass]
name: Sidecar
runs-on: ubuntu-20.04
continue-on-error: ${{ matrix.experimental }}
strategy:
matrix:
cassandra-version: ['binary:3.0.25', 'binary:3.11.16', 'binary:4.0.6']
storage-type: [cassandra]
test-type: [sidecar]
grim-max: [1]
grim-min: [1]
# all versions but trunk have the same cucumber options, but we can't declare that more effectively (yet)
include:
- cassandra-version: 'binary:3.0.25'
cucumber-options: '--tags ~@cassandra_4_0_onwards --tags @sidecar --tags ~@cassandra_3_11_onwards'
experimental: false
jdk-version: '8'
- cassandra-version: 'binary:3.11.16'
cucumber-options: '--tags ~@cassandra_4_0_onwards --tags @sidecar'
experimental: false
jdk-version: '8'
- cassandra-version: 'binary:4.0.6'
cucumber-options: '--tags @sidecar'
experimental: false
jdk-version: '11'
#- cassandra-version: 'github:apache/trunk'
# cucumber-options: '--tags @sidecar'
# experimental: true
# jdk-version: '11'
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup
uses: ./.github/actions/setup
- name: Install CCM
run: |
pip install pyyaml
pip install ccm
- name: Setup CCM Cluster
run: ./.github/scripts/configure-ccm.sh
env:
TEST_TYPE: ${{ matrix.test-type }}
CASSANDRA_VERSION: ${{ matrix.cassandra-version }}
STORAGE_TYPE: ${{ matrix.storage-type }}
JDK_VERSION: ${{ matrix.jdk-version }}
- name: Run Tests
run: ./.github/scripts/run-tests.sh || ./.github/scripts/run-tests.sh
env:
TEST_TYPE: ${{ matrix.test-type }}
CASSANDRA_VERSION: ${{ matrix.cassandra-version }}
STORAGE_TYPE: ${{ matrix.storage-type }}
GRIM_MAX: ${{ matrix.grim-max }}
GRIM_MIN: ${{ matrix.grim-min }}
CUCUMBER_OPTIONS: ${{ matrix.cucumber-options }}
JDK_VERSION: ${{ matrix.jdk-version }}
- uses: codecov/codecov-action@v1
its-http-management:
needs: [ build ]
name: HTTP Management
runs-on: ubuntu-20.04
continue-on-error: ${{ matrix.experimental }}
strategy:
matrix:
cassandra-version: [ 'binary:4.0.6', 'binary:3.11.16' ]
storage-type: [ "ccm" ]
test-type: [ "http-api" ]
grim-max: [ 1 ]
grim-min: [ 1 ]
# all versions but trunk have the same cucumber options, but we can't declare that more effectively (yet)
include:
- cassandra-version: 'binary:3.11.16'
cucumber-options: '--tags ~@cassandra_4_0_onwards --tags @http_management'
experimental: false
jdk-version: '8'
- cassandra-version: 'binary:4.0.6'
cucumber-options: '--tags @http_management'
experimental: false
jdk-version: '11'
- cassandra-version: 'binary:3.11.16'
cucumber-options: '--tags ~@cassandra_4_0_onwards'
experimental: false
jdk-version: '8'
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup
uses: ./.github/actions/setup
- name: Install CCM
run: |
pip install pyyaml
pip install ccm
- name: Setup CCM Cluster
run: ./.github/scripts/configure-ccm.sh
env:
TEST_TYPE: ${{ matrix.test-type }}
CASSANDRA_VERSION: ${{ matrix.cassandra-version }}
STORAGE_TYPE: ${{ matrix.storage-type }}
JDK_VERSION: ${{ matrix.jdk-version }}
- name: Run Tests
run: ./.github/scripts/run-tests.sh
env:
TEST_TYPE: ${{ matrix.test-type }}
CASSANDRA_VERSION: ${{ matrix.cassandra-version }}
STORAGE_TYPE: ${{ matrix.storage-type }}
GRIM_MAX: ${{ matrix.grim-max }}
GRIM_MIN: ${{ matrix.grim-min }}
CUCUMBER_OPTIONS: ${{ matrix.cucumber-options }}
JDK_VERSION: ${{ matrix.jdk-version }}
# Upload logs
- name: Prepare a unique name for logs
if: ${{ failure() }}
run: |
name=${{ matrix.test-type }}-${{ matrix.cassandra-version }}
# The option to enable + in sed regexps differs by OS so we avoid it
name=$(echo -n "$name" | sed -e 's/[ \t:\/\\"<>|*?]/-/g' -e 's/--*/-/g')
echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV
- name: Archive logs
if: ${{ failure() }}
uses: actions/upload-artifact@v3
with:
name: reaper-logs-${{ env.ARTIFACT_NAME }}
path: /tmp/log
# Add a tmate action to debug failures.
- name: Setup tmate session
if: ${{ failure() }}
uses: mxschmitt/action-tmate@v3
timeout-minutes: 30
- uses: codecov/codecov-action@v1
its-each:
needs: [docker-tests, its-ccm-local, its-ccm-cass]
name: Each DC Availability
runs-on: ubuntu-20.04
continue-on-error: ${{ matrix.experimental }}
strategy:
matrix:
cassandra-version: ['binary:4.0.6']
storage-type: [cassandra]
test-type: [each]
grim-max: [1]
grim-min: [1]
# Reduced scope to shorten the integration test times
include:
- cassandra-version: 'binary:4.0.6'
cucumber-options: '--tags @sidecar'
experimental: false
#- cassandra-version: 'github:apache/trunk'
# cucumber-options: '--tags @sidecar'
# experimental: true
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup
uses: ./.github/actions/setup
- name: Install CCM
run: |
pip install pyyaml
pip install ccm
- name: Setup CCM Cluster
run: ./.github/scripts/configure-ccm.sh
env:
TEST_TYPE: ${{ matrix.test-type }}
CASSANDRA_VERSION: ${{ matrix.cassandra-version }}
STORAGE_TYPE: ${{ matrix.storage-type }}
JDK_VERSION: ${{ matrix.jdk-version }}
- name: Run Tests
run: ./.github/scripts/run-tests.sh || ./.github/scripts/run-tests.sh
env:
TEST_TYPE: ${{ matrix.test-type }}
CASSANDRA_VERSION: ${{ matrix.cassandra-version }}
STORAGE_TYPE: ${{ matrix.storage-type }}
GRIM_MAX: ${{ matrix.grim-max }}
GRIM_MIN: ${{ matrix.grim-min }}
CUCUMBER_OPTIONS: ${{ matrix.cucumber-options }}
JDK_VERSION: ${{ matrix.jdk-version }}
- uses: codecov/codecov-action@v1
its-distributed:
needs: [docker-tests, its-ccm-local, its-ccm-cass]
name: Distributed tests
runs-on: ubuntu-20.04
continue-on-error: ${{ matrix.experimental }}
strategy:
matrix:
cassandra-version: ['binary:3.0.25', 'binary:3.11.16', 'binary:4.0.6']
storage-type: [cassandra]
test-type: [ccm]
grim-max: [2]
grim-min: [2]
include:
- cassandra-version: 'binary:3.0.25'
cucumber-options: '--tags ~@cassandra_4_0_onwards --tags ~@cassandra_3_11_onwards'
experimental: false
jdk-version: '8'
- cassandra-version: 'binary:3.11.16'
cucumber-options: '--tags ~@cassandra_4_0_onwards'
experimental: false
jdk-version: '8'
- cassandra-version: 'binary:4.0.6'
experimental: false
jdk-version: '11'
#- cassandra-version: 'github:apache/trunk'
# experimental: true
# jdk-version: '11'
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup
uses: ./.github/actions/setup
- name: Install CCM
run: |
pip install pyyaml
pip install ccm
- name: Setup CCM Cluster
run: ./.github/scripts/configure-ccm.sh
env:
TEST_TYPE: ${{ matrix.test-type }}
CASSANDRA_VERSION: ${{ matrix.cassandra-version }}
STORAGE_TYPE: ${{ matrix.storage-type }}
JDK_VERSION: ${{ matrix.jdk-version }}
- name: Run Tests
run: ./.github/scripts/run-tests.sh || ./.github/scripts/run-tests.sh
env:
TEST_TYPE: ${{ matrix.test-type }}
CASSANDRA_VERSION: ${{ matrix.cassandra-version }}
STORAGE_TYPE: ${{ matrix.storage-type }}
CUCUMBER_OPTIONS: ${{ matrix.cucumber-options }}
GRIM_MAX: ${{ matrix.grim-max }}
GRIM_MIN: ${{ matrix.grim-min }}
JDK_VERSION: ${{ matrix.jdk-version }}
- uses: codecov/codecov-action@v1
its-flapping:
needs: [docker-tests, its-ccm-local, its-ccm-cass]
name: Flapping reapers
runs-on: ubuntu-20.04
continue-on-error: ${{ matrix.experimental }}
strategy:
matrix:
cassandra-version: ['binary:4.0.6']
storage-type: [cassandra]
test-type: [ccm]
grim-max: [4]
grim-min: [2]
cucumber-options: ['--tags ~@diag_event_subscriptions']
include:
# all versions but trunk need to exclude trunk tests
- cassandra-version: 'binary:4.0.6'
experimental: true
jdk-version: '11'
#- cassandra-version: 'github:apache/trunk'
# experimental: true
# jdk-version: '11'
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup
uses: ./.github/actions/setup
- name: Install CCM
run: |
pip install pyyaml
pip install ccm
- name: Setup CCM Cluster
run: ./.github/scripts/configure-ccm.sh
env:
TEST_TYPE: ${{ matrix.test-type }}
CASSANDRA_VERSION: ${{ matrix.cassandra-version }}
STORAGE_TYPE: ${{ matrix.storage-type }}
JDK_VERSION: ${{ matrix.jdk-version }}
- name: Run Tests
run: |
./.github/scripts/run-tests.sh || ./.github/scripts/run-tests.sh || ./.github/scripts/run-tests.sh
env:
TEST_TYPE: ${{ matrix.test-type }}
CASSANDRA_VERSION: ${{ matrix.cassandra-version }}
STORAGE_TYPE: ${{ matrix.storage-type }}
CUCUMBER_OPTIONS: ${{ matrix.cucumber-options }}
GRIM_MAX: ${{ matrix.grim-max }}
GRIM_MIN: ${{ matrix.grim-min }}
JDK_VERSION: ${{ matrix.jdk-version }}
- uses: codecov/codecov-action@v1
release:
name: Release Reaper
needs: [its-distributed, its-flapping, its-sidecar, its-each]
if: startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/tag')
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup
uses: ./.github/actions/setup
- name: Compute version number
run: |
VERSION=$(printf 'VER\t${project.version}' | mvn help:evaluate | grep '^VER' | cut -f2)
if [ "${GITHUB_REF}" = "refs/heads/master" ]
then
# We're building a beta version
VERSION=$(printf 'VER\t${project.version}' | mvn help:evaluate | grep '^VER' | cut -f2)
DATE=$(date +"%Y%m%d")
# Bintray doesn't like snapshots, but accepts betas :)
BETA_VERSION=$(echo $VERSION | sed "s/SNAPSHOT/BETA/")
mvn -B versions:set "-DnewVersion=${BETA_VERSION}-${DATE}"
fi
echo "Releasing version ${VERSION}"
- name: Build Reaper
run: |
MAVEN_OPTS="-Xmx384m" mvn -B install -DskipTests
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
with:
platforms: 'arm64,arm'
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Build packages
run: ./.github/scripts/before_deploy.sh
- name: tags
run: |
echo "Tags: ${{ env.DOCKER_TAG1 }} and ${{ env.DOCKER_TAG2 }}"
- name: Build image and push
id: docker_build
uses: docker/build-push-action@v4
with:
file: src/server/src/main/docker/Dockerfile
context: .
build-args: |
SHADED_JAR=${{ env.SHADED_JAR }}
push: true
tags: ${{ env.DOCKER_TAG1 }},${{ env.DOCKER_TAG2 }}
platforms: linux/amd64,linux/arm64
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache
- name: Locate deb and rpm packages for upload
run: |
REPO=reaper-deb
PACKAGE=cassandra-reaper
VERSION=$(printf 'VER\t${project.version}' | mvn help:evaluate | grep '^VER' | cut -f2)
if [ "${GITHUB_REF}" = "refs/heads/master" ]
then
echo "BETA_RELEASE=yes" >> $GITHUB_ENV
else
echo "BETA_RELEASE=no" >> $GITHUB_ENV
fi
debReleaseFile=$(ls src/packages/*.deb |grep ${VERSION})
rpmReleaseFile=$(ls src/packages/*.rpm)
echo "DEB_RELEASE_FILE=${debReleaseFile}" >> $GITHUB_ENV
echo "RPM_RELEASE_FILE=${rpmReleaseFile}" >> $GITHUB_ENV
- name: Upload artifacts to GitHub Release
uses: fnkr/github-action-ghr@v1
if: github.event_name == 'release' && github.event.action == 'published'
env:
GHR_PATH: src/packages
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
## Stable releases
- name: Push Debian to Cloudsmith
id: push-deb
if: ${{ env.BETA_RELEASE == 'no' }}
uses: cloudsmith-io/action@master
with:
api-key: ${{ secrets.CLOUDSMITH_API_KEY }}
command: 'push'
format: 'deb'
owner: 'thelastpickle'
repo: 'reaper'
distro: 'any-distro'
release: 'any-version'
republish: 'true'
file: ${{ env.DEB_RELEASE_FILE }}
- name: Push RPM to Cloudsmith
id: push-rpm
if: ${{ env.BETA_RELEASE == 'no' }}
uses: cloudsmith-io/action@master
with:
api-key: ${{ secrets.CLOUDSMITH_API_KEY }}
command: "push"
format: "rpm"
owner: 'thelastpickle'
repo: 'reaper'
distro: 'any-distro'
release: 'any-version'
republish: 'true'
file: ${{ env.RPM_RELEASE_FILE }}
## Beta releases
- name: Push Debian Beta to Cloudsmith
id: push-beta-deb
if: ${{ env.BETA_RELEASE == 'yes' }}
uses: cloudsmith-io/action@master
with:
api-key: ${{ secrets.CLOUDSMITH_API_KEY }}
command: 'push'
format: 'deb'
owner: 'thelastpickle'
repo: 'reaper-beta'
distro: 'any-distro'
release: 'any-version'
republish: 'true'
file: ${{ env.DEB_RELEASE_FILE }}
- name: Push RPM Beta to Cloudsmith
id: push-beta-rpm
if: ${{ env.BETA_RELEASE == 'yes' }}
uses: cloudsmith-io/action@master
with:
api-key: ${{ secrets.CLOUDSMITH_API_KEY }}
command: "push"
format: "rpm"
owner: 'thelastpickle'
repo: 'reaper-beta'
distro: 'any-distro'
release: 'any-version'
republish: 'true'
file: ${{ env.RPM_RELEASE_FILE }}