Skip to content
PR tests

[CLIENT-3027] CI/CD: Use Dockerfile to build Docker image for Aerospike enterprise server with security enabled #3182

Sign in to view logs

[CLIENT-3027] CI/CD: Use Dockerfile to build Docker image for Aerospike enterprise server with security enabled

[CLIENT-3027] CI/CD: Use Dockerfile to build Docker image for Aerospike enterprise server with security enabled #3182

Workflow file for this run

.github/workflows/tests.yml at bc7be56
name: PR tests
env:
LOWEST_SUPPORTED_PY_VERSION: '3.8'
# Trigger test workflow whenever:
# 1. A pull request is updated (e.g with new commits)
# 2. Commits are pushed directly to the dev, stage or master branch
on:
push:
branches: ["dev*", "stage*", "master*", "v*-backport*"]
pull_request:
branches: ["dev*", "stage*", "v*-backport*"]
types: [
# Default triggers
opened,
synchronize,
reopened,
# Additional triggers
labeled,
unlabeled
]
workflow_dispatch:
inputs:
test-server-rc:
type: boolean
default: false
required: true
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
- uses: actions/setup-python@v2
with:
python-version: '3.9'
architecture: 'x64'
- uses: pre-commit/action@v3.0.0
build:
runs-on: ubuntu-latest
strategy:
matrix:
py-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
fail-fast: false
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
fetch-depth: 0
- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.py-version }}
architecture: 'x64'
- run: sudo apt update
- name: Install build dependencies (apt packages)
run: sudo apt install python3-dev libssl-dev -y
- name: Install build dependencies (pip packages)
run: python3 -m pip install -r requirements.txt
- name: Build client
run: python3 -m build
env:
CFLAGS: '-Werror'
- name: Send wheel to test jobs
uses: actions/upload-artifact@v3
with:
name: wheel-${{ matrix.py-version }}
path: ./dist/*.whl
test-memray:
needs: build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: "3.8"
architecture: 'x64'
- uses: actions/download-artifact@v3
with:
name: wheel-3.8
- name: Install client
run: pip install *.whl
- name: Install test dependencies
run: pip install -r test/requirements.txt
- name: Run Aerospike server
run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server
- name: Create config.conf
run: cp config.conf.template config.conf
working-directory: test
- uses: ./.github/actions/wait-for-as-server-to-start
with:
container-name: aerospike
- name: Get number of tests
run: echo "NUM_TESTS=$(python3 -m pytest new_tests/ --collect-only -q | tail -n 1 | awk '{print $1;}')" >> $GITHUB_ENV
working-directory: test
- name: Run tests
# Get number of tests since setting to 0 doesn't work properly
# pytest-memray currently throws a ZeroDivision error due to having a bug
# We ignore this for now
run: python -m pytest ./new_tests --memray --memray-bin-path=./ --most-allocations=${{ env.NUM_TESTS }} || true
working-directory: test
generate-coverage-report:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
fetch-depth: 0
- uses: actions/setup-python@v2
with:
python-version: "3.9"
architecture: 'x64'
- name: Build client
# Use old build command to generate object .o files in build/temp*/src/main directory
# They will also contain .gcno files there
# and .gcda files will be generated there after running the tests
# The build frontend doesn't generate these .o files (may be wrong?)
run: COVERAGE=1 python3 setup.py build
- name: Install client
# Install in user directory to prevent permission errors
run: python3 setup.py install --user
- run: pip install -r requirements.txt
working-directory: test
- name: Run Aerospike server
uses: ./.github/actions/run-ee-server
with:
use-server-rc: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }}
docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }}
docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }}
- run: python3 -m pytest --cov=aerospike_helpers --cov-report xml:coverage.xml ./new_tests
working-directory: test
- name: Copy over source files to build dir
if: ${{ !cancelled() }}
# The build/temp*/src/main directory will contain a hierarchy of object .o files
# But the source files must be stored in the same folder hierarchy at build/temp*/src/main/src/main
run: |
cd build/temp*/src/main
mkdir -p src/main/
mkdir -p cdt_types/src/main/cdt_types
mkdir -p client/src/main/client
mkdir -p geospatial/src/main/geospatial
mkdir -p global_hosts/src/main/global_hosts
mkdir -p key_ordered_dict/src/main/key_ordered_dict
mkdir -p nullobject/src/main/nullobject
mkdir -p query/src/main/query
mkdir -p scan/src/main/scan
cd ../../../../
cp src/main/*.c build/temp*/src/main/src/main
cp src/main/cdt_types/*.c build/temp*/src/main/cdt_types/src/main/cdt_types/
cp src/main/client/*.c build/temp*/src/main/client/src/main/client/
cp src/main/geospatial/*.c build/temp*/src/main/geospatial/src/main/geospatial/
cp src/main/global_hosts/*.c build/temp*/src/main/global_hosts/src/main/global_hosts/
cp src/main/key_ordered_dict/*.c build/temp*/src/main/key_ordered_dict/src/main/key_ordered_dict/
cp src/main/nullobject/*.c build/temp*/src/main/nullobject/src/main/nullobject/
cp src/main/query/*.c build/temp*/src/main/query/src/main/query/
cp src/main/scan/*.c build/temp*/src/main/scan/src/main/scan/
- name: Generate coverage report for all object files
if: ${{ !cancelled() }}
run: |
cd build/temp*/src/main
find . -type f -name "*.o" -execdir gcov {} \;
- name: Move aerospike_helpers coverage report to this directory
if: ${{ !cancelled() }}
run: mv test/coverage.xml build/temp*/src/main
- name: Upload coverage report folder to Github
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v3
with:
name: coverage-report
path: build/temp*/src/main/
coverage-upload:
needs: generate-coverage-report
if: ${{ !cancelled() }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- uses: actions/download-artifact@v3
with:
name: coverage-report
path: ./coverage-report
- uses: codecov/codecov-action@v3
with:
directory: coverage-report
verbose: true # optional (default = false)
fail_ci_if_error: true
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
stubtest:
needs: build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
- uses: actions/setup-python@v2
with:
python-version: ${{ env.LOWEST_SUPPORTED_PY_VERSION }}
architecture: 'x64'
- uses: actions/download-artifact@v3
with:
name: wheel-${{ env.LOWEST_SUPPORTED_PY_VERSION }}
- name: Install client
run: pip install *.whl
- run: pip install mypy
- run: stubtest aerospike --allowlist stubtest-allowlist
# Run this when testing new server features on server release candidate
# to make sure the tests don't regress on the last server release.
test-ce-latest-release:
runs-on: ubuntu-latest
if: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }}
needs: build
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
- uses: actions/setup-python@v2
with:
python-version: ${{ env.LOWEST_SUPPORTED_PY_VERSION }}
architecture: 'x64'
- uses: actions/download-artifact@v3
with:
name: wheel-${{ env.LOWEST_SUPPORTED_PY_VERSION }}
- name: Install client
run: pip install *.whl
- name: Install test dependencies
run: pip install -r test/requirements.txt
- name: Run Aerospike server
run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server
- name: Create config.conf
run: cp config.conf.template config.conf
working-directory: test
- uses: ./.github/actions/wait-for-as-server-to-start
with:
container-name: aerospike
- name: Run tests
run: python -m pytest ./new_tests
working-directory: test
test-ce:
runs-on: ubuntu-latest
needs: build
strategy:
matrix:
py-version: [
"3.8",
"3.9",
"3.10",
"3.11",
"3.12"
]
fail-fast: false
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.py-version }}
architecture: 'x64'
- uses: actions/download-artifact@v3
with:
name: wheel-${{ matrix.py-version }}
- name: Install client
run: pip install *.whl
- name: Install test dependencies
run: pip install -r test/requirements.txt
- if: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }}
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }}
password: ${{ secrets.DOCKER_HUB_BOT_PW }}
- name: Run Aerospike server release candidate with latest tag
if: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }}
run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server-rc:latest
- name: Run Aerospike server
if: ${{ !contains(github.event.pull_request.labels.*.name, 'new-server-features') }}
run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server
- name: Create config.conf
run: cp config.conf.template config.conf
working-directory: test
- uses: ./.github/actions/wait-for-as-server-to-start
with:
container-name: aerospike
- name: Run tests
run: python -m pytest ./new_tests -vv
working-directory: test
test-lowest-supported-server:
runs-on: ubuntu-latest
needs: build
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
- uses: actions/setup-python@v2
with:
python-version: ${{ env.LOWEST_SUPPORTED_PY_VERSION }}
architecture: 'x64'
- uses: actions/download-artifact@v3
with:
name: wheel-${{ env.LOWEST_SUPPORTED_PY_VERSION }}
- name: Install client
run: pip install *.whl
- name: Install test dependencies
run: pip install -r test/requirements.txt
- name: Run lowest supported server
run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server:${{ vars.LOWEST_SUPPORTED_SERVER_VERSION }}
- name: Create config.conf
run: cp config.conf.template config.conf
working-directory: test
- uses: ./.github/actions/wait-for-as-server-to-start
with:
container-name: aerospike
- name: Run tests
run: python -m pytest ./new_tests
working-directory: test
test-ee:
runs-on: ubuntu-latest
needs: build
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
- uses: actions/setup-python@v2
with:
python-version: ${{ env.LOWEST_SUPPORTED_PY_VERSION }}
architecture: 'x64'
- uses: actions/download-artifact@v3
with:
name: wheel-${{ env.LOWEST_SUPPORTED_PY_VERSION }}
- name: Install client
run: pip install *.whl
- name: Install test dependencies
run: pip install -r test/requirements.txt
- uses: ./.github/actions/run-ee-server
with:
use-server-rc: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }}
docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }}
docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }}
- name: Run tests
run: python -m pytest ./new_tests/test_admin_*.py
working-directory: test
- name: Show logs if failed
if: ${{ failure() }}
run: |
docker container logs aerospike
cat ./configs/aerospike.conf
spellcheck-docs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
- uses: actions/setup-python@v2
with:
python-version: '3.9'
architecture: 'x64'
- name: Install dependencies for checking spelling in docs
# TODO: find way to split up dependencies
run: python -m pip install -r doc/requirements.txt
- name: Check spelling
run: sphinx-build -b spelling . spelling -W --keep-going
working-directory: doc
check-links-docs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: '3.9'
architecture: 'x64'
- name: Install documentation dependencies
run: python -m pip install -r doc/requirements.txt
- name: Check spelling
run: sphinx-build -b linkcheck . links
working-directory: doc
test-metrics-node-close-listener:
needs: build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ env.LOWEST_SUPPORTED_PY_VERSION }}
architecture: 'x64'
- name: Download aerolab
run: wget https://github.com/aerospike/aerolab/releases/download/7.6.0-7b5bbde/aerolab-linux-amd64-7.6.0.deb
- name: Install aerolab
run: sudo dpkg -i *.deb
- name: Tell aerolab to use Docker
run: aerolab config backend -t docker
- uses: actions/download-artifact@v3
with:
name: wheel-${{ env.LOWEST_SUPPORTED_PY_VERSION }}
- run: python3 -m pip install *.whl
- run: python3 test_node_close_listener.py
working-directory: test/metrics
test-metrics-cluster-name:
needs: build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ env.LOWEST_SUPPORTED_PY_VERSION }}
architecture: 'x64'
- uses: actions/download-artifact@v3
with:
name: wheel-${{ env.LOWEST_SUPPORTED_PY_VERSION }}
- run: python3 -m pip install *.whl
- run: python3 -m pip install -r requirements.txt
working-directory: test/metrics
- run: python3 test_cluster_name.py
working-directory: test/metrics