N2k accum changes #2909
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# NOTE: | |
# When we build kotekan on the Github VM we need to take into account a few | |
# limitations: first we can't lock very much memory so we need to disable | |
# mlock the buffers. Second, the VMs run across a range of CPU | |
# architectures (from Haswell to Skylake-AVX512 as of 2020/03). Because our | |
# use of ccache shares compiled objects we must target our build for the | |
# lowest architecture available or we will occasionally crash with SIGILL | |
# when a newer instruction is called than is available. | |
name: kotekan-ci-tests | |
on: | |
pull_request: | |
branches: | |
- chord | |
- develop | |
- master | |
push: | |
branches: | |
- chord | |
- develop | |
- master | |
env: | |
REGISTRY: ghcr.io | |
IMG_IWYU: "ghcr.io/kotekan/kotekan/kotekan-iwyu" | |
IMG_1804: "ghcr.io/${{ github.repository }}/kotekan-1804" | |
IMG_2204: "ghcr.io/${{ github.repository }}/kotekan-2204" | |
PYTEST_TIMEOUT: 60 | |
jobs: | |
# 2204 Tests | |
# Ubuntu 22.04 docker image build and push | |
build-docker-2204: | |
runs-on: ubuntu-latest | |
permissions: write-all | |
steps: | |
- name: Checkout repository | |
uses: actions/checkout@v3 | |
- name: Check for file changes | |
uses: tj-actions/changed-files@v37 | |
id: changed-files | |
- name: Login to GitHub Container Registry | |
uses: docker/login-action@v2 | |
with: | |
registry: ${{ env.REGISTRY }} | |
username: ${{ github.actor }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
# Rebuild (or build) for a push/pr from a local branch (separate workflow for PR merges) | |
- name: Check to see if a docker rebuild is needed | |
id: docker_img | |
run: | | |
echo "docker_build=" >> $GITHUB_OUTPUT; # Default to no rebuild | |
if [[ ${{ contains(steps.changed-files.outputs.all_modified_files, 'tools/docker/22.04/Dockerfile') }} == "true" || ${DOCKER_BUILD} ]] ; then | |
echo "Modified Dockerfile found (or re-build requested). Building on local branch." | |
if [ "${GITHUB_EVENT_NAME}" = "push" ]; then | |
IMAGE_NAME="${IMG_2204}:$(echo ${GITHUB_REF_NAME} | tr / .)" | |
else | |
IMAGE_NAME="${IMG_2204}:$(echo ${GITHUB_HEAD_REF} | tr / .)" | |
fi | |
echo "docker_build=1" >> $GITHUB_OUTPUT; | |
else | |
if [ "${GITHUB_EVENT_NAME}" = "pull_request" || "${GITHUB_EVENT_NAME}" = "pull_request_target" ]; then | |
echo "Dockerfile was not modified. Using image from branch ${GITHUB_BASE_REF}." | |
IMAGE_NAME="${IMG_2204}:$(echo ${GITHUB_BASE_REF} | tr / .)" | |
else | |
# This can happen when we're merging a chord branch back to chord, eg | |
echo "Dockerfile was not modified. Using image from branch ${GITHUB_REF_NAME}." | |
IMAGE_NAME="${IMG_2204}:$(echo ${GITHUB_REF_NAME} | tr / .)" | |
fi | |
fi | |
if docker manifest inspect ${IMAGE_NAME} >/dev/null; then | |
echo "Docker package found." | |
else | |
echo "Docker package NOT found. Setting docker_build=1." | |
echo "docker_build=1" >> $GITHUB_OUTPUT; | |
fi | |
echo "img_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT; | |
echo "Image name: ${IMAGE_NAME}" | |
- name: Output environment variable details | |
run: | | |
echo "The job_id is: $GITHUB_JOB" | |
echo "The id of this action is: $GITHUB_ACTION" | |
echo "The run id is: $GITHUB_RUN_ID" | |
echo "The repository is: $GITHUB_REPOSITORY" | |
echo "The GitHub Actor's username is: $GITHUB_ACTOR" | |
echo "GitHub SHA: $GITHUB_SHA" | |
echo "GitHub event name: $GITHUB_EVENT_NAME" | |
echo "GitHub Ref: $GITHUB_REF" | |
echo "GitHub Head Ref: $GITHUB_HEAD_REF" | |
echo "GitHub Base Ref: $GITHUB_BASE_REF" | |
echo "GitHub Ref Name: $GITHUB_REF_NAME" | |
echo "Docker 1804 image: $IMG_1804" | |
echo "Docker 2204 image: $IMG_2204" | |
echo "Docker IWYU image: $IMG_IWYU" | |
echo "Used docker image name: ${{ steps.docker_img.outputs.img_name }}" | |
echo "docker_build: ${{ steps.docker_img.outputs.docker_build }}" | |
echo "img_name: ${{ steps.docker_img.outputs.img_name }}" | |
- name: Build and Push Docker Image | |
if: steps.docker_img.outputs.docker_build | |
run: | | |
echo "Building and pushing ${{ steps.docker_img.outputs.img_name }}" | |
docker build --cache-from=${IMG_2204} -t "${{ steps.docker_img.outputs.img_name }}" - < tools/docker/22.04/Dockerfile | |
docker push "${{ steps.docker_img.outputs.img_name }}" | |
outputs: | |
docker_img_name: "${{ steps.docker_img.outputs.img_name }}" | |
# Build kotekan documentation | |
build-docs: | |
runs-on: ubuntu-latest | |
needs: build-docker-2204 | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Login to GitHub Container Registry | |
uses: docker/login-action@v2 | |
with: | |
registry: ${{ env.REGISTRY }} | |
username: ${{ github.actor }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- name: Build kotekan docs | |
run: | | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan/build-docs/ "${{ needs.build-docker-2204.outputs.docker_img_name }}") | |
docker run "${OPTS[@]}" \ | |
cmake \ | |
-Wdev -Werror=dev -Wdeprecated -Werror=deprecated \ | |
-DDOXYGEN_EXECUTABLE=/usr/local/bin/doxygen \ | |
-DWERROR=ON \ | |
-DCOMPILE_DOCS=ON \ | |
.. | |
docker run "${OPTS[@]}" make doc | |
docker run "${OPTS[@]}" make sphinx | |
# Linting | |
lint: | |
runs-on: ubuntu-latest | |
needs: build-docker-2204 | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Login to GitHub Container Registry | |
uses: docker/login-action@v2 | |
with: | |
registry: ${{ env.REGISTRY }} | |
username: ${{ github.actor }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- name: Run cmakelint | |
run: | | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan/ ${{ needs.build-docker-2204.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" tools/cmakelint.sh /code/kotekan | |
- name: Run black | |
run: | | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan/ ${{ needs.build-docker-2204.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" black --exclude docs --check . | |
- name: Run clang-format | |
run: | | |
mkdir -p build-check-format | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan/build-check-format/ ${{ needs.build-docker-2204.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" cmake -Wdev -Werror=dev -Wdeprecated -Werror=deprecated .. | |
docker run "${OPTS[@]}" make clang-format | |
git diff --exit-code | |
# Build a basic version of kotekan | |
build-base-2204: | |
runs-on: ubuntu-latest | |
needs: build-docker-2204 | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Login to GitHub Container Registry | |
uses: docker/login-action@v2 | |
with: | |
registry: ${{ env.REGISTRY }} | |
username: ${{ github.actor }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- name: Cache ccache files | |
uses: actions/cache@v3 | |
with: | |
path: .ccache | |
key: ccache-base-build-2204-${{ github.sha }} | |
restore-keys: | | |
ccache-base-build-2204 | |
ccache-full-build-2204 | |
- name: Build kotekan | |
run: | | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan/build-2204/ ${{ needs.build-docker-2204.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" ccache -s | |
docker run "${OPTS[@]}" \ | |
cmake -Wdev -Werror=dev -Wdeprecated -Werror=deprecated \ | |
-DWERROR=ON \ | |
-DCMAKE_LINK_WHAT_YOU_USE=ON \ | |
-DCMAKE_BUILD_TYPE=Test \ | |
-DARCH=haswell \ | |
-DCCACHE=ON \ | |
-DWITH_TESTS=ON .. | |
docker run "${OPTS[@]}" make -j 2 | |
docker run "${OPTS[@]}" ccache -s | |
# Build a full version of kotekan and run the unit tests | |
build-full-tests-2204: | |
runs-on: ubuntu-latest | |
needs: build-docker-2204 | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Login to GitHub Container Registry | |
uses: docker/login-action@v2 | |
with: | |
registry: ${{ env.REGISTRY }} | |
username: ${{ github.actor }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- name: Cache ccache files | |
uses: actions/cache@v3 | |
with: | |
path: .ccache | |
key: ccache-full-build-2204-${{ github.sha }} | |
restore-keys: | | |
ccache-full-build-2204 | |
ccache-base-build-2204 | |
- name: Free disk space | |
if: ${{ !env.ACT }} # Don't free up if running locally | |
run: | | |
df -h | |
docker rmi $(docker image ls -aq) | |
df -h | |
sudo apt-get clean | |
sudo apt-get autoclean | |
df -h | |
- name: Build kotekan | |
run: | | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan/build-2204/ ${{ needs.build-docker-2204.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" ccache -s | |
docker run "${OPTS[@]}" \ | |
cmake -Wdev -Werror=dev -Wdeprecated -Werror=deprecated \ | |
-DWERROR=ON \ | |
-DCMAKE_LINK_WHAT_YOU_USE=ON \ | |
-DCMAKE_BUILD_TYPE=Test \ | |
-DUSE_ASDF=ON \ | |
-DUSE_HDF5=ON -DHIGHFIVE_PATH=/code/build/HighFive \ | |
-DUSE_LAPACK=ON -DBLAZE_PATH=/code/build/blaze \ | |
-DARCH=haswell \ | |
-DNO_MEMLOCK=ON \ | |
-DUSE_OMP=ON \ | |
-DWITH_BOOST_TESTS=ON \ | |
-DCCACHE=ON .. | |
docker run "${OPTS[@]}" make -j 2 | |
docker run "${OPTS[@]}" rm -r lib | |
docker run "${OPTS[@]}" ccache -s | |
df -h | |
- name: Run direct Kotekan tests | |
run: | | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan ${{ needs.build-docker-2204.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" ./build-2204/kotekan/kotekan --bind-address 0:23000 --config config/tests/julia_hello_world.yaml | |
- name: Run parallel python tests | |
run: | | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan/tests/ -e PYTHONPATH=/code/kotekan/python/ ${{ needs.build-docker-2204.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" pytest -v -n auto --dist=loadfile --timeout=${PYTEST_TIMEOUT} -x -m 'not serial' | |
- name: Run serial python tests | |
run: | | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan/tests/ -e PYTHONPATH=/code/kotekan/python/ ${{ needs.build-docker-2204.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" bash -c "redis-server --daemonize yes; pytest -v -x --timeout=${PYTEST_TIMEOUT} -m serial" | |
- name: Run boost tests | |
run: | | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan/build-2204/tests/ -e PYTHONPATH=/code/kotekan/python/ ${{ needs.build-docker-2204.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" pytest -v -x --timeout=${PYTEST_TIMEOUT} | |
# Build an OpenCL version of kotekan and run the unit tests | |
build-opencl-2204: | |
runs-on: self-hosted | |
container: | |
image: summerwind/actions-runner:ubuntu-22.04 | |
options: --user root | |
needs: build-docker-2204 | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Login to GitHub Container Registry | |
uses: docker/login-action@v2 | |
with: | |
registry: ${{ env.REGISTRY }} | |
username: ${{ github.actor }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- name: Cache ccache files | |
uses: actions/cache@v3 | |
with: | |
path: .ccache | |
key: ccache-opencl-2204-${{ github.sha }} | |
restore-keys: | | |
ccache-opencl-2204 | |
ccache-base-build-2204 | |
- name: Build kotekan | |
shell: bash | |
run: | | |
OPTS=(--pull=always --rm --mount type=bind,src=/home/lwlab/kotekan-runner/_work/kotekan/kotekan/,target=/code/kotekan -w /code/kotekan/build-2204/ ${{ needs.build-docker-2204.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" ccache -s | |
docker run "${OPTS[@]}" \ | |
cmake -Wdev -Werror=dev -Wdeprecated -Werror=deprecated \ | |
-DWERROR=ON \ | |
-DCMAKE_BUILD_TYPE=Test \ | |
-DUSE_HDF5=ON -DHIGHFIVE_PATH=/code/build/HighFive \ | |
-DUSE_LAPACK=ON -DBLAZE_PATH=/code/build/blaze \ | |
-DNO_MEMLOCK=ON \ | |
-DUSE_OMP=ON \ | |
-DWITH_BOOST_TESTS=ON \ | |
-DUSE_CUDA=OFF \ | |
-DUSE_OPENCL=ON \ | |
-DCCACHE=ON .. | |
docker run "${OPTS[@]}" make -j 2 | |
docker run "${OPTS[@]}" rm -r lib | |
docker run "${OPTS[@]}" ccache -s | |
df -h | |
- name: Run parallel python tests | |
shell: bash | |
run: | | |
OPTS=(--rm --mount type=bind,src=/home/lwlab/kotekan-runner/_work/kotekan/kotekan/,target=/code/kotekan -w /code/kotekan/tests/ -e PYTHONPATH=/code/kotekan/python/ ${{ needs.build-docker-2204.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" pytest -v -n auto --dist=loadfile --timeout=${PYTEST_TIMEOUT} -x -m 'not serial' | |
- name: Run serial python tests | |
shell: bash | |
run: | | |
OPTS=(--rm --mount type=bind,src=/home/lwlab/kotekan-runner/_work/kotekan/kotekan/,target=/code/kotekan -w /code/kotekan/tests/ -e PYTHONPATH=/code/kotekan/python/ ${{ needs.build-docker-2204.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" bash -c "redis-server --daemonize yes; pytest -v -x --timeout=${PYTEST_TIMEOUT} -m serial" | |
- name: Run boost tests | |
shell: bash | |
run: | | |
OPTS=(--rm --mount type=bind,src=/home/lwlab/kotekan-runner/_work/kotekan/kotekan/,target=/code/kotekan -w /code/kotekan/build-2204/tests/ -e PYTHONPATH=/code/kotekan/python/ ${{ needs.build-docker-2204.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" pytest -v -x --timeout=${PYTEST_TIMEOUT} | |
- name: Clean up | |
run: | | |
rm -rf kotekan | |
# Build a clang version of kotekan | |
# Note that this build isn't working at the moment, so for now only run it upon request. | |
build-clang-2204: | |
runs-on: ubuntu-latest | |
needs: build-docker-2204 | |
steps: | |
- name: Checkout repository | |
uses: actions/checkout@v3 | |
- name: Login to GitHub Container Registry | |
uses: docker/login-action@v2 | |
with: | |
registry: ${{ env.REGISTRY }} | |
username: ${{ github.actor }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- name: Cache ccache files | |
uses: actions/cache@v3 | |
with: | |
path: .ccache | |
key: ccache-clang-build-2204-${{ github.sha }} | |
restore-keys: | | |
ccache-clang-build-2204 | |
- name: Build kotekan | |
env: | |
CC: clang | |
CXX: clang++ | |
if: env.RUN_2204_CLANG_TESTS | |
run: | | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan/build-2204/ ${{ needs.build-docker-2204.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" ccache -s | |
docker run "${OPTS[@]}" cmake -Wdev -Werror=dev -Wdeprecated -Werror=deprecated \ | |
-DWERROR=ON \ | |
-DCMAKE_LINK_WHAT_YOU_USE=ON \ | |
-DUSE_OLD_DPDK=ON \ | |
-DCMAKE_BUILD_TYPE=Test \ | |
-DUSE_ASDF=ON \ | |
-DUSE_HDF5=ON \ | |
-DHIGHFIVE_PATH=/code/build/HighFive \ | |
-DUSE_LAPACK=ON \ | |
-DBLAZE_PATH=/code/build/blaze \ | |
-DARCH=haswell \ | |
-DNO_MEMLOCK=ON \ | |
-DUSE_OMP=ON \ | |
-DWITH_TESTS=ON \ | |
-DCCACHE=ON .. | |
docker run "${OPTS[@]}" make -j 2 | |
docker run "${OPTS[@]}" ccache -s | |
# 1804 Tests | |
# Build the docker image | |
build-docker-1804: | |
runs-on: ubuntu-latest | |
permissions: write-all | |
steps: | |
- name: Checkout repository | |
uses: actions/checkout@v3 | |
- name: Check for file changes | |
uses: tj-actions/changed-files@v37 | |
id: changed-files | |
- name: Login to GitHub Container Registry | |
uses: docker/login-action@v2 | |
with: | |
registry: ${{ env.REGISTRY }} | |
username: ${{ github.actor }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
# Rebuild (or build) for a push/pr from a local branch (separate workflow for PR merges) | |
- name: Check to see if a docker rebuild is needed | |
id: docker_img | |
run: | | |
echo "docker_build=" >> $GITHUB_OUTPUT; # Default to no rebuild | |
if [[ ${{ contains(steps.changed-files.outputs.all_modified_files, 'tools/docker/18.04/Dockerfile') }} == "true" || ${DOCKER_BUILD} ]] ; then | |
echo "Modified Dockerfile found (or re-build requested). Building on local branch." | |
if [ "${GITHUB_EVENT_NAME}" = "push" ]; then | |
IMAGE_NAME="${IMG_1804}:$(echo ${GITHUB_REF_NAME} | tr / .)" | |
else | |
IMAGE_NAME="${IMG_1804}:$(echo ${GITHUB_HEAD_REF} | tr / .)" | |
fi | |
echo "docker_build=1" >> $GITHUB_OUTPUT; | |
else | |
echo "Dockerfile was not modified. Using develop branch image." | |
IMAGE_NAME="${IMG_1804}:develop" | |
fi | |
if docker manifest inspect ${IMAGE_NAME} >/dev/null; then | |
echo "Docker package found." | |
else | |
echo "Docker package NOT found. Setting docker_build=1." | |
echo "docker_build=1" >> $GITHUB_OUTPUT; | |
fi | |
echo "img_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT; | |
echo "Image name: ${IMAGE_NAME}" | |
- name: Output environment variable details | |
run: | | |
echo "The job_id is: $GITHUB_JOB" | |
echo "The id of this action is: $GITHUB_ACTION" | |
echo "The run id is: $GITHUB_RUN_ID" | |
echo "The repository is: $GITHUB_REPOSITORY" | |
echo "The GitHub Actor's username is: $GITHUB_ACTOR" | |
echo "GitHub SHA: $GITHUB_SHA" | |
echo "GitHub event name: $GITHUB_EVENT_NAME" | |
echo "GitHub Ref: $GITHUB_REF" | |
echo "GitHub Head Ref: $GITHUB_HEAD_REF" | |
echo "GitHub Ref Name: $GITHUB_REF_NAME" | |
echo "Docker 1804 image: $IMG_1804" | |
echo "Docker 2204 image: $IMG_2204" | |
echo "Docker IWYU image: $IMG_IWYU" | |
echo "Used docker image name: ${{ steps.docker_img.outputs.img_name }}" | |
echo "docker_build: ${{ steps.docker_img.outputs.docker_build }}" | |
echo "img_name: ${{ steps.docker_img.outputs.img_name }}" | |
- name: Build and Push Docker Image | |
if: steps.docker_img.outputs.docker_build | |
run: | | |
echo "Building and pushing ${{ steps.docker_img.outputs.img_name }}" | |
docker build --cache-from=${IMG_1804} -t "${{ steps.docker_img.outputs.img_name }}" - < tools/docker/18.04/Dockerfile | |
docker push "${{ steps.docker_img.outputs.img_name }}" | |
outputs: | |
docker_img_name: "${{ steps.docker_img.outputs.img_name }}" | |
# Build a basic version of kotekan | |
build-base-1804: | |
runs-on: ubuntu-latest | |
needs: build-docker-1804 | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Login to GitHub Container Registry | |
uses: docker/login-action@v2 | |
with: | |
registry: ${{ env.REGISTRY }} | |
username: ${{ github.actor }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- name: Cache ccache files | |
uses: actions/cache@v3 | |
with: | |
path: .ccache | |
key: ccache-base-build-1804-${{ github.sha }} | |
restore-keys: | | |
ccache-base-build-1804 | |
ccache-full-build-1804 | |
- name: Build kotekan | |
run: | | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan/build-1804/ ${{ needs.build-docker-1804.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" ccache -s | |
docker run "${OPTS[@]}" \ | |
cmake -Wdev -Werror=dev -Wdeprecated -Werror=deprecated \ | |
-DWERROR=ON \ | |
-DCMAKE_LINK_WHAT_YOU_USE=ON \ | |
-DCMAKE_BUILD_TYPE=Test \ | |
-DARCH=haswell \ | |
-DCCACHE=ON \ | |
-DWITH_TESTS=ON .. | |
docker run "${OPTS[@]}" make -j 2 | |
docker run "${OPTS[@]}" ccache -s | |
# Build a full version of kotekan and run the unit tests | |
build-full-tests-1804: | |
runs-on: ubuntu-latest | |
needs: build-docker-1804 | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Free disk space | |
if: ${{ !env.ACT }} # Don't free up if running locally | |
run: | | |
df -h | |
docker rmi $(docker image ls -aq) | |
df -h | |
sudo apt-get clean | |
sudo apt-get autoclean | |
df -h | |
- name: Cache ccache files | |
uses: actions/cache@v1 | |
with: | |
path: .ccache | |
key: ccache-full-build-1804-${{ github.sha }} | |
restore-keys: | | |
ccache-full-build-1804 | |
ccache-base-build-1804 | |
- name: Login to GitHub Container Registry | |
uses: docker/login-action@v2 | |
with: | |
registry: ${{ env.REGISTRY }} | |
username: ${{ github.actor }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- name: Build kotekan | |
run: | | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan/build-1804/ ${{ needs.build-docker-1804.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" ccache -s | |
docker run "${OPTS[@]}" \ | |
cmake -Wdev -Werror=dev -Wdeprecated -Werror=deprecated \ | |
-DWERROR=ON \ | |
-DCMAKE_LINK_WHAT_YOU_USE=ON \ | |
-DCMAKE_BUILD_TYPE=Test \ | |
-DUSE_HDF5=ON -DHIGHFIVE_PATH=/code/build/HighFive \ | |
-DUSE_LAPACK=ON -DBLAZE_PATH=/code/build/blaze \ | |
-DARCH=haswell \ | |
-DNO_MEMLOCK=ON \ | |
-DUSE_OMP=ON \ | |
-DWITH_BOOST_TESTS=ON \ | |
-DCCACHE=ON .. | |
docker run "${OPTS[@]}" make -j 2 | |
docker run "${OPTS[@]}" rm -r lib | |
docker run "${OPTS[@]}" ccache -s | |
df -h | |
- name: Run parallel python tests | |
run: | | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan/tests/ -e PYTHONPATH=/code/kotekan/python/ ${{ needs.build-docker-1804.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" pytest -v -n auto --dist=loadfile --timeout=${PYTEST_TIMEOUT} -x -m 'not serial' | |
- name: Run serial python tests | |
run: | | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan/tests/ -e PYTHONPATH=/code/kotekan/python/ ${{ needs.build-docker-1804.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" bash -c "redis-server --daemonize yes; pytest -v -x --timeout=${PYTEST_TIMEOUT} -m serial" | |
- name: Run boost tests | |
run: | | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan/build-1804/tests/ -e PYTHONPATH=/code/kotekan/python/ ${{ needs.build-docker-1804.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" pytest -v -x --timeout=${PYTEST_TIMEOUT} | |
# Build a full CHIME version of kotekan | |
build-chime-1804: | |
runs-on: ubuntu-latest | |
needs: build-docker-1804 | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Cache ccache files | |
uses: actions/cache@v1 | |
with: | |
path: .ccache | |
key: ccache-chime-build-1804-${{ github.sha }} | |
restore-keys: | | |
ccache-chime-build-1804 | |
ccache-full-build-1804 | |
- name: Login to GitHub Container Registry | |
uses: docker/login-action@v2 | |
with: | |
registry: ${{ env.REGISTRY }} | |
username: ${{ github.actor }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- name: Build kotekan | |
run: | | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan/build-1804/ ${{ needs.build-docker-1804.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" ccache -s | |
docker run "${OPTS[@]}" \ | |
cmake -Wdev -Werror=dev -Wdeprecated -Werror=deprecated \ | |
-DWERROR=ON \ | |
-DCMAKE_LINK_WHAT_YOU_USE=ON \ | |
-DCMAKE_BUILD_TYPE=Test \ | |
-DUSE_OLD_DPDK=ON \ | |
-DUSE_HDF5=ON -DHIGHFIVE_PATH=/code/build/HighFive \ | |
-DUSE_LAPACK=ON -DBLAZE_PATH=/code/build/blaze \ | |
-DARCH=haswell \ | |
-DNO_MEMLOCK=ON \ | |
-DUSE_OMP=ON \ | |
-DUSE_HSA=ON \ | |
-DCCACHE=ON .. | |
docker run "${OPTS[@]}" make -j 2 | |
docker run "${OPTS[@]}" ccache -s | |
# Build a full CHIME version with clang | |
build-clang-1804: | |
runs-on: ubuntu-latest | |
needs: build-docker-1804 | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Cache ccache files | |
uses: actions/cache@v1 | |
with: | |
path: .ccache | |
key: ccache-clang-build-1804-${{ github.sha }} | |
restore-keys: | | |
ccache-clang-build-1804 | |
- name: Login to GitHub Container Registry | |
uses: docker/login-action@v2 | |
with: | |
registry: ${{ env.REGISTRY }} | |
username: ${{ github.actor }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- name: Build kotekan | |
env: | |
CC: clang | |
CXX: clang++ | |
run: | | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan/build-1804/ ${{ needs.build-docker-1804.outputs.docker_img_name }}) | |
docker run "${OPTS[@]}" ccache -s | |
docker run "${OPTS[@]}" cmake -Wdev -Werror=dev -Wdeprecated -Werror=deprecated \ | |
-DWERROR=ON \ | |
-DCMAKE_LINK_WHAT_YOU_USE=ON \ | |
-DUSE_OLD_DPDK=ON \ | |
-DCMAKE_BUILD_TYPE=Test \ | |
-DUSE_HDF5=ON \ | |
-DHIGHFIVE_PATH=/code/build/HighFive \ | |
-DUSE_LAPACK=ON \ | |
-DBLAZE_PATH=/code/build/blaze \ | |
-DARCH=haswell \ | |
-DNO_MEMLOCK=ON \ | |
-DUSE_OMP=ON \ | |
-DWITH_TESTS=ON \ | |
-DCCACHE=ON .. | |
docker run "${OPTS[@]}" make -j 2 | |
docker run "${OPTS[@]}" ccache -s | |
# IWYU tests | |
# Build the iwyu docker image | |
build-docker-iwyu: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Check if iwyu Dockerfile changed | |
uses: technote-space/get-diff-action@v4 | |
id: git-diff-docker-iwyu | |
with: | |
PATTERNS: tools/iwyu/docker/18.04/Dockerfile | |
- name: Extract branch name | |
shell: bash | |
# Get either branch name of a push or source branch of PR, replace all '/' with '.': | |
run: > | |
if [ "${GITHUB_EVENT_NAME}" = "push" ]; then | |
echo "branch=$(echo ${GITHUB_REF_NAME} | tr / .)" >> $GITHUB_OUTPUT; | |
else | |
echo "branch=$(echo ${GITHUB_HEAD_REF} | tr / .)" >> $GITHUB_OUTPUT; | |
fi | |
id: extract_branch | |
- name: Pull IWYU docker image | |
run: | | |
docker login ghcr.io -u ${GITHUB_REPOSITORY} -p "${{ secrets.GITHUB_TOKEN }}" | |
docker pull --disable-content-trust ${IMG_IWYU}:${{ steps.extract_branch.outputs.branch }} | |
echo ::set-output name=status::success | |
if: steps.git-diff-docker-iwyu.outputs.diff | |
continue-on-error: true | |
id: pull-docker-iwyu | |
- name: Pull docker image for develop branch | |
run: | | |
docker login ghcr.io -u ${GITHUB_REPOSITORY} -p "${{ secrets.GITHUB_TOKEN }}" | |
docker pull --disable-content-trust ${IMG_IWYU}:develop | |
if: steps.git-diff-docker-iwyu.outputs.diff != '' && steps.pull-docker-iwyu.outputs.status != 'success' | |
- name: Build iwyu docker image | |
run: > | |
docker build --cache-from=rocm/dev-ubuntu-18.04,${IMG_1804},${IMG_IWYU} -t ${IMG_IWYU}:${{ steps.extract_branch.outputs.branch }} - < tools/iwyu/docker/18.04/Dockerfile | |
if: steps.git-diff-docker-iwyu.outputs.diff | |
- name: Push docker image | |
run: | | |
docker push ${IMG_IWYU}:${{ steps.extract_branch.outputs.branch }} | |
if: steps.git-diff-docker-iwyu.outputs.diff | |
iwyu: | |
runs-on: ubuntu-latest | |
needs: build-docker-iwyu | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Check if IWYU Dockerfile changed | |
uses: technote-space/get-diff-action@v4 | |
id: git-diff-docker-iwyu | |
with: | |
PATTERNS: tools/iwyu/docker/18.04/Dockerfile | |
- name: Extract branch name for iwyu docker container | |
shell: bash | |
# Get either branch name of a push or source branch of PR, replace all '/' with '.'. | |
# If Dockerfile didn't change, use 'develop'. | |
run: > | |
if [ -n "${{ steps.git-diff-docker-iwyu.outputs.diff }}" ]; then | |
if [ "${GITHUB_EVENT_NAME}" = "push" ]; then | |
echo "branch=$(echo ${GITHUB_REF_NAME} | tr / .)" >> $GITHUB_OUTPUT; | |
else | |
echo "branch=$(echo ${GITHUB_HEAD_REF} | tr / .)" >> $GITHUB_OUTPUT; | |
fi | |
else | |
echo "branch=develop" >> $GITHUB_OUTPUT; | |
fi | |
id: extract_branch | |
- name: Configure kotekan | |
run: | | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan/build-1804/ ${IMG_IWYU}:${{ steps.extract_branch.outputs.branch }}) | |
docker login ghcr.io -u ${GITHUB_REPOSITORY} -p "${{ secrets.GITHUB_TOKEN }}" | |
docker run "${OPTS[@]}" \ | |
cmake -Wdev -Werror=dev -Wdeprecated -Werror=deprecated \ | |
-DWERROR=ON \ | |
-DCMAKE_LINK_WHAT_YOU_USE=ON \ | |
-DCMAKE_BUILD_TYPE=Test \ | |
-DUSE_OLD_DPDK=ON \ | |
-DUSE_HDF5=ON -DHIGHFIVE_PATH=/code/build/HighFive \ | |
-DUSE_LAPACK=ON -DBLAZE_PATH=/code/build/blaze \ | |
-DARCH=haswell \ | |
-DNO_MEMLOCK=ON \ | |
-DUSE_OMP=ON \ | |
-DWITH_TESTS=ON \ | |
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ | |
-DIWYU=ON \ | |
-DCCACHE=ON .. | |
- name: Run iwyu | |
run: | | |
OPTS=(--rm --mount type=bind,src=$(pwd),target=/code/kotekan -w /code/kotekan/build-iwyu/ ${IMG_IWYU}:${{ steps.extract_branch.outputs.branch }}) | |
docker run "${OPTS[@]}" /code/kotekan/tools/iwyu/docker/iwyu.sh |