migration from Travis CI to GitHub Actions (#30)

* commit changes beginning migration from Travis CI to GitHub Actions * update GitHub cache action update GitHub cache action (v2 -> v3) https://github.com/marketplace/actions/cache in connection with deprecation of the save-state command ( https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/ ) * update docker login action v1 -> v2 * bump docker/setup-buildx-action v1 -> v2 * update GitHub checkout action v2 -> v3 * update GitHub setup-python action v2 -> v4 * update docker/setup-buildx-action@v1 -> v2 in docs build action * bump ReadTheDocs python version 3.6 -> 3.8 * update viral-baseimage to 2.2.0 containing upgrade from Ubuntu 18.04 LTS (bionic) to Ubuntu 22.04 LTS (jammy) * remove Travis CI-related files * update README build status badge to reflect GitHub Actions state * change gap2seq version to gap2seq 3.1.1a to reflect new build on bioconda; bump viral-core to 2.2.1
broadinstitute · Mar 3, 2023 · 750446f · 750446f
1 parent cfee741
commit 750446f
Show file tree

Hide file tree

Showing 12 changed files with 278 additions and 129 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -0,0 +1,245 @@
+name: "viral-assemble CI"
+
+on:
+  push:
+  pull_request:
+    branches:
+      - master
+  release:
+    types:
+      - created
+  schedule:
+    - cron: '0 18 * * 1' # weekly at 18:00 on Mondays
+
+env:
+  HOME: "${{ github.workspace }}"
+  CACHE_DIR: "${{ github.workspace }}/misc_cache"
+  MINICONDA_DIR: "${{ github.workspace }}/miniconda"
+  GATK_PATH: "${{ github.workspace }}/misc_cache"
+  PYTHONIOENCODING: UTF8
+
+  DOCKER_REGISTRY: "quay.io"
+  DOCKER_REPO_PROD: "quay.io/broadinstitute/viral-assemble"
+  DOCKER_REPO_DEV: "quay.io/broadinstitute/viral-assemble"
+
+  #BOTO_CONFIG: /dev/null # bogus value to override config on travis
+  _JAVA_OPTIONS: "-Xmx3g" # overrides jvm opts set elsewhere; see: https://stackoverflow.com/questions/28327620/difference-between-java-options-java-tool-options-and-java-opts
+
+  # TravisCI variables described here:
+  #   https://docs.travis-ci.com/user/environment-variables/
+  # GitHub Actions environment variables and context described here:
+  #   https://docs.github.com/en/actions/reference/environment-variables
+  #   https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#env-context
+  GITHUB_ACTIONS_COMMIT: ${{ github.sha }}
+  GITHUB_ACTIONS_BUILD_DIR: ${{ github.workspace }}
+  #GITHUB_ACTIONS_BRANCH: ${{ github.base_ref }}
+  GITHUB_ACTIONS_BRANCH: ${{ github.ref }}
+  GITHUB_ACTIONS_PULL_REQUEST: ${{ github.event.number }}
+  GITHUB_ACTIONS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
+  GITHUB_ACTIONS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
+  GITHUB_ACTIONS_BASE_REF: ${{ github.base_ref }}
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  build_docker:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: checkout repository
+        uses: actions/checkout@v3
+      # fetch git tags (tagged releases) because
+      # actions/checkout@v3 does either a full checkout or a shallow checkout without tags
+      - name: fetch tags
+        run: git fetch --prune --unshallow --tags
+      - name: Programmatic environment setup
+        run: |
+          set -e -x
+          # $GITHUB_ENV is available for subsequent steps
+          GITHUB_ACTIONS_TAG=$(git describe --tags --exact-match && sed 's/^v//g' || echo '')
+          echo "GITHUB_ACTIONS_TAG=$GITHUB_ACTIONS_TAG" >> $GITHUB_ENV
+          # 
+          # Set GITHUB_ACTIONS_BRANCH
+          # TRAVIS_BRANCH: (via https://docs.travis-ci.com/user/environment-variables/ )
+          #   for push builds, or builds not triggered by a pull request, this is the name of the branch.
+          #   for builds triggered by a pull request this is the name of the branch targeted by the pull request.
+          #   for builds triggered by a tag, this is the same as the name of the tag (TRAVIS_TAG).
+          # if GITHUB_ACTIONS_PULL_REQUEST_BRANCH is set, this is a pull request build
+          if [[ $GITHUB_ACTIONS_PULL_REQUEST_BRANCH ]]; then
+            GITHUB_ACTIONS_BRANCH=${GITHUB_ACTIONS_BASE_REF##*/}
+          # if event_name=="release", this is a tagged build
+          elif [[ "${{ github.event_name }}" == "release" ]]; then
+            GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_TAG
+          else
+            GITHUB_ACTIONS_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+          fi
+          echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH"
+          echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV
+
+      - name: Login to docker registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.DOCKER_REGISTRY }}
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_ROBOT_TOKEN }}
+      # cache; see: https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows
+      - name: setup cache
+        id: docker_cache
+        uses: actions/cache@v3
+        env:
+          cache-name: old-docker-tag
+        with:
+          path: "$CACHE_DIR"
+          key: ${{ runner.os }}-${{ env.cache-name }}
+      # - name: Pull old docker image from cache
+      #   if: steps.docker_cache.outputs.cache-hit != 'true'
+      #   run: |
+      - name: Attempt to pull older image from cache
+        if: steps.docker_cache.outputs.cache-hit == 'true'
+        shell: bash
+        run: |
+          set -e
+          # restore old tag from cache if present
+          if [ -f "$CACHE_DIR/old-docker-tag.txt" ]; then 
+            OLD_DOCKER_TAG=$(cat $CACHE_DIR/old-docker-tag.txt)
+          else 
+            OLD_DOCKER_TAG=$DOCKER_REPO_PROD
+          fi 
+          echo "old docker tag = $OLD_DOCKER_TAG"
+
+          # attempt to pull tag from cache
+          if docker pull $OLD_DOCKER_TAG; then
+            echo _CACHE_FROM="--cache-from $OLD_DOCKER_TAG" >> $GITHUB_ENV
+          else
+            echo "_CACHE_FROM=" >> $GITHUB_ENV
+          fi
+      - name: Build docker image
+        shell: bash
+        run: |
+          set -e -x
+          git describe --tags --always | tee VERSION
+
+          if [ -n "$GITHUB_ACTIONS_TAG" ]; then
+            echo "Release $GITHUB_ACTIONS_TAG"
+          elif [ -n "$GITHUB_ACTIONS_PULL_REQUEST_BRANCH" ]; then
+            echo "LABEL quay.expires-after=10w" >> Dockerfile
+          elif [[ "$GITHUB_ACTIONS_BRANCH" != "master" ]]; then
+            echo "LABEL quay.expires-after=10w" >> Dockerfile
+          fi
+
+          echo "Building with cache from: $_CACHE_FROM"
+          docker build -t local/build-container:build $_CACHE_FROM .
+      - name: Deploy docker image
+        run: |
+          github_actions_ci/deploy-docker.sh
+      - name: store tag ID in cache
+        run: |
+          mkdir -p $CACHE_DIR
+          github_actions_ci/list-docker-tags.sh | tail -1 | tee $CACHE_DIR/old-docker-tag.txt
+
+  test_py36_in_docker:
+    needs: build_docker
+    runs-on: ubuntu-20.04
+    env:
+      GITHUB_ACTIONS_PYTHON_VERSION: 3.8
+      PYTEST_ADDOPTS: "-rsxX -n 2 --durations=25 --fixture-durations=10 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov read_utils --cov reports --cov tools --cov util --cov file_utils"
+    steps:
+      - name: checkout repository
+        uses: actions/checkout@v3
+      # fetch git tags (tagged releases) because 
+      # actions/checkout@v3 does either a full checkout or a shallow checkout without tags
+      - name: fetch tags
+        run: git fetch --prune --unshallow --tags
+      - name: Programmatic environment setup
+        run: |
+          set -e -x
+          # $GITHUB_ENV is available for subsequent steps
+          GITHUB_ACTIONS_TAG=$(git describe --tags --exact-match && sed 's/^v//g' || echo '')
+          echo "GITHUB_ACTIONS_TAG=$GITHUB_ACTIONS_TAG" >> $GITHUB_ENV
+          # 
+          # Set GITHUB_ACTIONS_BRANCH
+          # TRAVIS_BRANCH: (via https://docs.travis-ci.com/user/environment-variables/ )
+          #   for push builds, or builds not triggered by a pull request, this is the name of the branch.
+          #   for builds triggered by a pull request this is the name of the branch targeted by the pull request.
+          #   for builds triggered by a tag, this is the same as the name of the tag (TRAVIS_TAG).
+          # if GITHUB_ACTIONS_PULL_REQUEST_BRANCH is set, this is a pull request build
+          if [[ $GITHUB_ACTIONS_PULL_REQUEST_BRANCH ]]; then
+            GITHUB_ACTIONS_BRANCH=${GITHUB_ACTIONS_BASE_REF##*/}
+          # if event_name=="release", this is a tagged build
+          elif [[ "${{ github.event_name }}" == "release" ]]; then
+            GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_TAG
+          else
+            GITHUB_ACTIONS_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+          fi
+          echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH"
+          echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV
+      - name: install python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "${{ env.GITHUB_ACTIONS_PYTHON_VERSION }}"
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: pull docker image
+        run: |
+          set -e -x
+          DOCKER_TAG=$(github_actions_ci/list-docker-tags.sh | tail -1)
+          echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
+          echo "pulling $DOCKER_TAG"
+          docker pull $DOCKER_TAG
+          mkdir coverage
+      - name: test with docker
+        run: |
+          docker run -e _JAVA_OPTIONS -e PYTEST_ADDOPTS -v `pwd`/coverage:/coverage -v `pwd`/test:/opt/viral-ngs/source/test --entrypoint /bin/bash $DOCKER_TAG -c 'set -e; cd /opt/viral-ngs/source; pytest test/unit; cp .coverage /coverage'
+      - name: run coveralls
+        run: |
+          mv coverage/.coverage .
+          pip install coveralls>=1.3.0
+          coveralls --service=github
+
+  ## note: this test_docs job does not actually produce the output on readthedocs
+  ## readthedocs does its own build trigger. this job exists simply to alert us
+  ## of build failures of the docs because otherwise we would never know.
+  test_docs:
+    needs: build_docker
+    runs-on: ubuntu-20.04
+    steps:
+      - name: checkout repository
+        uses: actions/checkout@v3
+      # fetch git tags (tagged releases) because 
+      # actions/checkout@v3 does either a full checkout or a shallow checkout without tags
+      - name: fetch tags
+        run: git fetch --prune --unshallow --tags
+      - name: Programmatic environment setup
+        run: |
+          set -e -x
+          # $GITHUB_ENV is available for subsequent steps
+          GITHUB_ACTIONS_TAG=$(git describe --tags --exact-match && sed 's/^v//g' || echo '')
+          echo "GITHUB_ACTIONS_TAG=$GITHUB_ACTIONS_TAG" >> $GITHUB_ENV
+          # 
+          # Set GITHUB_ACTIONS_BRANCH
+          # TRAVIS_BRANCH: (via https://docs.travis-ci.com/user/environment-variables/ )
+          #   for push builds, or builds not triggered by a pull request, this is the name of the branch.
+          #   for builds triggered by a pull request this is the name of the branch targeted by the pull request.
+          #   for builds triggered by a tag, this is the same as the name of the tag (TRAVIS_TAG).
+          # if GITHUB_ACTIONS_PULL_REQUEST_BRANCH is set, this is a pull request build
+          if [[ $GITHUB_ACTIONS_PULL_REQUEST_BRANCH ]]; then
+            GITHUB_ACTIONS_BRANCH=${GITHUB_ACTIONS_BASE_REF##*/}
+          # if event_name=="release", this is a tagged build
+          elif [[ "${{ github.event_name }}" == "release" ]]; then
+            GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_TAG
+          else
+            GITHUB_ACTIONS_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+          fi
+          echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH"
+          echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: pull docker image
+        run: |
+          set -e -x
+          DOCKER_TAG=$(github_actions_ci/list-docker-tags.sh | tail -1)
+          echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
+          echo "pulling $DOCKER_TAG"
+          docker pull $DOCKER_TAG
+      - name: test building docs
+        run: |
+          set -e -x
+          docker run --entrypoint /bin/bash -v `pwd`:/opt/viral-ngs/source $DOCKER_TAG -c 'set -e; cd /opt/viral-ngs/source; github_actions_ci/install-pip-docs.sh; cd docs; make html'
diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -7,6 +7,7 @@ version: 2
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
+  builder: html
   configuration: docs/conf.py
 
 # Build documentation with MkDocs
@@ -18,6 +19,6 @@ formats: all
 
 # Optionally set the version of Python and requirements required to build your docs
 python:
-  version: 3.6
+  version: 3.8
   install:
     - requirements: docs/requirements.txt
diff --git a/.travis.yml b/.travis.yml
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM quay.io/broadinstitute/viral-core:2.1.33
+FROM quay.io/broadinstitute/viral-core:2.2.1
 
 LABEL maintainer "viral-ngs@broadinstitute.org"
 

diff --git a/README.md b/README.md
@@ -1,5 +1,5 @@
 [![Docker Repository on Quay](https://quay.io/repository/broadinstitute/viral-assemble/status "Docker Repository on Quay")](https://quay.io/repository/broadinstitute/viral-assemble)
-[![Build Status](https://travis-ci.com/broadinstitute/viral-assemble.svg?branch=master)](https://travis-ci.com/broadinstitute/viral-assemble)
+[![Build Status](https://github.com/broadinstitute/viral-assemble/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/broadinstitute/viral-assemble/actions)
 [![Documentation Status](https://readthedocs.org/projects/viral-assemble/badge/?version=latest)](https://viral-assemble.readthedocs.io/en/latest/?badge=latest)
 <!--
 [![broad-viral-badge](https://img.shields.io/badge/install%20from-broad--viral-green.svg?style=flat-square)](https://anaconda.org/broad-viral/viral-ngs)

diff --git a/docs/conf.py b/docs/conf.py
@@ -76,7 +76,7 @@ def _get_viral_core():
 
 # General information about the project.
 project = 'viral-ngs'
-copyright = '2015, Broad Institute Viral Genomics'
+copyright = '2023, Broad Institute Viral Genomics'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -1,7 +1,8 @@
-Sphinx>=5.3.0
-sphinx-jinja2-compat
-sphinx-argparse>=0.1.15
-sphinx_rtd_theme>=0.1.9
-matplotlib>=2.2.4
-#PyYAML==5.1
-mock>=2.0.0
+jinja2==3.1.2 # https://github.com/readthedocs/readthedocs.org/issues/9037#issuecomment-1077818554
+Sphinx==5.3.0 #override sphinx pinning done by RTD: https://docs.readthedocs.io/en/stable/build-default-versions.html#external-dependencies
+sphinx-argparse
+sphinx-rtd-theme==1.1.1
+matplotlib==2.2.4
+PyYAML==6.0
+mock==5.0.1
+recommonmark
diff --git a/github_actions_ci/deploy-docker.sh b/github_actions_ci/deploy-docker.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e -x -o pipefail
+
+echo "Deploying docker image to $DOCKER_REGISTRY"
+# tag and deploy
+for TAG in `github_actions_ci/list-docker-tags.sh`; do
+	echo "Pushing docker image to docker registry as $TAG"
+	docker tag local/build-container:build $TAG
+	docker push $TAG # ToDo: uncomment when tag is parsed correctly.
+done
diff --git a/travis/install-pip-docs.sh → github_actions_ci/install-pip-docs.sh b/travis/install-pip-docs.sh → github_actions_ci/install-pip-docs.sh