SCSB-40 add JWST and RomanCal coverage tests (#55)

* add JWST and RomanCal coverage tests * cache tox envs * reorder `tox.ini` * explicit python * attach python versions to tox envs * minor fix * dependencies in root env * dependencies in child envs * install from extras * add python 3.7 * restore matrices * `astropy` only supports Python >=3.8 * rearrange arguments * add `CRDS_*` variables to tox envs * pass environment variables * reorder ini file * restore coverage report * test direct checkout of relevant pipelines * call `cd` with `bash` * revert previous change * use JWST CRDS server when running `romancal` tests * add `pip freeze` before test command for debugging purposes * update `CRDS_PATH` in workflow * try with public CRDS * pin local install to `0.3.1` * pin local install to `0.3.1` * add duplicate workflow cancel from JWST pipeline * explicitly set version `0.3.1` for `romancal` tests * revert using PUB CRDS (was just for testing) * add to changelog
spacetelescope · Jun 2, 2022 · a781c95 · a781c95
1 parent 04f7d47
commit a781c95
Show file tree

Hide file tree

Showing 4 changed files with 205 additions and 86 deletions.
diff --git a/.github/workflows/cancel_workflows.yml b/.github/workflows/cancel_workflows.yml
@@ -0,0 +1,19 @@
+name: Cancel duplicate workflows
+
+on:
+  workflow_run:
+    workflows: ["CI"]
+    types:
+      - requested
+
+# Note: This has to be in workflow_run so it works for PRs from forks.  And only cancel
+# pull_request triggers, not push, i.e. merges into master.
+jobs:
+  cancel:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.event == 'pull_request' }}
+    steps:
+    - name: Cancel previous runs
+      uses: styfle/cancel-workflow-action@3d86a7cc43670094ac248017207be0295edbc31d  # 0.8.0
+      with:
+        workflow_id: ${{ github.event.workflow.id }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -12,77 +12,129 @@ on:
     # * is a special character in YAML so you have to quote this string
     - cron: '0 9 * * 1'
 
+env:
+  CRDS_PATH: $HOME/crds_cache
+  CRDS_CLIENT_RETRY_COUNT: 3
+  CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
+
 jobs:
-  tox:
+  lint:
     name: ${{ matrix.name }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - toxenv: style
+            name: Code style checks
+          - toxenv: bandit
+            name: Bandit security audit
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v2
+      - uses: actions/cache@v3
+        id: tox_cache
+        with:
+          path: ${{ env.pythonLocation }}
+          key: tox-install-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml', '**/setup.*') }}
+      - run: pip install tox
+        if: steps.tox_cache.outputs.cache-hit != 'true'
+      - uses: actions/cache@v3
+        with:
+          path: .tox
+          key: tox-${{ matrix.toxenv }}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml', '**/setup.*') }}
+      - run: tox -e ${{ matrix.toxenv }}
+  test:
+    name: test
+    needs: [ lint ]
     runs-on: ${{ matrix.runs-on }}
     strategy:
       fail-fast: false
       matrix:
+        runs-on: [ ubuntu-latest, macos-latest ]
+        toxenv: [ py38, py39, py310 ]
         include:
-          - name: Python 3.9
-            runs-on: ubuntu-latest
-            python-version: 3.9
-            toxenv: py39
-
-          - name: Python 3.8
-            runs-on: ubuntu-latest
-            python-version: 3.8
-            toxenv: py38
-
-          - name: Coverage
-            runs-on: ubuntu-latest
-            python-version: 3.8
-            toxenv: coverage
-
-          - name: Code style checks
-            runs-on: ubuntu-latest
-            python-version: 3.8
-            toxenv: style
-
-          - name: Bandit security audit
-            runs-on: ubuntu-latest
-            python-version: 3.8
-            toxenv: bandit
-
-          - name: macOS
-            runs-on: macos-latest
+          - toxenv: py38
             python-version: 3.8
-            toxenv: py38
+          - toxenv: py39
+            python-version: 3.9
+          - toxenv: py310
+            python-version: "3.10"
     steps:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+      - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install tox
-        run: |
-          python -m pip install --upgrade pip
-          pip install tox
-      - name: Run tox
-        run: tox -e ${{ matrix.toxenv }}
-
+      - uses: actions/cache@v3
+        id: tox_cache
+        with:
+          path: ${{ env.pythonLocation }}
+          key: tox-install-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml', '**/setup.*') }}
+      - run: pip install tox
+        if: steps.tox_cache.outputs.cache-hit != 'true'
+      - uses: actions/cache@v3
+        with:
+          path: .tox
+          key: tox-${{ matrix.toxenv }}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml', '**/setup.*') }}
+      - run: tox -e ${{ matrix.toxenv }}
+  test_with_coverage:
+    name: ${{ matrix.name }}
+    needs: [ lint ]
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        toxenv: [ coverage, jwst-cov, romancal-cov ]
+        python-version: [ 3.9 ]
+        include:
+          - toxenv: coverage
+            name: test base library with coverage
+          - toxenv: jwst-cov
+            name: test JWST pipeline with coverage
+            crds-server: https://jwst-crds.stsci.edu
+          - toxenv: romancal-cov
+            name: test Roman pipeline with coverage
+            crds-server: https://roman-crds-test.stsci.edu
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: actions/cache@v3
+        id: tox_cache
+        with:
+          path: ${{ env.pythonLocation }}
+          key: tox-install-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml', '**/setup.*') }}
+      - run: pip install tox
+        if: steps.tox_cache.outputs.cache-hit != 'true'
+      - uses: actions/cache@v3
+        with:
+          path: .tox
+          key: tox-${{ matrix.toxenv }}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml', '**/setup.*') }}
+      - run: tox -e ${{ matrix.toxenv }}
+        env:
+          CRDS_SERVER_URL: ${{ matrix.crds-server }}
+      - uses: codecov/codecov-action@v2
+        with:
+          file: ./coverage.xml
   # Kept in a separate job because it needs extra system dependencies
   # that can't be installed by tox.
   build-docs:
     name: Build documentation and check warnings
+    needs: [ test ]
     runs-on: ubuntu-latest
     steps:
-      - name: Install system packages
-        run: |
-          sudo apt-get install graphviz texlive-latex-extra dvipng
+      - run: sudo apt-get install graphviz texlive-latex-extra dvipng
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+      - uses: actions/setup-python@v2
         with:
-          python-version: 3.8
-      - name: Install tox
-        run: |
-          python -m pip install --upgrade pip
-          pip install tox
-      - name: Run tox
-        run: tox -e build-docs
+          python-version: 3.9
+      - run: pip install tox
+      - run: tox -e build-docs
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -5,6 +5,8 @@
 
 - Update datamodel with 'SKIPPED' status when step.skip set to True [#53]
 
+- Update CI workflows to cache test environments and depend upon style and security checks [#55]
+
 0.3.3 (2022-04-07)
 ==================
 

diff --git a/tox.ini b/tox.ini
@@ -1,48 +1,94 @@
-[tox]
-envlist= py38,style,bandit
-
-[testenv]
-extras= test
+[testtenv]
+extras = test
 # astropy will complain if the home directory is missing
-passenv= HOME
-commands=
-    pytest
-
-[testenv:egg_info]
-deps=
-commands=
-    python setup.py egg_info
-
-[testenv:twine]
-deps=
-    twine
-commands=
-    twine check {distdir}/*
+passenv =
+    HOME
+deps =
+    pytest-xdist
+commands =
+    pip freeze
+    pytest -n auto
 
 [testenv:style]
-deps=
+deps =
     flake8
-commands=
+commands =
     flake8 --count src
 
-[testenv:coverage]
-deps=
-    codecov
-    coverage
-commands=
-    coverage run --source=stpipe -m pytest
-    coverage report -m
-    codecov -e TOXENV
-passenv= TOXENV CI TRAVIS TRAVIS_* CODECOV_* DISPLAY HOME
-
 [testenv:bandit]
-deps=
+deps =
     bandit
-commands=
+commands =
     bandit -r -ll src
 
 [testenv:build-docs]
-basepython= python3.8
-extras= docs
-commands=
+basepython = python3.8
+extras = docs
+commands =
     sphinx-build -W docs/source build/docs
+
+[testenv:coverage]
+basepython = python3.9
+extras = test
+deps =
+    codecov
+    pytest-cov
+    pytest-xdist
+passenv =
+    CI
+    CODECOV_*
+    CRDS_*
+    DISPLAY
+    HOME
+    TOXENV
+    TRAVIS
+    TRAVIS_*
+commands =
+    pip freeze
+    pytest -n auto --cov-report=xml --cov={toxinidir}/src/stpipe
+    coverage report -m
+
+[testenv:jwst-cov]
+basepython = python3.9
+extras = test
+deps =
+    codecov
+    jwst[test] @ git+https://github.com/spacetelescope/jwst.git
+    pytest-cov
+    pytest-xdist
+passenv =
+    CI
+    CODECOV_*
+    CRDS_*
+    DISPLAY
+    HOME
+    TOXENV
+    TRAVIS
+    TRAVIS_*
+commands =
+    pip freeze
+    pytest -n auto --cov-report=xml --cov={toxinidir}/src/stpipe --ignore-glob=timeconversion --ignore-glob=associations --pyargs {posargs:jwst}
+    coverage report -m
+
+[testenv:romancal-cov]
+basepython = python3.9
+extras = test
+deps =
+    codecov
+    pytest-cov
+    pytest-xdist
+    romancal[test] @ git+https://github.com/spacetelescope/romancal.git
+passenv =
+    CI
+    CODECOV_*
+    CRDS_*
+    DISPLAY
+    HOME
+    TOXENV
+    TRAVIS
+    TRAVIS_*
+commands =
+    pip install -U stpipe==0.3.1
+    pip freeze
+    pytest -n auto --cov-report=xml --cov={toxinidir}/src/stpipe --pyargs {posargs:romancal}
+    coverage report -m