[EXP] Proof of concept of a new way to do regression test (#71)

* Added one regression test as proof of concept

* Improve exe detection in test helper [skip ci]

* Improvements for test mechanisms.

Adjusted Travis recipe for remote data. Disabled Mac OSX remote data test on Travis because it takes too long.

Test now checks central storage first if available and handle undefined xREF.

Added base classes for test suits to reduce code redundancy.

Added tests for test helpers.

Added PoC tests for ACS/HRC, ACS/SBC, STIS/FUV, and WFC3/IR.

* Updated tests to account for upstream changes

* Add HTTP access to CDBS downloads

* Add Jenkinsfile

* Fix CI test failures

* Use pytest-remotedata plugin. Add slow marker. [ci skip]

* Move all Linux builds to Jenkins. Only run data tests on one of the Jenkins build. Joe also parallelized Jenkins tests. [ci skip]

* Run XUnitBuilder no matter what

* Set Jenkins not to fail with skipped tests.

* Don't run XUnitBuilder for build-only jobs

* Use shared utils for Jenkinsfile.

* Generalize config execution function name.

* Change 'build_mode' to 'name' to reflect CI utils update.

* TST: Break ACS tests into many files

* TST: Fix Jenkins broken pipe

* TST: Use ci_watson plugin

TST: Use ci_watson feature branch for now

* POC overhaul to use ci-watson and Artifactory.

* Jenkins download ref files instead of direct access

* Why no slash

* Fix Jenkins file not found. Remove Linux builds from Travis. Remove unnecessary Mac builds from Travis. Add Jenkins badge to README.

* Ignore ver and history for WFC3.
Increase rtol for STIS.

* EXP: Disable all PCTECORR tests for now. Run STIS with default rtol again.

* Remove slow marker to let ONE PTCECORR test run on push event

* UNDO THIS: Try RT run here and see

* Add helper function to fix keywords

* RT: Say no to Gandalf

* POC: Undo RT check. This is it. Booyah.

* RT use ci-watson from master [skip ci]

* Jenkins use ci-watson from master

* Use released version of ci-watson

* Joe does not want me to kill all cores

* DOC: Fix verbiage of envopt

.gitignore

@@ -3,3 +3,5 @@
 bin.*
 build.*
 Makefile
+__pycache__
+.cache

.travis.yml

@@ -16,31 +16,6 @@ matrix:
   fast_finish: true
 
   include:
-  - os: linux
-    compiler: gcc
-    env:
-      - PYTHON_VERSION=2.7
-
-  - os: linux
-    compiler: gcc
-    env:
-      - PYTHON_VERSION=3.5
-
-  - os: linux
-    compiler: gcc
-
-  - os: osx
-    compiler: clang
-    env:
-      - PYTHON_VERSION=2.7
-      - CC="clang"
-
-  - os: osx
-    compiler: clang
-    env:
-      - PYTHON_VERSION=3.5
-      - CC="clang"
-
   - os: osx
     compiler: clang
     env:

Jenkinsfile

@@ -0,0 +1,49 @@
+// Obtain files from source control system.
+if (utils.scm_checkout()) return
+
+// Config data to share between builds.
+CFLAGS = ''
+LDFLAGS = ''
+DEFAULT_FLAGS = "${CFLAGS} ${LDFLAGS}"
+// Some waf flags cause a prompt for input during configuration, hence the 'yes'.
+configure_cmd = "yes '' | ./waf configure --prefix=./_install ${DEFAULT_FLAGS}"
+
+
+// Define each build configuration, copying and overriding values as necessary.
+bc0 = new BuildConfig()
+bc0.nodetype = "linux-stable"
+bc0.build_mode = "debug"
+bc0.env_vars = ['PATH=./_install/bin:$PATH']
+bc0.build_cmds = ["conda config --add channels http://ssb.stsci.edu/astroconda",
+                  "conda install -q -y cfitsio pkg-config",
+                  "${configure_cmd} --debug",
+                  "./waf build",
+                  "./waf install",
+                  "calacs.e --version"]
+
+bc1 = utils.copy(bc0)
+bc1.build_mode = "release"
+// Would be nice if Jenkins can access /grp/hst/cdbs/xxxx directly.
+bc1.env_vars = ['PATH=./_install/bin:$PATH',
+                'OMP_NUM_THREADS=8',
+                'TEST_BIGDATA=https://bytesalad.stsci.edu/artifactory/scsb-hstcal']
+bc1.build_cmds = ["conda config --add channels http://ssb.stsci.edu/astroconda",
+                  "conda install -q -y cfitsio pkg-config pytest requests astropy",
+                  "pip install ci-watson",
+                  "${configure_cmd} --release-with-symbols",
+                  "./waf build",
+                  "./waf install",
+                  "calacs.e --version"]
+bc1.test_cmds = ["pytest tests --basetemp=tests_output --junitxml results.xml --bigdata -v"]
+bc1.failedUnstableThresh = 1
+bc1.failedFailureThresh = 6
+
+
+bc2 = utils.copy(bc0)
+bc2.build_mode = "optimized"
+bc2.build_cmds[2] = "${configure_cmd} --O3"
+
+
+// Iterate over configurations that define the (distibuted) build matrix.
+// Spawn a host of the given nodetype for each combination and run in parallel.
+utils.run([bc0, bc1, bc2])

JenkinsfileRT

@@ -0,0 +1,36 @@
+// Obtain files from source control system.
+// [skip ci] and [ci skip] have no effect here.
+if (utils.scm_checkout(['skip_disable':true])) return
+
+// Config data to share between builds.
+CFLAGS = ''
+LDFLAGS = ''
+DEFAULT_FLAGS = "${CFLAGS} ${LDFLAGS}"
+// Some waf flags cause a prompt for input during configuration, hence the 'yes'.
+configure_cmd = "yes '' | ./waf configure --prefix=./_install ${DEFAULT_FLAGS}"
+
+
+// Run nightly tests, which include the slow ones.
+bc1 = new BuildConfig()
+bc1.nodetype = "linux-stable"
+bc1.build_mode = "release"
+bc1.env_vars = ['PATH=./_install/bin:$PATH',
+                'OMP_NUM_THREADS=8',
+                'TEST_BIGDATA=https://bytesalad.stsci.edu/artifactory/scsb-hstcal',
+                'jref=/grp/hst/cdbs/jref/',
+                'iref=/grp/hst/cdbs/iref/',
+                'oref=/grp/hst/cdbs/oref/']
+bc1.build_cmds = ["conda config --add channels http://ssb.stsci.edu/astroconda",
+                  "conda install -q -y cfitsio pkg-config pytest requests astropy",
+                  "pip install ci-watson",
+                  "${configure_cmd} --release-with-symbols",
+                  "./waf build",
+                  "./waf install",
+                  "calacs.e --version"]
+bc1.test_cmds = ["pytest tests --basetemp=tests_output --junitxml results.xml --bigdata --slow -v"]
+bc1.failedUnstableThresh = 1
+bc1.failedFailureThresh = 6
+
+// Iterate over configurations that define the (distibuted) build matrix.
+// Spawn a host of the given nodetype for each combination and run in parallel.
+utils.run([bc1])

README.md

@@ -1,6 +1,8 @@
 # HSTCAL
-[![Build Status](https://travis-ci.org/spacetelescope/hstcal.svg?branch=master)](https://travis-ci.org/spacetelescope/hstcal)
 
+[![Travis CI](https://travis-ci.org/spacetelescope/hstcal.svg?branch=master)](https://travis-ci.org/spacetelescope/hstcal)
+
+[![Jenkins CI](https://ssbjenkins.stsci.edu/job/STScI/job/hstcal/job/master/badge/icon)](https://ssbjenkins.stsci.edu/job/STScI/job/hstcal/job/master/)
 
 Calibration software for HST/WFC3, HST/ACS, and HST/STIS.
 

tests/__init__.py

@@ -0,0 +1,29 @@
+"""
+This directory contains HSTCAL regression tests, which are written in
+Python but call the C executables.
+
+These environment variables must be set up prior::
+
+    export BIG_DATA=/path/to/bigdata/scsb_hstcal
+    export jref=/grp/hst/cdbs/jref/
+    export iref=/grp/hst/cdbs/iref/
+    export oref=/grp/hst/cdbs/oref/
+
+Example::
+
+    pytest tests --big-data [--slow] [-sv] [--basetemp=/my/tmp]
+
+"""
+
+import sys
+
+__minimum_python_version__ = '3.5'
+
+
+class UnsupportedPythonError(Exception):
+    pass
+
+
+# This is the same check as the one at the top of setup.py
+if sys.version_info < tuple((int(val) for val in __minimum_python_version__.split('.'))):  # noqa
+    raise UnsupportedPythonError("HSTCAL tests do not support Python < {}".format(__minimum_python_version__))  # noqa

tests/acs/test_hrc.py

@@ -0,0 +1,33 @@
+"""Tests for ACS/HRC."""
+
+import subprocess
+
+from ..helpers import BaseACS
+
+
+class TestMosaicBox(BaseACS):
+    """
+    Process an HRC mosaic dataset using the standard HRC-MOSAIC-BOX pattern
+    with CR-SPLIT=1 at each of the 4 dither positions.
+
+    .. note:: This was ``hrc_asn1``.
+
+    """
+    detector = 'hrc'
+
+    def test_4point_mosaic(self):
+        rootname = 'j6m901020'
+        asn_file = rootname + '_asn.fits'
+
+        # Prepare input files.
+        self.get_input_file(asn_file)
+
+        # Run CALACS
+        subprocess.call(['calacs.e', asn_file, '-v'])
+
+        # Compare results
+        outputs = [('j6m901bzq_flt.fits', 'j6m901bzq_flt_ref.fits'),
+                   ('j6m901c3q_flt.fits', 'j6m901c3q_flt_ref.fits'),
+                   ('j6m901d9q_flt.fits', 'j6m901d9q_flt_ref.fits'),
+                   ('j6m901deq_flt.fits', 'j6m901deq_flt_ref.fits')]
+        self.compare_outputs(outputs)

tests/acs/test_sbc.py

@@ -0,0 +1,31 @@
+"""Tests for ACS/SBC."""
+import subprocess
+import pytest
+
+from ..helpers import BaseACS
+
+
+class TestSingleFrame(BaseACS):
+    """Process a single SBC dataset."""
+    detector = 'sbc'
+
+    # NOTE:
+    # j9ic01vpq = pre-SM4, was sbc_single1
+    # jbdf10ykq = post-SM4, was sbc_single2
+    @pytest.mark.parametrize(
+        'rootname',
+        ['j9ic01vpq',
+         'jbdf10ykq'])
+    def test_single_frame(self, rootname):
+        raw_file = '{}_raw.fits'.format(rootname)
+
+        # Prepare input file.
+        self.get_input_file(raw_file)
+
+        # Run CALACS
+        subprocess.call(['calacs.e', raw_file, '-v'])
+
+        # Compare results
+        outputs = [('{}_flt.fits'.format(rootname),
+                    '{}_flt_ref.fits'.format(rootname))]
+        self.compare_outputs(outputs)

tests/acs/test_wfc_asn.py

@@ -0,0 +1,48 @@
+"""Tests for ACS/WFC ASN files."""
+import subprocess
+import pytest
+
+from ..helpers import BaseACS
+
+
+# NOTE: This is slow test due to PCTECORR=PERFORM
+@pytest.mark.slow
+class TestFullFrameASN(BaseACS):
+    """
+    Process pre- and post-SM4 fullframe WFC datasets using CR-SPLIT=2 with all
+    standard calibration steps turned on.
+    """
+    detector = 'wfc'
+
+    # NOTE:
+    # j6lq01010 = pre-SM4, was wfc_asn1
+    # jbdf08010 = post-SM4, was wfc_asn2
+    # NOTE:
+    # j6lq01naq and jbdf08ufq excluded from comparison for now, see
+    # https://github.com/spacetelescope/hstcal/pull/71#issuecomment-414132998
+    @pytest.mark.parametrize(
+        ('rootname', 'outroots'),
+        [('j6lq01010', ['j6lq01011', 'j6lq01ndq']),
+         ('jbdf08010', ['jbdf08011', 'jbdf08uhq'])])
+    def test_fullframe(self, rootname, outroots):
+        asn_file = rootname + '_asn.fits'
+
+        # Prepare input files.
+        self.get_input_file(asn_file)
+
+        # Run CALACS
+        subprocess.call(['calacs.e', asn_file, '-v'])
+
+        # Compare results.
+        # The first outroot is the output from whole ASN,
+        # the rest are individual members.
+        outputs = [('{}_crj.fits'.format(outroots[0]),
+                    '{}_crj_ref.fits'.format(outroots[0])),
+                   ('{}_crc.fits'.format(outroots[0]),
+                    '{}_crc_ref_gen2cte.fits'.format(outroots[0]))]
+        for outroot in outroots[1:]:
+            outputs += [('{}_flt.fits'.format(outroot),
+                         '{}_flt_ref.fits'.format(outroot)),
+                        ('{}_flc.fits'.format(outroot),
+                         '{}_flc_ref_gen2cte.fits'.format(outroot))]
+        self.compare_outputs(outputs)

tests/acs/test_wfc_single.py

@@ -0,0 +1,52 @@
+import subprocess
+import pytest
+
+from ..helpers import BaseACS
+
+
+class TestFullFrameSingle(BaseACS):
+    """
+    Process pre- and post-SM4 single fullframe WFC datasets
+    using all standard calibration steps and the
+    Generation 2 CTE correction.
+
+    For pre-SM4, apply the original BLEVCORR algorithm,
+    but no bias drift correction.
+
+    For post-SM4, apply the 'new' BLEVCORR algorithm,
+    which includes bias shift, cross talk, and destripe corrections.
+    """
+    detector = 'wfc'
+
+    def _single_raw_calib(self, rootname):
+        raw_file = '{}_raw.fits'.format(rootname)
+
+        # Prepare input file.
+        self.get_input_file(raw_file)
+
+        # Run CALACS
+        subprocess.call(['calacs.e', raw_file, '-v'])
+
+        # Compare results
+        outputs = [('{}_flt.fits'.format(rootname),
+                    '{}_flt_ref.fits'.format(rootname)),
+                   ('{}_flc.fits'.format(rootname),
+                    '{}_flc_ref_gen2cte.fits'.format(rootname))]
+        self.compare_outputs(outputs)
+
+    # NOTE: This is slow test due to PCTECORR=PERFORM
+    # NOTE:
+    # j6lq01naq = pre-SM4, was wfc_single1
+    # jbdf08ufq = post-SM4, was wfc_single2
+    @pytest.mark.slow
+    @pytest.mark.parametrize(
+        'rootname', ['j6lq01naq', 'jbdf08ufq'])
+    def test_fullframe_single(self, rootname):
+        self._single_raw_calib(rootname)
+
+    # NOTE: This is not marked slow to run one PCTECORR=PERFORM
+    #       for a push event on GitHub. This alone takes about 8 mins.
+    # NOTE:
+    # jbdf08uf2 = post-SM4 with FLSHCORR, was wfc_single3
+    def test_fullframe_single_postsm4_flshcorr(self):
+        self._single_raw_calib('jbdf08uf2')

tests/acs/test_wfc_sub.py

@@ -0,0 +1,52 @@
+import subprocess
+import pytest
+
+from ..helpers import BaseACS
+
+
+class TestWFCSubarray(BaseACS):
+    """
+    Process a single WFC subarray with all standard calibration steps on.
+    """
+    detector = 'wfc'
+
+    # NOTE: This has PCTECORR=OMIT.
+    # NOTE:
+    # j9j902b6q = was wfc_sub1
+    # jb2t11seq = Post-SM4 data with overscan, was wfc_sub2
+    # j8c103xaq = Pre-SM4, was wfc_sub3
+    # jb2t11se2 = Post-SM4 with overscan and FLSHCORR, was wfc_sub4
+    @pytest.mark.parametrize(
+        'rootname', ['j9j902b6q', 'jb2t11seq', 'j8c103xaq', 'jb2t11se2'])
+    def test_subarray(self, rootname):
+        raw_file = '{}_raw.fits'.format(rootname)
+
+        # Prepare input file.
+        self.get_input_file(raw_file)
+
+        # Run CALACS
+        subprocess.call(['calacs.e', raw_file, '-v'])
+
+        # Compare results
+        outputs = [('{}_flt.fits'.format(rootname),
+                    '{}_flt_ref.fits'.format(rootname))]
+        self.compare_outputs(outputs)
+
+    # NOTE: This is slow test due to PCTECORR=PERFORM
+    @pytest.mark.slow
+    def test_subarray_pctecorr(self):
+        """
+        This was ``wfc_sub5``, single 2K subarray processed with PCTECORR.
+        """
+        raw_file = 'jb5s01fnq_raw.fits'
+
+        # Prepare input file.
+        self.get_input_file(raw_file)
+
+        # Run CALACS
+        subprocess.call(['calacs.e', raw_file, '-v'])
+
+        # Compare results
+        outputs = [('jb5s01fnq_flt.fits', 'jb5s01fnq_flt_ref.fits'),
+                   ('jb5s01fnq_flc.fits', 'jb5s01fnq_flc_ref_gen2cte.fits')]
+        self.compare_outputs(outputs)

tests/conftest.py

@@ -0,0 +1,3 @@
+"""Custom ``pytest`` configurations."""
+
+pytest_plugins = ["pytest_ciwatson"]