Merge pull request #15 from chummels/enable-test

Re-enable answer test and prepare for stable release (closes #7)

.travis.yml

@@ -25,6 +25,8 @@ env:
     YT_DIR=$HOME/yt-git
     TRIDENT_ION_DATA=$HOME/.trident
     TRIDENT_ANSWER_DATA=$HOME/answer_test_data
+    YT_GOLD=38b79c094ca9
+    TRIDENT_GOLD=gold-standard-v1
 
 matrix:
   include:
@@ -45,14 +47,14 @@ install:
     pip install --upgrade wheel
     pip install --upgrade setuptools
     # Install dependencies
-    pip install mock $NUMPY $SCIPY $H5PY $CYTHON $MATPLOTLIB $SYMPY $FASTCACHE $IPYTHON flake8 pytest
+    pip install mock $NUMPY $SCIPY $H5PY $CYTHON $MATPLOTLIB $SYMPY $FASTCACHE $IPYTHON flake8 pytest gitpython
     # install yt
     if [ ! -f $YT_DIR/README.md ]; then
         git clone --branch=master https://github.com/yt-project/yt $YT_DIR
     fi
     pushd $YT_DIR
     git pull
-    git checkout master
+    git checkout $YT_GOLD
     pip install -e .
     popd
     # install trident
@@ -69,11 +71,39 @@ before_script:
     mv hm2012_lr.h5 $TRIDENT_ION_DATA
     cat $TRIDENT_ION_DATA/config.tri
     # download answer test data
-    cd tests
+    pushd tests
     export RUN_DOWNLOAD_TEST=1
     travis_wait py.test test_download.py
+    popd
 
 script:
   - |
+    # install gold standard and generate answers
+    # this uses the yt gold standard version
     export RUN_DOWNLOAD_TEST=0
+    export TRIDENT_GENERATE_TEST_RESULTS=1
+    git checkout $TRIDENT_GOLD
+    pip install -e .
+    pushd tests
+    py.test
+    popd
+    # gold-standard yt + trident tip
+    export TRIDENT_GENERATE_TEST_RESULTS=0
+    git checkout HEAD
+    pip install -e .
+    pushd tests
+    py.test
+    popd
+    # yt tip + trident tip
+    pushd $YT_DIR
+    git checkout HEAD
+    pip install -e .
+    popd
+    pushd tests
+    py.test
+    popd
+    # yt tip + trident gold standard
+    git checkout $TRIDENT_GOLD
+    pip install -e .
+    pushd tests
     py.test

doc/source/installation.rst

@@ -13,7 +13,7 @@ Versions of Trident
 Currently, there are two versions of Trident: the `stable version 
 <http://trident.readthedocs.io/en/stable>`_, and the `developent version
 <http://trident.readthedocs.io/en/latest>`_.  The stable version is tried,
-and tested, and it operates on a stable version of yt.  The development 
+and tested, and it normally operates on a stable version of yt.  The development 
 version is actively being updated with new features, and it is also tied to
 the development version of yt, so occasionally unforseen bugs can crop
 up as these new features are added.  
@@ -23,6 +23,16 @@ Don't worry if you want to change later, you can always
 switch between the two versions easily enough by following the directions
 in :ref:`uninstallation`.
 
+.. warning::
+
+    The current stable version of yt, yt 3.4.0, has a 
+    `bug <https://github.com/yt-project/yt/issues/1610>`_ affecting the 
+    temperature field in cosmological SPH datasets at z>0. We currently 
+    recommend using the development version of yt, where this bug is fixed, 
+    for use with trident (both stable and dev) until this bugfix makes it 
+    into yt stable. The directions below will result in the yt development 
+    branch being installed.
+
 .. _step-1:
 
 Step 1: Install yt  
@@ -39,20 +49,14 @@ We find that the easiest way to install yt is with the all-in-one install
 script, which installs yt and its dependencies via a new conda installation::
 
     $ wget https://raw.githubusercontent.com/yt-project/yt/master/doc/install_script.sh
-    $ ... edit the install_script.sh to mark INST_SCIPY=1 and INST_ASTROPY=1
-    $ ... if you want to use the dev version of yt and trident, mark INST_YT_SOURCE=1
+    $ ... edit the install_script.sh to mark INST_SCIPY=1, INST_ASTROPY=1,
+    $ ... and INST_YT_SOURCE=1
     $ bash install_script.sh
     $ ... update your path flag as described by the install_script.sh
 
 Alternatively, if you already have conda installed, you can skip the commands
 above and just run the following command to get yt and its dependencies.  
-
-To get the stable version of yt (for the stable version of trident), type::
-
-    $ conda install -c conda-forge yt
-
-To get the nightly build of the development version of yt (for the development 
-version of trident), type::
+To get the nightly build of the development version of yt, type::
 
     $ conda install -c http://use.yt/with_conda/ -c conda-forge yt
 

doc/source/testing.rst

@@ -1,38 +1,67 @@
 .. _testing:
 
+Testing
+=======
+
+We maintain a series of tests in Trident to make sure the code gives consistent
+results and to catch accidental breakages in our source code and dependencies.
+These tests are run by `Travis <https://travis-ci.org/>`_ automatically and 
+regularly to assure consistency in functionality, but you can run them locally
+too (see below).  The tests consist of a mix of unit tests (tests to assure Trident 
+functions don't actively fail) and answer tests (tests comparing newly 
+generated results against some old established results to assure consistency).
+
+.. _running-the-tests:
+
 Running the Test Suite
-======================
+----------------------
 
 Running the test suite requires a version of Trident installed from
 source (see :ref:`install-dev`).
 
-Trident maintains a suite of unit and answer tests to ensure that
-development doesn't change code behavior in unexpected ways.  The
-tests are run using the ``pytest`` Python module.  This can be
+The tests are run using the ``pytest`` Python module.  This can be
 installed with ``conda`` or ``pip``.
 
 .. code-block:: bash
 
    $ conda install pytest
 
-The test suite requires a number of datasets as well as results
-files for answer comparison.  Trident comes with a helper script
-that will download all the datasets and untar them.  Before running
-this, make sure you have the ``answer_test_data_dir`` variable set in
-your config file (see :ref:`step-3`).  This variable should point to
-a directory.  The helper script is located in the ``tests`` directory
-of the Trident source.
+The test suite requires a number of datasets for testing functionality.
+Trident comes with a helper script that will download all the datasets and 
+untar them.  Before running this, make sure you have the 
+``answer_test_data_dir`` variable set in your config file (see :ref:`step-3`).  
+This variable should point to a directory where these datasets will be stored.  
+The helper script is located in the ``tests`` directory of the Trident source.
 
 .. code-block:: bash
 
    $ cd tests
    $ python download_test_data.py
 
-Once the test data has been downloaded, the test suite is run by
-calling ``py.test`` from within the ``tests`` directory.
+If this is your first time running the tests, then you need to generate a
+"gold standard" for the answer tests. Follow :ref:`generating-answer-tests` 
+before continuing with running the tests, otherwise your answer tests will 
+fail.
+
+Make sure you're on the desired version of yt and trident that you want to 
+test and use (usually the tip of the development branch i.e., ``master``).  
+
+.. code-block:: bash
+
+   $ export TRIDENT_GENERATE_TEST_RESULTS=0
+   $ cd /path/to/yt/
+   $ git checkout master
+   $ pip install -e .
+   $ cd /path/to/trident
+   $ git checkout master
+   $ pip install -e .
+
+The test suite is run by calling ``py.test`` from within the ``tests`` 
+directory.
 
 .. code-block:: bash
 
+   $ cd tests
    $ py.test
    ============================= test session starts ==============================
    platform darwin -- Python 3.6.0, pytest-3.0.7, py-1.4.32, pluggy-0.4.0
@@ -54,19 +83,75 @@ calling ``py.test`` from within the ``tests`` directory.
 
    ========================= 52 passed in 117.32 seconds ==========================
 
-Generating Test Results
-=======================
+If a test fails for some reason, you will be given a detailed traceback and
+reason for it failing.  You can use this to identify what is wrong with your
+source or perhaps a change in the code of your dependencies.  The tests should
+take less than five minutes to run.
 
-If new tests have been added or the code's behavior has changed (in a good way)
-such that the tests no longer pass, new results must be generated.  Before
-generating new results, be sure to update the results version number in
-``tests/test_results_version.txt``.  Then, set the ``TRIDENT_GENERATE_TEST_RESULTS``
-environment variable to 1 and rerun the tests:
+.. _generating-answer-tests:
+
+Generating Test Results
+-----------------------
+
+In order to assure the Trident codebase gives consistent results over time, 
+we compare the outputs of tests of new versions of Trident against an older, 
+vetted version of the code we think gives accurate results.  To create this
+"gold standard" result from the older version of the code, you must roll back 
+the Trident and yt source back to the older "trusted" versions of the code.  
+You can find the tags for the most recent trusted versions of the code by 
+running ``gold_standard_versions.py`` and then rebuilding yt and Trident 
+with these versions of the code.  Lastly, set the 
+``TRIDENT_GENERATE_TEST_RESULTS`` environment variable to 1 and run the tests:
 
 .. code-block:: bash
 
    $ cd tests
+   $ python gold_standard_versions.py
+   
+   Latest Gold Standard Commit Tags
+   yt = 38b79c094ca9
+   Trident = gold-standard-v1
+
+   To update to them, `git checkout <tag>` in appropriate repository
+
+   $ cd /path/to/yt
+   $ git checkout 38b79c094ca9
+   $ pip install -e .
+   $ cd /path/to/trident
+   $ git checkout gold-standard-v1
+   $ pip install -e .
    $ export TRIDENT_GENERATE_TEST_RESULTS=1
+   $ cd tests
    $ py.test
 
-The results must then be tarred up and uploaded to the Trident website.
+The test results should now be stored in the ``answer_test_data_dir`` that
+you specified in your Trident configuration file. You may now run the actual 
+tests (see :ref:`running-the-tests`) with your current version of yt and 
+Trident comparing against these gold standard results.
+
+.. _updating-the-test-results:
+
+Updating the Test Results
+--------------------------
+
+Periodically, the gold standard for our answer tests must be updated as bugs 
+are caught or new more accurate behavior is enabled that causes the answer
+tests to fail.  The first thing to do
+is to identify the most accurate version of the code (e.g., changesets for 
+yt and trident that give the desired behavior).  Tag the Trident changeset with
+the next gold standard iteration.  You can see the current iteration by looking
+in the ``.travis.yml`` file at the ``TRIDENT_GOLD`` entry--enumerate this and
+tag the changeset.  Update the ``.travis.yml`` file so that the ``YT_GOLD`` and
+``TRIDENT_GOLD`` entries point to your desired changeset and tag.  Finally,
+you have to explicitly push the new tag (hereafter ``gold-standard-v2``) to 
+the repository.
+
+.. code-block:: bash
+
+   $ git tag gold-standard-v2 <trident-changeset>
+   $ ... edit .travis.yml files to update YT_GOLD=<yt changeset>
+   $ ... and TRIDENT_GOLD=<gold-standard-v2
+   $ git add .travis.yml
+   $ git commit
+   $ git push origin
+   $ git push origin gold-standard-v2

tests/download_test_data.py

@@ -18,16 +18,10 @@
 from trident.utilities import \
     download_file
 
-from trident.testing import \
-    test_results_version
-
 def download_datasets(local_dir=None, progress_bar=True):
     if local_dir is None:
         local_dir = answer_test_data_dir
     urls = open("test_datasets.txt", "r").readlines()
-    test_results = "http://trident-project.org/data/tests/test_results_%d.tar.gz" % \
-      test_results_version
-    urls += [test_results]
 
     for url in urls:
         if url.strip().startswith("#"):

tests/gold_standard_versions.py

@@ -0,0 +1,36 @@
+"""
+Testing utilities for Trident
+
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2017, Trident Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file LICENSE, distributed with this software.
+#-----------------------------------------------------------------------------
+
+from __future__ import print_function
+from trident import path as trident_path
+import os
+
+def get_gold_standard_version():
+    f = open(os.path.join(trident_path, '../.travis.yml'), 'r')
+    for line in f:
+        line = line.lstrip()
+        if line.startswith('YT_GOLD'):
+            line_list = line.split('=')
+            yt_gold = line_list[1]
+        if line.startswith('TRIDENT_GOLD'):
+            line_list = line.split('=')
+            trident_gold = line_list[1]
+    f.close()
+    print()
+    print('Latest Gold Standard Commit Tags\n')
+    print('yt = %s' % yt_gold, end='')
+    print('Trident = %s' % trident_gold)
+    print('To update to them, `git checkout <tag>` in appropriate repository')
+
+if __name__ == "__main__":
+    get_gold_standard_version()

tests/test_generate.py

@@ -19,14 +19,14 @@ def test_result_generate():
     if not generate_results:
         return
     try:
-        import hglib
+        import git
     except ImportError:
         raise RuntimeError(
-            "Generating test results requires python-hglib. " + \
-            "Install with: pip install python-hglib")
+            "Generating test results requires gitpython. " + \
+            "Install with: pip install gitpython")
 
-    repo = hglib.open("..")
-    my_hash = repo.identify().decode("utf").strip().split()[0]
+    repo = git.Repo(search_parent_directories=True)
+    my_hash = repo.head.object.hexsha[:8]
     fh = open(os.path.join(test_results_dir, "TRIDENT_VERSION"), "w")
     fh.write("%s\n" % my_hash)
     fh.close()

tests/test_pipelines.py

@@ -43,7 +43,6 @@ def test_verify():
 
 class PipelineTest(TempDirTest):
 
-    @unittest.skip("temporarily disabled")
     def test_enzo_small_simple(self):
         """
         This is an answer test, which compares the results of this test

tests/test_results_version.txt

@@ -1,2 +1,2 @@
 # this is the current version of the test results
-version=0
+version=1

trident/testing.py

@@ -27,24 +27,11 @@
 from trident.config import \
     parse_config
 
-def get_test_results_version():
-    filename = "../tests/test_results_version.txt"
-    lines = open(filename).readlines()
-    for line in lines:
-        if line.startswith("version="):
-            version = int(line.split("=")[1])
-            return version
-    # if we're here, we didnt' get a version
-    raise RuntimeError(
-        "Couldn't get test result version from %s." % filename)
-test_results_version = get_test_results_version()
-
 # If TRIDENT_GENERATE_TEST_RESULTS=1, just generate test results.
 generate_results = int(os.environ.get("TRIDENT_GENERATE_TEST_RESULTS", 0)) == 1
 answer_test_data_dir = ensure_dir(
   os.path.abspath(os.path.expanduser(parse_config('answer_test_data_dir'))))
-test_results_dir = \
-  os.path.join(answer_test_data_dir, "test_results_%s" % test_results_version)
+test_results_dir = os.path.join(answer_test_data_dir, "test_results")
 if generate_results:
     ensure_dir(test_results_dir)