Merge bf50fe0 into f19ce7f

.travis.yml

@@ -6,6 +6,7 @@ python:
   - 3.5
   - 3.6
   - 3.7
+  - 3.8
 
 install: pip install -U tox tox-travis coverage coveralls
 

docs/config_test.rst

@@ -26,6 +26,8 @@ Optional fields
 
 **description** *[string]*
     A short description to help identification of this version.
+**minimum_app_version** *[string]*
+    The minimum software version required for the test to be executed (see :ref:`config_sw`). If the software under test does not satisfy this requirement, then the test is removed from the test suite before execution. This allows you to run the latest test suite on old software without test failures.
 **input_files** *[list of strings]*
     A list of required input files. Each path is specified relative to the location of the configuration file itself.
 **output_files** *[dict of string-string pairs]*

nrtest/__init__.py

@@ -119,6 +119,7 @@ class Test(Metadata):
     ]
     execute_optional_fields = {
         'description': None,
+        'minimum_app_version': None,
         'input_files': [],
         'output_files': {},
         'fail_strings': [],
@@ -127,6 +128,7 @@ class Test(Metadata):
         'name',
         'version',
         'description',
+        'minimum_app_version',
         'output_files',
         'passed',
         'error_msg',

nrtest/execute.py

@@ -7,6 +7,9 @@
 import json
 import datetime
 
+# third-party imports
+from packaging import version
+
 # project imports
 from .process import source, execute, monitor
 from .utility import color, copy_file_and_path, rmtree, which
@@ -131,7 +134,12 @@ def validate_testsuite(ts):
         return False
 
     for t in ts.tests:
-        if not validate_test(t):
+        if t.minimum_app_version and \
+           version.parse(ts.app.version) < version.parse(t.minimum_app_version):
+            logging.info('Skipping test (app version too old): "%s"' % t.name)
+            ts.tests.remove(t)
+
+        elif not validate_test(t):
             return False
 
     p = ts.benchmark_path
@@ -158,15 +166,16 @@ def validate_test(test):
             logger.error('Unable to find "%s" property' % field)
             return False
 
-    p = test.input_dir
-    if not os.path.isdir(p):
-        logger.error('Input directory not found: "%s"' % p)
-        return False
-
-    for fname in test.input_files:
-        p = os.path.join(test.input_dir, fname)
-        if not os.path.isfile(p):
-            logger.error('Input file not found: "%s"' % p)
+    if len(test.input_files) > 0:
+        p = test.input_dir
+        if not os.path.isdir(p):
+            logger.error('Input directory not found: "%s"' % p)
             return False
 
+        for fname in test.input_files:
+            p = os.path.join(test.input_dir, fname)
+            if not os.path.isfile(p):
+                logger.error('Input file not found: "%s"' % p)
+                return False
+
     return True

nrtest/testsuite.py

@@ -27,6 +27,10 @@ def __init__(self, app, tests, benchmark_path):
         self.tests = sorted(tests)
         self.benchmark_path = benchmark_path
 
+        for t in self.tests:
+            t.input_dir = None
+            t.output_dir = os.path.join(benchmark_path, slugify(t.name))
+
     @classmethod
     def read_config(cls, app_config_path, test_config_paths, benchmark_path):
         """Constructs a TestSuite instance from a set of JSON config files,

setup.py

@@ -17,6 +17,7 @@
 requirements = [
     'six',
     'psutil>=2.0',
+    'packaging',
 ]
 
 # bundled actions

tests/test_config.py

@@ -22,6 +22,7 @@
     'name': 'test',
     'version': 1.0,
     'description': None,
+    'minimum_app_version': None,
     'output_files': {'output.txt': 'default'},
     'passed': True,
     'error_msg': None,

tests/test_testsuite.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+test_testsuite
+----------------------------------
+
+Tests for validation of TestSuite.
+"""
+
+import os.path
+import tempfile
+import unittest
+
+from nrtest import Application, Test
+from nrtest.execute import validate_testsuite
+from nrtest.testsuite import TestSuite
+from nrtest.utility import rmtree
+
+app1 = Application.for_execution({
+    'name': 'app',
+    'version': '1.3.0',
+    'exe': 'echo',
+})
+
+app2 = Application.for_execution({
+    'name': 'app',
+    'version': '1.2.0',
+    'exe': 'echo',
+})
+
+test1 = Test.for_execution({
+    'name': 'cat',
+    'version': '1.0',
+    'args': ['cat'],
+})
+
+test2 = Test.for_execution({
+    'name': 'cat',
+    'version': '1.0',
+    'args': ['dog'],
+    'minimum_app_version': '1.3.0',
+})
+
+
+class TestMinimumAppVersion(unittest.TestCase):
+    def setUp(self):
+        self.benchmark_path = os.path.join(tempfile.gettempdir(), "benchmark")
+
+    def tearDown(self):
+        rmtree(self.benchmark_path)
+
+    def test_sufficient_app_version(self):
+        ts = TestSuite(app1, [test1, test2], self.benchmark_path)
+        self.assertTrue(validate_testsuite(ts))
+        self.assertEqual(len(ts.tests), 2)
+
+    def test_insufficient_app_version(self):
+        ts = TestSuite(app2, [test1, test2], self.benchmark_path)
+        self.assertTrue(validate_testsuite(ts))
+        self.assertEqual(len(ts.tests), 1)
+
+
+if __name__ == '__main__':
+    import sys
+    sys.exit(unittest.main())

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py27, py33, py34, py35
+envlist = py27, py33, py34, py35, py36, py37, py38
 
 [testenv]
 deps =