diff --git a/reframe/core/exceptions.py b/reframe/core/exceptions.py index 3764c8c2ff..91e46528c1 100644 --- a/reframe/core/exceptions.py +++ b/reframe/core/exceptions.py @@ -207,6 +207,10 @@ class JobNotStartedError(JobError): """Raised when trying to operate on a unstarted job.""" +class DependencyError(ReframeError): + """Raised when a dependency problem is encountered.""" + + class ReframeDeprecationWarning(DeprecationWarning): """Warning for deprecated features of the ReFrame framework.""" diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 7f75eefdd3..5bb21eeb3c 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -3,7 +3,8 @@ # __all__ = ['RegressionTest', - 'RunOnlyRegressionTest', 'CompileOnlyRegressionTest'] + 'RunOnlyRegressionTest', 'CompileOnlyRegressionTest', + 'DEPEND_EXACT', 'DEPEND_BY_ENV', 'DEPEND_FULLY'] import inspect @@ -22,7 +23,8 @@ from reframe.core.buildsystems import BuildSystem, BuildSystemField from reframe.core.deferrable import deferrable, _DeferredExpression, evaluate from reframe.core.environments import Environment, EnvironmentSnapshot -from reframe.core.exceptions import (BuildError, PipelineError, SanityError, +from reframe.core.exceptions import (BuildError, DependencyError, + PipelineError, SanityError, PerformanceError) from reframe.core.launchers.registry import getlauncher from reframe.core.schedulers import Job @@ -31,6 +33,12 @@ from reframe.utility.sanity import assert_reference +# Dependency kinds +DEPEND_EXACT = 1 +DEPEND_BY_ENV = 2 +DEPEND_FULLY = 3 + + class RegressionTest: """Base class for regression tests. @@ -627,6 +635,12 @@ def __init__(self, name=None, prefix=None): # Performance logging self._perf_logger = logging.null_logger + # List of dependencies specified by the user + self._userdeps = [] + + # Weak reference to the test case associated with this check + self._case = None + # Export read-only views to interesting fields @property def current_environ(self): @@ -749,9 +763,6 @@ def build_stdout(self): def build_stderr(self): return self._build_job.stderr - def __repr__(self): - return debug.repr(self) - def info(self): """Provide live information of a running test. @@ -1187,6 +1198,35 @@ def cleanup(self, remove_files=False, unload_env=True): self._current_environ.unload() self._current_partition.local_env.unload() + # Dependency API + def user_deps(self): + return util.SequenceView(self._userdeps) + + def depends_on(self, target, how=DEPEND_BY_ENV, subdeps=None): + if not isinstance(target, str): + raise TypeError("target argument must be of type: `str'") + + if not isinstance(how, int): + raise TypeError("how argument must be of type: `int'") + + if (subdeps is not None and + not isinstance(subdeps, typ.Dict[str, typ.List[str]])): + raise TypeError("subdeps argument must be of type " + "`Dict[str, List[str]]' or `None'") + + self._userdeps.append((target, how, subdeps)) + + def getdep(self, target, environ): + if self._case is None or self._case() is None: + raise DependencyError('no test case is associated with this test') + + for d in self._case().deps: + if d.check.name == target and d.environ.name == environ: + return d.check + + raise DependencyError('could not resolve dependency to (%s, %s)' % + (target, environ)) + def __str__(self): return "%s(name='%s', prefix='%s')" % (type(self).__name__, self.name, self.prefix) diff --git a/reframe/frontend/dependency.py b/reframe/frontend/dependency.py new file mode 100644 index 0000000000..674927cebc --- /dev/null +++ b/reframe/frontend/dependency.py @@ -0,0 +1,81 @@ +# +# Test case graph functionality +# + +import collections + +import reframe as rfm +from reframe.core.exceptions import DependencyError + + +def build_deps(cases): + """Build dependency graph from test cases. + + The graph is represented as an adjacency list in a Python dictionary + holding test cases. The dependency information is also encoded inside each + test cases. + """ + + # Index cases for quick access + cases_by_part = {} + cases_revmap = {} + for c in cases: + cname = c.check.name + pname = c.partition.fullname + ename = c.environ.name + cases_by_part.setdefault((cname, pname), []) + cases_revmap.setdefault((cname, pname, ename), None) + cases_by_part[cname, pname].append(c) + cases_revmap[cname, pname, ename] = c + + def resolve_dep(target, from_map, *args): + errmsg = 'could not resolve dependency: %s' % target + try: + ret = from_map[args] + except KeyError: + raise DependencyError(errmsg) + else: + if not ret: + raise DependencyError(errmsg) + + return ret + + # NOTE on variable names + # + # c stands for check or case depending on the context + # p stands for partition + # e stands for environment + # t stands for target + + graph = {} + for c in cases: + graph[c] = c.deps + cname = c.check.name + pname = c.partition.fullname + ename = c.environ.name + for dep in c.check.user_deps(): + tname, how, subdeps = dep + if how == rfm.DEPEND_FULLY: + c.deps.extend(resolve_dep(c, cases_by_part, tname, pname)) + elif how == rfm.DEPEND_BY_ENV: + c.deps.append(resolve_dep(c, cases_revmap, + tname, pname, ename)) + elif how == rfm.DEPEND_EXACT: + for env, tenvs in subdeps.items(): + if env != ename: + continue + + for te in tenvs: + c.deps.append(resolve_dep(c, cases_revmap, + tname, pname, te)) + + return graph + + +def print_deps(graph): + for c, deps in graph.items(): + print(c, '->', deps) + + +def validate_deps(graph): + """Validate dependency graph.""" diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py index 48d7f7e51c..7d8abfa0ec 100644 --- a/reframe/frontend/executors/__init__.py +++ b/reframe/frontend/executors/__init__.py @@ -1,6 +1,7 @@ import abc import copy import sys +import weakref import reframe.core.debug as debug import reframe.core.logging as logging @@ -22,14 +23,33 @@ class TestCase: def __init__(self, check, partition, environ): self.__check_orig = check self.__check = copy.deepcopy(check) - self.__environ = copy.deepcopy(environ) self.__partition = copy.deepcopy(partition) + self.__environ = copy.deepcopy(environ) + self.__check._case = weakref.ref(self) + self.__deps = [] def __iter__(self): # Allow unpacking a test case with a single liner: # c, p, e = case return iter([self.__check, self.__partition, self.__environ]) + def __hash__(self): + return (hash(self.check.name) ^ + hash(self.partition.fullname) ^ + hash(self.environ.name)) + + def __eq__(self, other): + if not isinstance(other, type(self)): + return NotImplemented + + return (self.check.name == other.check.name and + self.environ.name == other.environ.name and + self.partition.fullname == other.partition.fullname) + + def __repr__(self): + return '(%r, %r, %r)' % (self.check.name, + self.partition.fullname, self.environ.name) + @property def check(self): return self.__check @@ -42,6 +62,10 @@ def partition(self): def environ(self): return self.__environ + @property + def deps(self): + return self.__deps + def clone(self): # Return a fresh clone, i.e., one based on the original check return TestCase(self.__check_orig, self.__partition, self.__environ) diff --git a/unittests/resources/checks_unlisted/dependencies/normal.py b/unittests/resources/checks_unlisted/dependencies/normal.py new file mode 100644 index 0000000000..10f9e2d7de --- /dev/null +++ b/unittests/resources/checks_unlisted/dependencies/normal.py @@ -0,0 +1,37 @@ +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.simple_test +class Test0(rfm.RunOnlyRegressionTest): + def __init__(self): + super().__init__() + self.valid_systems = ['sys0:p0', 'sys0:p1'] + self.valid_prog_environs = ['e0', 'e1'] + self.executable = 'echo' + self.executable_opts = [self.name] + self.sanity_patterns = sn.assert_found(self.name, self.stdout) + + +@rfm.parameterized_test(*([kind] for kind in ['fully', 'by_env', + 'exact', 'default'])) +class Test1(rfm.RunOnlyRegressionTest): + def __init__(self, kind): + super().__init__() + kindspec = { + 'fully': rfm.DEPEND_FULLY, + 'by_env': rfm.DEPEND_BY_ENV, + 'exact': rfm.DEPEND_EXACT, + } + self.valid_systems = ['sys0:p0', 'sys0:p1'] + self.valid_prog_environs = ['e0', 'e1'] + self.executable = 'echo' + self.executable_opts = [self.name] + self.sanity_patterns = sn.assert_found(self.name, self.stdout) + if kind == 'default': + self.depends_on('Test0') + elif kindspec[kind] == rfm.DEPEND_EXACT: + self.depends_on('Test0', kindspec[kind], + {'e0': ['e0', 'e1'], 'e1': ['e1']}) + else: + self.depends_on('Test0', kindspec[kind]) diff --git a/unittests/resources/settings.py b/unittests/resources/settings.py index 32ff7d934d..1d6713e37d 100644 --- a/unittests/resources/settings.py +++ b/unittests/resources/settings.py @@ -58,6 +58,21 @@ class ReframeSettings: 'descr': 'GPU partition', } } + }, + 'sys0': { + # System used for dependency checking + 'descr': 'System for test dependencies unit tests', + 'hostnames': [r'sys\d+'], + 'partitions': { + 'p0': { + 'scheduler': 'local', + 'environs': ['e0', 'e1'], + }, + 'p1': { + 'scheduler': 'local', + 'environs': ['e0', 'e1'], + } + } } }, 'environments': { @@ -90,7 +105,15 @@ class ReframeSettings: 'cc': 'gcc', 'cxx': 'g++', 'ftn': 'gfortran', - } + }, + 'e0': { + 'type': 'ProgEnvironment', + 'modules': ['m0'], + }, + 'e1': { + 'type': 'ProgEnvironment', + 'modules': ['m1'], + }, } }, 'modes': { diff --git a/unittests/test_config.py b/unittests/test_config.py index c23a476476..a1799d88e1 100644 --- a/unittests/test_config.py +++ b/unittests/test_config.py @@ -18,7 +18,7 @@ def get_partition(self, system, name): def test_load_success(self): self.site_config.load_from_dict(self.dict_config) - self.assertEqual(2, len(self.site_config.systems)) + self.assertEqual(3, len(self.site_config.systems)) system = self.site_config.systems['testsys'] self.assertEqual(2, len(system.partitions)) diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 4af6263e20..c765c1062e 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -1,18 +1,28 @@ +import collections import os +import pytest import tempfile import unittest +import reframe as rfm import reframe.core.runtime as rt +import reframe.frontend.dependency as dependency import reframe.frontend.executors as executors import reframe.frontend.executors.policies as policies import reframe.utility.os_ext as os_ext -from reframe.core.exceptions import JobNotStartedError +from reframe.core.exceptions import DependencyError, JobNotStartedError from reframe.frontend.loader import RegressionCheckLoader import unittests.fixtures as fixtures from unittests.resources.checks.hellocheck import HelloTest from unittests.resources.checks.frontend_checks import ( - KeyboardInterruptCheck, SystemExitCheck, SleepCheck, SleepCheckPollFail, - SleepCheckPollFailLate, BadSetupCheck, BadSetupCheckEarly, RetriesCheck + BadSetupCheck, + BadSetupCheckEarly, + KeyboardInterruptCheck, + RetriesCheck, + SleepCheck, + SleepCheckPollFail, + SleepCheckPollFailLate, + SystemExitCheck, ) @@ -384,3 +394,196 @@ def test_poll_fails_busy_loop(self): stats = self.runner.stats self.assertEqual(num_tasks, stats.num_cases()) self.assertEqual(num_tasks, len(stats.failures())) + + +class TestDependencies(unittest.TestCase): + class Node: + """A node in the test case graph. + + It's simply a wrapper to a (test_name, partition, environment) tuple + that can interact seemlessly with a real test case. + It's meant for convenience in unit testing. + """ + + def __init__(self, cname, pname, ename): + self.cname, self.pname, self.ename = cname, pname, ename + + def __eq__(self, other): + if isinstance(other, type(self)): + return (self.cname == other.cname and + self.pname == other.pname and + self.ename == other.ename) + + if isinstance(other, executors.TestCase): + return (self.cname == other.check.name and + self.pname == other.partition.fullname and + self.ename == other.environ.name) + + return NotImplemented + + def __hash__(self): + return hash(self.cname) ^ hash(self.pname) ^ hash(self.ename) + + def __repr__(self): + return 'Node(%r, %r, %r)' % (self.cname, self.pname, self.ename) + + def has_edge(graph, src, dst): + return dst in graph[src] + + def num_deps(graph, cname): + return sum(len(deps) for c, deps in graph.items() + if c.check.name == cname) + + def find_check(name, checks): + for c in checks: + if c.name == name: + return c + + return None + + def find_case(cname, ename, cases): + for c in cases: + if c.check.name == cname and c.environ.name == ename: + return c + + def setUp(self): + self.loader = RegressionCheckLoader([ + 'unittests/resources/checks_unlisted/dependencies/normal.py' + ]) + + # Set runtime prefix + rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests') + + def tearDown(self): + os_ext.rmtree(rt.runtime().resources.prefix) + + @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0') + def test_eq_hash(self): + find_case = TestDependencies.find_case + cases = executors.generate_testcases(self.loader.load_all()) + + case0 = find_case('Test0', 'e0', cases) + case1 = find_case('Test0', 'e1', cases) + case0_copy = case0.clone() + + assert case0 == case0_copy + assert hash(case0) == hash(case0_copy) + assert case1 != case0 + assert hash(case1) != hash(case0) + + @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0') + def test_build_deps(self): + Node = TestDependencies.Node + has_edge = TestDependencies.has_edge + num_deps = TestDependencies.num_deps + find_check = TestDependencies.find_check + find_case = TestDependencies.find_case + + checks = self.loader.load_all() + cases = executors.generate_testcases(checks) + + # Test calling getdep() before having built the graph + t = find_check('Test1_exact', checks) + with pytest.raises(DependencyError): + t.getdep('Test0', 'e0') + + # Build dependencies and continue testing + deps = dependency.build_deps(cases) + + # Check DEPEND_FULLY dependencies + assert num_deps(deps, 'Test1_fully') == 8 + for p in ['sys0:p0', 'sys0:p1']: + for e0 in ['e0', 'e1']: + for e1 in ['e0', 'e1']: + assert has_edge(deps, + Node('Test1_fully', p, e0), + Node('Test0', p, e1)) + + # Check DEPEND_BY_ENV + assert num_deps(deps, 'Test1_by_env') == 4 + assert num_deps(deps, 'Test1_default') == 4 + for p in ['sys0:p0', 'sys0:p1']: + for e in ['e0', 'e1']: + assert has_edge(deps, + Node('Test1_by_env', p, e), + Node('Test0', p, e)) + assert has_edge(deps, + Node('Test1_default', p, e), + Node('Test0', p, e)) + + # Check DEPEND_EXACT + assert num_deps(deps, 'Test1_exact') == 6 + for p in ['sys0:p0', 'sys0:p1']: + assert has_edge(deps, + Node('Test1_exact', p, 'e0'), + Node('Test0', p, 'e0')) + assert has_edge(deps, + Node('Test1_exact', p, 'e0'), + Node('Test0', p, 'e1')) + assert has_edge(deps, + Node('Test1_exact', p, 'e1'), + Node('Test0', p, 'e1')) + + # Pick a check to test getdep() + check_e0 = find_case('Test1_exact', 'e0', cases).check + check_e1 = find_case('Test1_exact', 'e1', cases).check + assert check_e0.getdep('Test0', 'e0').name == 'Test0' + assert check_e0.getdep('Test0', 'e1').name == 'Test0' + assert check_e1.getdep('Test0', 'e1').name == 'Test0' + with pytest.raises(DependencyError): + check_e0.getdep('TestX', 'e0') + + with pytest.raises(DependencyError): + check_e0.getdep('Test0', 'eX') + + with pytest.raises(DependencyError): + check_e1.getdep('Test0', 'e0') + + @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0') + def test_build_deps_unknown_test(self): + find_check = TestDependencies.find_check + checks = self.loader.load_all() + + # Add some inexistent dependencies + test0 = find_check('Test0', checks) + for depkind in ('default', 'fully', 'by_env', 'exact'): + test1 = find_check('Test1_' + depkind, checks) + if depkind == 'default': + test1.depends_on('TestX') + elif depkind == 'exact': + test1.depends_on('TestX', rfm.DEPEND_EXACT, {'e0': ['e0']}) + elif depkind == 'fully': + test1.depends_on('TestX', rfm.DEPEND_FULLY) + elif depkind == 'by_env': + test1.depends_on('TestX', rfm.DEPEND_BY_ENV) + + with pytest.raises(DependencyError): + dependency.build_deps(executors.generate_testcases(checks)) + + @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0') + def test_build_deps_unknown_target_env(self): + find_check = TestDependencies.find_check + checks = self.loader.load_all() + + # Add some inexistent dependencies + test0 = find_check('Test0', checks) + test1 = find_check('Test1_default', checks) + test1.depends_on('Test0', rfm.DEPEND_EXACT, {'e0': ['eX']}) + with pytest.raises(DependencyError): + dependency.build_deps(executors.generate_testcases(checks)) + + @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0') + def test_build_deps_unknown_source_env(self): + find_check = TestDependencies.find_check + num_deps = TestDependencies.num_deps + checks = self.loader.load_all() + + # Add some inexistent dependencies + test0 = find_check('Test0', checks) + test1 = find_check('Test1_default', checks) + test1.depends_on('Test0', rfm.DEPEND_EXACT, {'eX': ['e0']}) + + # Unknown source is ignored, because it might simply be that the test + # is not executed for eX + deps = dependency.build_deps(executors.generate_testcases(checks)) + assert num_deps(deps, 'Test1_default') == 4