From e766974143a6281d34a3a640fc9b6117f9137ab2 Mon Sep 17 00:00:00 2001 From: Anian Altherr Date: Thu, 26 Jan 2023 10:32:57 +0100 Subject: [PATCH 1/7] Allow to pass CI options for --ci-generate --- reframe/core/pipeline.py | 14 ++++++++++++++ reframe/frontend/ci.py | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index f4c5fcd265..e845862596 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -963,6 +963,20 @@ def pipeline_hooks(cls): #: :default: :class:`True` build_locally = variable(typ.Bool, value=True, loggable=True) + #: .. versionadded:: ?? + #: + #: Options for CI pipeline passed in JSON format + #: + #: Example: If we want a pipeline to run only when files in + #: backend or src/main.c have changed, we add: + #: + #: ci_options = {"only": {"changes": ["backend/*", "src/main.c"]} + #: + #: :type: `dict` + #: :default: ``{}`` + ci_options = variable(typ.Dict[str, object], + value={}, loggable=True) + # Special variables #: Dry-run mode diff --git a/reframe/frontend/ci.py b/reframe/frontend/ci.py index bd48df85ef..77e860a4fc 100644 --- a/reframe/frontend/ci.py +++ b/reframe/frontend/ci.py @@ -68,7 +68,8 @@ def rfm_command(testcase): 'artifacts': { 'paths': [f'{tc.check.unique_name}-report.json'] }, - 'needs': [t.check.unique_name for t in tc.deps] + 'needs': [t.check.unique_name for t in tc.deps], + **tc.check.ci_options } max_level = max(max_level, tc.level) From 7a29bc87e99dc83d767cac3b7835f66142c69927 Mon Sep 17 00:00:00 2001 From: Anian Altherr Date: Wed, 29 Mar 2023 14:57:33 +0200 Subject: [PATCH 2/7] Rename ci_options -> ci_extras, add 'gitlab' key --- reframe/core/pipeline.py | 6 +++--- reframe/frontend/ci.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index e845862596..e4d2630796 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -963,7 +963,7 @@ def pipeline_hooks(cls): #: :default: :class:`True` build_locally = variable(typ.Bool, value=True, loggable=True) - #: .. versionadded:: ?? + #: .. versionadded:: 4.2 #: #: Options for CI pipeline passed in JSON format #: @@ -974,8 +974,8 @@ def pipeline_hooks(cls): #: #: :type: `dict` #: :default: ``{}`` - ci_options = variable(typ.Dict[str, object], - value={}, loggable=True) + ci_extras = variable(typ.Dict[typ.Str['gitlab'], object], + value={}, loggable=False) # Special variables diff --git a/reframe/frontend/ci.py b/reframe/frontend/ci.py index 77e860a4fc..67dcbcef51 100644 --- a/reframe/frontend/ci.py +++ b/reframe/frontend/ci.py @@ -69,7 +69,7 @@ def rfm_command(testcase): 'paths': [f'{tc.check.unique_name}-report.json'] }, 'needs': [t.check.unique_name for t in tc.deps], - **tc.check.ci_options + **tc.check.ci_extras['gitlab'] } max_level = max(max_level, tc.level) From ed975cbd93b34e6b557609b1245832fac980d67d Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 3 Apr 2023 22:04:33 +0200 Subject: [PATCH 3/7] Fix unit tests --- reframe/core/pipeline.py | 3 +-- reframe/frontend/ci.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index e4d2630796..e61c48475a 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -974,8 +974,7 @@ def pipeline_hooks(cls): #: #: :type: `dict` #: :default: ``{}`` - ci_extras = variable(typ.Dict[typ.Str['gitlab'], object], - value={}, loggable=False) + ci_extras = variable(typ.Dict[typ.Str['gitlab'], object], value={}) # Special variables diff --git a/reframe/frontend/ci.py b/reframe/frontend/ci.py index 67dcbcef51..e64b06274f 100644 --- a/reframe/frontend/ci.py +++ b/reframe/frontend/ci.py @@ -69,7 +69,7 @@ def rfm_command(testcase): 'paths': [f'{tc.check.unique_name}-report.json'] }, 'needs': [t.check.unique_name for t in tc.deps], - **tc.check.ci_extras['gitlab'] + **tc.check.ci_extras.get('gitlab', {}) } max_level = max(max_level, tc.level) From c71a8d4acfddb4c8812f4b3b3712c55b12f0763a Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 4 Apr 2023 00:31:53 +0200 Subject: [PATCH 4/7] Add unit tests --- reframe/frontend/ci.py | 26 +++++++++++++--- unittests/test_ci.py | 71 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 88 insertions(+), 9 deletions(-) diff --git a/reframe/frontend/ci.py b/reframe/frontend/ci.py index e64b06274f..e4cd08257a 100644 --- a/reframe/frontend/ci.py +++ b/reframe/frontend/ci.py @@ -7,8 +7,8 @@ import sys import yaml -import reframe.core.exceptions as errors import reframe.core.runtime as runtime +from reframe.core.exceptions import ReframeError def _emit_gitlab_pipeline(testcases, child_pipeline_opts): @@ -46,6 +46,19 @@ def rfm_command(testcase): *child_pipeline_opts ]) + def _valid_ci_extras(extras): + '''Validate Gitlab CI pipeline extras''' + + for kwd in ('stage', 'script', 'needs'): + if kwd in extras: + errmsg = f"invalid keyword found: {kwd!r}" + if kwd == 'script': + errmsg += " (use 'before_script' or 'after_script')" + + raise ReframeError(f"could not validate 'ci_extras': {errmsg}") + + return extras + max_level = 0 # We need the maximum level to generate the stages section json = { 'cache': { @@ -62,14 +75,19 @@ def rfm_command(testcase): json['image'] = image_name for tc in testcases: + ci_extras = _valid_ci_extras(tc.check.ci_extras.get('gitlab', {})) + extra_artifacts = ci_extras.pop('artifacts', {}) + extra_artifact_paths = extra_artifacts.pop('paths', []) json[f'{tc.check.unique_name}'] = { 'stage': f'rfm-stage-{tc.level}', 'script': [rfm_command(tc)], 'artifacts': { - 'paths': [f'{tc.check.unique_name}-report.json'] + 'paths': [f'{tc.check.unique_name}-report.json', + *extra_artifact_paths], + **extra_artifacts }, 'needs': [t.check.unique_name for t in tc.deps], - **tc.check.ci_extras.get('gitlab', {}) + **ci_extras } max_level = max(max_level, tc.level) @@ -79,7 +97,7 @@ def rfm_command(testcase): def emit_pipeline(fp, testcases, child_pipeline_opts=None, backend='gitlab'): if backend != 'gitlab': - raise errors.ReframeError(f'unknown CI backend {backend!r}') + raise ReframeError(f'unknown CI backend {backend!r}') child_pipeline_opts = child_pipeline_opts or [] yaml.dump(_emit_gitlab_pipeline(testcases, child_pipeline_opts), stream=fp, diff --git a/unittests/test_ci.py b/unittests/test_ci.py index 0756db5577..5e7896c29e 100644 --- a/unittests/test_ci.py +++ b/unittests/test_ci.py @@ -13,18 +13,27 @@ import reframe.frontend.ci as ci import reframe.frontend.dependencies as dependencies import reframe.frontend.executors as executors +from reframe.core.exceptions import ReframeError from reframe.frontend.loader import RegressionCheckLoader +def _generate_test_cases(checks): + return dependencies.toposort( + dependencies.build_deps(executors.generate_testcases(checks))[0] + ) + + +@pytest.fixture +def hello_test(): + from unittests.resources.checks.hellocheck import HelloTest + return HelloTest() + + def test_ci_gitlab_pipeline(): loader = RegressionCheckLoader([ 'unittests/resources/checks_unlisted/deps_complex.py' ]) - cases = dependencies.toposort( - dependencies.build_deps( - executors.generate_testcases(loader.load_all()) - )[0] - ) + cases = _generate_test_cases(loader.load_all()) with io.StringIO() as fp: ci.emit_pipeline(fp, cases) pipeline = fp.getvalue() @@ -41,3 +50,55 @@ def test_ci_gitlab_pipeline(): schema = response.json() jsonschema.validate(yaml.safe_load(pipeline), schema) + + +def test_ci_gitlab_ci_extras(hello_test): + hello_test.ci_extras = { + 'gitlab': { + 'before_script': ['touch foo.txt'], + 'after_script': ['echo done'], + 'artifacts': { + 'paths': ['foo.txt'] + }, + 'only': { + 'changes': ['src/foo.c'] + } + } + } + cases = _generate_test_cases([hello_test]) + with io.StringIO() as fp: + ci.emit_pipeline(fp, cases) + pipeline = fp.getvalue() + + pipeline_json = yaml.safe_load(pipeline)['HelloTest'] + assert pipeline_json['before_script'] == ['touch foo.txt'] + assert pipeline_json['after_script'] == ['echo done'] + assert pipeline_json['artifacts'] == { + 'paths': ['HelloTest-report.json', 'foo.txt'] + } + assert pipeline_json['only'] == {'changes': ['src/foo.c']} + + +def test_ci_gitlab_ci_extras_invalid(hello_test): + with pytest.raises(TypeError): + hello_test.ci_extras = { + 'before_script': ['touch foo.txt'] + } + + with pytest.raises(TypeError): + hello_test.ci_extras = { + 'foolab': { + 'before_script': ['touch foo.txt'] + } + } + + # Check invalid keywords + for kwd in ('stage', 'script', 'needs'): + hello_test.ci_extras = { + 'gitlab': {kwd: 'something'} + } + cases = _generate_test_cases([hello_test]) + with io.StringIO() as fp: + with pytest.raises(ReframeError, + match=rf'invalid keyword found: {kwd!r}'): + ci.emit_pipeline(fp, cases) From 6e380ae9bd0379c91241117458061fcc9fe220ba Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 4 Apr 2023 22:55:10 +0200 Subject: [PATCH 5/7] Update docs --- reframe/core/pipeline.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index e61c48475a..65a247c2f9 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -965,16 +965,27 @@ def pipeline_hooks(cls): #: .. versionadded:: 4.2 #: - #: Options for CI pipeline passed in JSON format + #: Extra options to be passed to the child CI pipeline generated for this + #: test using the :option:`--ci-generate` option. #: - #: Example: If we want a pipeline to run only when files in - #: backend or src/main.c have changed, we add: + #: This variable is a dictionary whose keys refer the CI generate backend + #: and the values can be in any CI backend-specific format. #: - #: ci_options = {"only": {"changes": ["backend/*", "src/main.c"]} + #: Currently, the only key supported is ``'gitlab'`` and the values is a + #: Gitlab configuration in JSON format. For example, if we want a pipeline + #: to run only when files in ``backend`` or ``src/main.c`` have changed, + #: this variable should be set as follows: #: - #: :type: `dict` - #: :default: ``{}`` - ci_extras = variable(typ.Dict[typ.Str['gitlab'], object], value={}) + #: .. code-block:: python + #: + #: ci_extras = { + #: 'only': {'changes': ['backend/*', 'src/main.c']} + #: } + #: + #: :type: :class:`dict` or :class:`NoneType` + #: :default: :obj:`None` + ci_extras = variable(typ.Dict[typ.Str['gitlab'], object], type(None), + value=None) # Special variables From e3ff256d64915d36a6f20bd7c1a696daba3c9f62 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 4 Apr 2023 23:09:56 +0200 Subject: [PATCH 6/7] Fix unit tests --- reframe/frontend/ci.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reframe/frontend/ci.py b/reframe/frontend/ci.py index e4cd08257a..45efe6bda8 100644 --- a/reframe/frontend/ci.py +++ b/reframe/frontend/ci.py @@ -75,7 +75,8 @@ def _valid_ci_extras(extras): json['image'] = image_name for tc in testcases: - ci_extras = _valid_ci_extras(tc.check.ci_extras.get('gitlab', {})) + test_ci_extras = tc.check.ci_extras or {} + ci_extras = _valid_ci_extras(test_ci_extras.get('gitlab', {})) extra_artifacts = ci_extras.pop('artifacts', {}) extra_artifact_paths = extra_artifacts.pop('paths', []) json[f'{tc.check.unique_name}'] = { From 918d06c478156d24c9a8ceafc469b8fccea64ffc Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 5 Apr 2023 00:53:16 +0200 Subject: [PATCH 7/7] Revert default value of `ci_extras` --- reframe/core/pipeline.py | 7 +++---- reframe/frontend/ci.py | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 06f0a5ed31..65e09e2cef 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -981,10 +981,9 @@ def pipeline_hooks(cls): #: 'only': {'changes': ['backend/*', 'src/main.c']} #: } #: - #: :type: :class:`dict` or :class:`NoneType` - #: :default: :obj:`None` - ci_extras = variable(typ.Dict[typ.Str['gitlab'], object], type(None), - value=None) + #: :type: :class:`dict` + #: :default: ``{}`` + ci_extras = variable(typ.Dict[typ.Str['gitlab'], object], value={}) # Special variables diff --git a/reframe/frontend/ci.py b/reframe/frontend/ci.py index 45efe6bda8..e4cd08257a 100644 --- a/reframe/frontend/ci.py +++ b/reframe/frontend/ci.py @@ -75,8 +75,7 @@ def _valid_ci_extras(extras): json['image'] = image_name for tc in testcases: - test_ci_extras = tc.check.ci_extras or {} - ci_extras = _valid_ci_extras(test_ci_extras.get('gitlab', {})) + ci_extras = _valid_ci_extras(tc.check.ci_extras.get('gitlab', {})) extra_artifacts = ci_extras.pop('artifacts', {}) extra_artifact_paths = extra_artifacts.pop('paths', []) json[f'{tc.check.unique_name}'] = {