From 60fd1a49fcd13d87b06043bdf19ea763bd33d526 Mon Sep 17 00:00:00 2001 From: mahendrapaipuri Date: Wed, 25 Aug 2021 13:25:43 +0200 Subject: [PATCH 1/2] Add configurable options to generate CI function --- .gitignore | 1 + reframe/frontend/ci.py | 54 ++++++++++++++++++++++++++++++++++--- reframe/schemas/config.json | 31 +++++++++++++++++++-- 3 files changed, 80 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 4eaed2111c..af5ce57919 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ var/ *.egg-info/ .installed.cfg *.egg +external/ # Ignore Mac DS_Store files .DS_Store diff --git a/reframe/frontend/ci.py b/reframe/frontend/ci.py index c39d92e1a0..ce1b63e64b 100644 --- a/reframe/frontend/ci.py +++ b/reframe/frontend/ci.py @@ -3,7 +3,9 @@ # # SPDX-License-Identifier: BSD-3-Clause +import os import sys +import copy import yaml import reframe.core.exceptions as errors @@ -18,6 +20,19 @@ def _emit_gitlab_pipeline(testcases): prefix = 'rfm-stage/${CI_COMMIT_SHORT_SHA}' checkpath = config.get('general/0/check_search_path') recurse = config.get('general/0/check_search_recursive') + verbosity = 'v' * int(config.get('general/0/verbose')) + + # Collect the generate CI options + before_script = config.get('generate-ci/0/before_script') + after_script = config.get('generate-ci/0/after_script') + artifacts = config.get('generate-ci/0/artifacts') + artifacts_expiry = config.get('generate-ci/0/artifacts_expiry') + + # Need to append prefix to artifacts + if artifacts: + artifacts = [os.path.join(prefix, a) for a in artifacts] + else: + artifacts = [] def rfm_command(testcase): if config.filename != '': @@ -25,7 +40,6 @@ def rfm_command(testcase): else: config_opt = '' - report_file = f'{testcase.check.name}-report.json' if testcase.level: restore_files = ','.join( f'{t.check.name}-report.json' for t in tc.deps @@ -38,9 +52,11 @@ def rfm_command(testcase): f'--prefix={prefix}', config_opt, f'{" ".join("-c " + c for c in checkpath)}', f'-R' if recurse else '', - f'--report-file={report_file}', + f'--report-file={testcase.check.name}-report.json', + f'--report-junit={testcase.check.name}-report.xml', f'--restore-session={restore_files}' if restore_files else '', - '-n', testcase.check.name, '-r' + f'{"".join("-" + verbosity)}' if verbosity else '', + '-n', f'{testcase.check.name}$', '-r', # regex $ to indicate end ]) max_level = 0 # We need the maximum level to generate the stages section @@ -51,17 +67,47 @@ def rfm_command(testcase): }, 'stages': [] } + + # Name of the image used for CI. If user does not explicitly provide + # image keyword on the top of CI script, this variable does not exist + image_name = os.getenv('CI_JOB_IMAGE') + if image_name: + json = { + 'image': image_name, + **json, + } + for tc in testcases: json[f'{tc.check.name}'] = { 'stage': f'rfm-stage-{tc.level}', + 'before_script': copy.deepcopy(before_script), 'script': [rfm_command(tc)], + 'after_script': copy.deepcopy(after_script), 'artifacts': { - 'paths': [f'{tc.check.name}-report.json'] + 'paths': [f'{tc.check.name}-report.json', + f'{tc.check.name}-report.xml'] + artifacts, + 'expire_in': artifacts_expiry, }, 'needs': [t.check.name for t in tc.deps] } max_level = max(max_level, tc.level) + # Add a last job that gathers all artifacts from precedent jobs + json['GatherArtifacts'] = { + 'stage': f'rfm-stage-{max_level + 1}', + 'script': 'echo "Gathering artifacts from all jobs..."', + 'needs': [tc.check.name for tc in testcases], + 'artifacts': { + 'paths': [f'{tc.check.name}-report.json' for tc in testcases] + + [f'{tc.check.name}-report.xml' for tc in testcases] + + artifacts, + 'expire_in': artifacts_expiry, + }, + } + + # We incremented one stage by above gather artifacts job + max_level += 1 + json['stages'] = [f'rfm-stage-{m}' for m in range(max_level+1)] return json diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 8b1e3e990a..b72dc15a2f 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -247,7 +247,7 @@ "scheduler": { "type": "string", "enum": [ - "local", "pbs", "slurm", + "local", "oar", "pbs", "slurm", "sge", "squeue", "torque" ] }, @@ -370,7 +370,7 @@ "properties": { "name": { "type": "string", - "enum": ["local", "pbs", "sge", "slurm", "squeue", "torque"] + "enum": ["local", "oar", "pbs", "sge", "slurm", "squeue", "torque"] }, "ignore_reqnodenotavail": {"type": "boolean"}, "resubmit_on_errors": { @@ -385,6 +385,29 @@ "additionalProperties": false } }, + "generate-ci": { + "type": "array", + "items": { + "type": "object", + "properties": { + "before_script": { + "type": "array", + "items": {"type": "string"} + }, + "after_script": { + "type": "array", + "items": {"type": "string"} + }, + "artifacts": { + "type": "array", + "items": {"type": "string"} + }, + "target_systems": {"$ref": "#/defs/system_ref"}, + "artifacts_expiry": {"type": "string"} + }, + "additionalProperties": false + } + }, "logging": { "type": "array", "items": { @@ -481,6 +504,10 @@ "required": ["systems", "environments", "logging"], "additionalProperties": false, "defaults": { + "generate-ci/after_script": ["echo 'noop'"], + "generate-ci/before_script": ["echo 'noop'"], + "generate-ci/artifacts_expiry": "30 days", + "generate-ci/target_systems": ["*"], "environments/modules": [], "environments/variables": [], "environments/cc": "cc", From d31053e86273457c15f79efbe93904e7c0ed46b2 Mon Sep 17 00:00:00 2001 From: mahendrapaipuri Date: Wed, 25 Aug 2021 18:18:25 +0200 Subject: [PATCH 2/2] Place all artifacts into one folder in pipeline. Restrict configurable artifacts to rfm supported ones --- reframe/frontend/ci.py | 15 ++++++++++----- reframe/schemas/config.json | 5 ++++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/reframe/frontend/ci.py b/reframe/frontend/ci.py index ce1b63e64b..ab51262c14 100644 --- a/reframe/frontend/ci.py +++ b/reframe/frontend/ci.py @@ -30,7 +30,8 @@ def _emit_gitlab_pipeline(testcases): # Need to append prefix to artifacts if artifacts: - artifacts = [os.path.join(prefix, a) for a in artifacts] + artifacts = [os.path.join(prefix, a) if a in [ + 'stage', 'output'] else a for a in artifacts] else: artifacts = [] @@ -60,6 +61,8 @@ def rfm_command(testcase): ]) max_level = 0 # We need the maximum level to generate the stages section + art_folder = 'rfm-artifacts' # Folder where we place all artifacts + json = { 'cache': { 'key': '${CI_COMMIT_REF_SLUG}', @@ -95,12 +98,14 @@ def rfm_command(testcase): # Add a last job that gathers all artifacts from precedent jobs json['GatherArtifacts'] = { 'stage': f'rfm-stage-{max_level + 1}', - 'script': 'echo "Gathering artifacts from all jobs..."', + 'script': [f'echo \"Gathering artifacts from all jobs into ' + f'{art_folder} folder\"', + f'mkdir -p {art_folder}', + f'cp -r {" ".join(a for a in artifacts)} *.json *.xml ' + f'{art_folder}'], 'needs': [tc.check.name for tc in testcases], 'artifacts': { - 'paths': [f'{tc.check.name}-report.json' for tc in testcases] + - [f'{tc.check.name}-report.xml' for tc in testcases] + - artifacts, + 'paths': [art_folder], 'expire_in': artifacts_expiry, }, } diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index b72dc15a2f..a045063c83 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -400,7 +400,10 @@ }, "artifacts": { "type": "array", - "items": {"type": "string"} + "items": { + "type": "string", + "enum": ["perflogs", "output", "stage"] + } }, "target_systems": {"$ref": "#/defs/system_ref"}, "artifacts_expiry": {"type": "string"}