From 3308962d2f653bb5c8e1876764ad00789b245da1 Mon Sep 17 00:00:00 2001 From: Diego Rodriguez Date: Wed, 5 Aug 2020 13:25:31 +0200 Subject: [PATCH] cli: add release-helm command * Closes #363. --- .gitignore | 3 +- reana/reana_dev/cli.py | 2 + reana/reana_dev/git.py | 22 ++++----- reana/reana_dev/helm.py | 99 ++++++++++++++++++++++++++++++++++++++ reana/reana_dev/release.py | 75 ++++++++++++++++++++++------- reana/reana_dev/utils.py | 26 +++++++++- 6 files changed, 195 insertions(+), 32 deletions(-) create mode 100644 reana/reana_dev/helm.py diff --git a/.gitignore b/.gitignore index 749e9fdf..a66461ab 100644 --- a/.gitignore +++ b/.gitignore @@ -74,4 +74,5 @@ target/ helm/reana/Chart.lock # Helm releases -.deploy +.cr-release-packages +.cr-index diff --git a/reana/reana_dev/cli.py b/reana/reana_dev/cli.py index dd169d42..c503cd3c 100644 --- a/reana/reana_dev/cli.py +++ b/reana/reana_dev/cli.py @@ -14,6 +14,7 @@ from reana.reana_dev.cluster import cluster_commands_list from reana.reana_dev.docker import docker_commands_list from reana.reana_dev.git import git_commands_list +from reana.reana_dev.helm import helm_commands_list from reana.reana_dev.kind import kind_commands_list from reana.reana_dev.kubectl import kubectl_commands_list from reana.reana_dev.python import python_commands_list @@ -178,5 +179,6 @@ def help(): + python_commands_list + run_commands_list + release_commands_list + + helm_commands_list ): reana_dev.add_command(cmd) diff --git a/reana/reana_dev/git.py b/reana/reana_dev/git.py index bccc6566..33c6d4fb 100644 --- a/reana/reana_dev/git.py +++ b/reana/reana_dev/git.py @@ -212,6 +212,15 @@ def is_last_commit_release_commit(package): return current_commit.split()[1] == "release:" +def git_push_to_origin(components): + """Push current branch to origin.""" + for component in components: + branch = run_command("git branch --show-current", component, return_output=True) + run_command( + f"git push --force origin {branch}", component, + ) + + @click.group() def git_commands(): """Git commands group.""" @@ -875,15 +884,6 @@ def _create_commit_or_amend(components): f"git add {' '.join(files_to_commit)} && {commit_cmd}", component, ) - def _push_to_origin(components): - for component in components: - branch = run_command( - "git branch --show-current", component, return_output=True - ) - run_command( - f"git push --force origin {branch}", component, - ) - components = select_components(component) for module in REPO_LIST_SHARED: @@ -897,9 +897,9 @@ def _push_to_origin(components): ) _create_commit_or_amend(components) - ctx.invoke(git_diff, component=component) + ctx.invoke(git_diff, component=[component]) if push: - _push_to_origin(components) + git_push_to_origin(components) @click.option( diff --git a/reana/reana_dev/helm.py b/reana/reana_dev/helm.py new file mode 100644 index 00000000..b809b197 --- /dev/null +++ b/reana/reana_dev/helm.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +# +# This file is part of REANA. +# Copyright (C) 2020 CERN. +# +# REANA is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""`reana-dev`'s Helm commands.""" + +import os +import re +import sys + +import click + +from reana.config import REPO_LIST_CLUSTER +from reana.reana_dev.git import ( + git_diff, + git_is_current_version_tagged, + git_push_to_origin, +) +from reana.reana_dev.utils import ( + display_message, + get_component_version_as_semver2, + get_srcdir, + is_component_dockerised, +) + + +@click.group() +def helm_commands(): + """Helm commands group.""" + + +@click.option("--user", "-u", default="reanahub", help="DockerHub user name [reanahub]") +@click.option( + "--push", + is_flag=True, + default=False, + help="Should the feature branch with the upgrade be pushed to origin?", +) +@helm_commands.command(name="helm-upgrade-components") +@click.pass_context +def helm_upgrade_components(ctx, user, push): # noqa: D301 + """Upgrade REANA Helm dependencies.""" + + def _update_values_yaml(new_docker_images): + """Update all images in ``values.yaml``, skipping the ones up to date.""" + values_yaml_relative_path = "helm/reana/values.yaml" + values_yaml_abs_path = os.path.join( + get_srcdir("reana"), values_yaml_relative_path + ) + values_yaml = "" + + with open(values_yaml_abs_path) as f: + values_yaml = f.read() + for docker_image in new_docker_images: + image_name, _ = docker_image.split(":") + if image_name in values_yaml: + values_yaml = re.sub( + f"{image_name}:.*", lambda _: docker_image, values_yaml, count=1 + ) + + with open(values_yaml_abs_path, "w") as f: + f.write(values_yaml) + + display_message( + f"{values_yaml_relative_path} successfully updated.", component="reana" + ) + + remaining_docker_releases = [] + new_docker_images = [] + for component in REPO_LIST_CLUSTER: + if not is_component_dockerised(component): + continue + if not git_is_current_version_tagged(component): + remaining_docker_releases.append(component) + else: + new_docker_images.append( + f"{user}/{component}:{get_component_version_as_semver2(component)}" + ) + + if remaining_docker_releases: + line_by_line_missing_releases = "\n".join(remaining_docker_releases) + click.secho( + "The following components are missing to be released:\n" + f"{line_by_line_missing_releases}", + fg="red", + ) + sys.exit(1) + + _update_values_yaml(new_docker_images) + ctx.invoke(git_diff, component=["reana"]) + if push: + git_push_to_origin(["reana"]) + + +helm_commands_list = list(helm_commands.commands.values()) diff --git a/reana/reana_dev/release.py b/reana/reana_dev/release.py index 4ee0209a..3e6fc30d 100644 --- a/reana/reana_dev/release.py +++ b/reana/reana_dev/release.py @@ -8,11 +8,13 @@ """`reana-dev`'s release commands.""" +import os import sys +import tempfile +from shutil import which from time import sleep import click -import semver from reana.reana_dev.docker import docker_push from reana.reana_dev.git import ( @@ -23,13 +25,12 @@ from reana.reana_dev.utils import ( display_message, fetch_latest_pypi_version, + get_component_version_as_semver2, get_current_component_version_from_source_files, - get_current_tag, + get_srcdir, is_component_dockerised, - parse_pep440_version, run_command, select_components, - translate_pep440_to_semver2, ) @@ -112,21 +113,8 @@ def release_docker(ctx, component, user, image_name): # noqa: D301 cannot_release_on_dockerhub.append(component_) is_component_releasable(component_, exit_code=True, display=True) - current_tag = get_current_tag(component_) full_image_name = f"{user}/{image_name or component_}" - docker_tag = "" - - if parse_pep440_version(current_tag): - docker_tag = translate_pep440_to_semver2(current_tag) - elif semver.VersionInfo.isvalid(current_tag): - docker_tag = current_tag - else: - display_message( - f"The component's latest tag ({current_tag}) is not a " - "valid version (nor PEP440 nor semver2 compliant).", - component_, - ) - sys.exit(1) + docker_tag = get_component_version_as_semver2(component_) run_command( f"docker tag {full_image_name}:latest {full_image_name}:{docker_tag}", @@ -202,4 +190,55 @@ def release_pypi(ctx, component, timeout): # noqa: D301 click.secho(f"{component} successfully released on PyPI", fg="green") +@click.option("--user", "-u", default="reanahub", help="DockerHub user name [reanahub]") +@release_commands.command(name="release-helm") +@click.pass_context +def release_helm(ctx, user): # noqa: D301 + """Release REANA as a Helm chart.""" + component = "reana" + version = get_current_component_version_from_source_files(component) + is_chart_releaser_installed = which("cr") + github_pages_branch = "gh-pages" + package_path = ".cr-release-packages" + index_path = ".cr-index" + repository = f"https://{user}.github.io/{component}" + + is_component_releasable(component, exit_code=True, display=True) + if not is_chart_releaser_installed: + click.secho( + "Please install chart-releaser to be able to do a Helm release", fg="red", + ) + sys.exit(1) + + if not os.getenv("CR_TOKEN"): + click.secho( + "Please provide your GitHub token as CR_TOKEN environment variable", + fg="red", + ) + sys.exit(1) + + for cmd in [ + f"rm -rf {package_path}", + f"mkdir {package_path}", + f"rm -rf {index_path}", + f"mkdir {index_path}", + f"helm package helm/reana --destination {package_path} --dependency-update", + f"cr upload -o {user} -r {component}", + f"cr index -o {user} -r {component} -c {repository}", + ]: + run_command(cmd, component) + + with tempfile.TemporaryDirectory() as gh_pages_worktree: + run_command( + f"git worktree add '{gh_pages_worktree}' gh-pages && " + f"cd {gh_pages_worktree} && " + f"cp -f {get_srcdir(component) + os.sep + index_path}/index.yaml {gh_pages_worktree}/index.yaml && " + f"git add index.yaml && " + f"git commit -m 'index.yaml: {version}' && " + f"git push origin {github_pages_branch} && " + f"cd - && " + f"git worktree remove '{gh_pages_worktree}'", + ) + + release_commands_list = list(release_commands.commands.values()) diff --git a/reana/reana_dev/utils.py b/reana/reana_dev/utils.py index 0a571f8f..08955de9 100644 --- a/reana/reana_dev/utils.py +++ b/reana/reana_dev/utils.py @@ -663,7 +663,9 @@ def translate_pep440_to_semver2(pep440_version): number = parsed_pep440_version.pre[1] dev_post_pre_semver2 = f"{prefix}.{number}" - semver2_version_string = f"{parsed_pep440_version.major}.{parsed_pep440_version.minor}.{parsed_pep440_version.micro}-{dev_post_pre_semver2}" + semver2_version_string = f"{parsed_pep440_version.major}.{parsed_pep440_version.minor}.{parsed_pep440_version.micro}" + if dev_post_pre_semver2: + semver2_version_string += f"-{dev_post_pre_semver2}" if semver.VersionInfo.isvalid(semver2_version_string): return semver2_version_string else: @@ -713,7 +715,7 @@ def get_current_tag(component): :param component: standard component name :type component: str """ - cmd = "git describe --tags" + cmd = "git describe --tags --abbrev=0" return run_command(cmd, component, return_output=True, display=True) @@ -724,3 +726,23 @@ def validate_mode_option(ctx, param, value): "Supported values are '{}'.".format("', '".join(CLUSTER_DEPLOYMENT_MODES)) ) return value + + +def get_component_version_as_semver2(component): + """Get Docker release friendly component version (semver2).""" + docker_tag = "" + current_tag = get_current_tag(component) + + if parse_pep440_version(current_tag): + docker_tag = translate_pep440_to_semver2(current_tag) + elif semver.VersionInfo.isvalid(current_tag): + docker_tag = current_tag + else: + display_message( + f"The component's latest tag ({current_tag}) is not a " + "valid version (nor PEP440 nor semver2 compliant).", + component, + ) + sys.exit(1) + + return docker_tag