Skip to content

Commit

Permalink
Merge 45fe7a6 into 7dffa6d
Browse files Browse the repository at this point in the history
  • Loading branch information
@mvidalgarcia
mvidalgarcia committed Aug 11, 2020
2 parents 7dffa6d + 45fe7a6 commit 2d3182c
Show file tree
Hide file tree
Showing 2 changed files with 203 additions and 75 deletions.
155 changes: 80 additions & 75 deletions reana/reana_dev/git.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,11 @@
)
from reana.reana_dev.utils import (
display_message,
fetch_latest_pypi_version,
get_srcdir,
run_command,
select_components,
update_module_in_cluster_components,
)


Expand Down Expand Up @@ -724,95 +726,98 @@ def git_push(component): # noqa: D301
run_command(cmd, component)


@git_commands.command(name="git-shared-modules-upgrade")
@git_commands.command(name="git-upgrade-shared-modules")
@click.option(
"--commit-and-publish",
"--component",
"-c",
multiple=True,
default=["CLUSTER", "CLIENT"],
help="Which components? [shortname|name|.|CLUSTER|ALL]",
)
@click.option(
"--use-latest-known-tag",
is_flag=True,
default=False,
help="Should the upgrade use the latest known tag of the shared modules?"
" If so, the latest known tag of the shared modules will be used to upgrade the"
" provided components. Else, the upgrade will only occur if the latest commit of the"
" shared modules points at a tag, in other case, the command will be aborted.",
)
@click.option(
"--amend",
is_flag=True,
default=False,
help="Should the changes be committed and pull requests created?"
" If so, a commit and a PR with"
' "installation: shared packages version bump" message will'
" be created for all affected CLUSTER components.",
help="Should the changes be integrated as part of the latest commit of each component?.",
)
def git_shared_modules_upgrade(commit_and_publish):
"""Upgrade all cluster components to latest REANA-Commons version."""
@click.option(
"--push",
is_flag=True,
default=False,
help="Should the feature branches with the upgrades be pushed?.",
)
@click.pass_context
def git_upgrade_shared_modules(
ctx, component, use_latest_known_tag, amend, push
): # noqa: D301
"""Upgrade selected components to latest REANA-Commons/DB version.
def _fetch_latest_pypi_version(package):
"""Fetch latest released version of a package."""
import requests
\b
:param components: The option ``component`` can be repeated. The value may
consist of:
* (1) standard component name such as
'reana-workflow-controller';
* (2) short component name such as 'r-w-controller';
* (3) special value '.' indicating component of the
current working directory;
* (4) special value 'CLUSTER' that will expand to
cover all REANA cluster components [default];
* (5) special value 'CLIENT' that will expand to
cover all REANA client components;
* (6) special value 'DEMO' that will expand
to include several runable REANA demo examples;
* (7) special value 'ALL' that will expand to include
all REANA repositories.
:type component: str
"""

pypi_rc_info = requests.get(
"https://pypi.python.org/pypi/{}/json".format(package)
)
return sorted(pypi_rc_info.json()["releases"].keys())[-1]

def _update_module_in_cluster_components(module, new_version):
"""Update the specified module version in all affected components."""
components_to_update = {
"reana-commons": COMPONENTS_USING_SHARED_MODULE_COMMONS,
"reana-db": COMPONENTS_USING_SHARED_MODULE_DB,
}
for component in components_to_update[module]:
update_setup_py_dep_cmd = (
'LN=`cat setup.py | grep -n -e "{module}.*>=" | cut -f1 -d: `'
'&& sed -i.bk "`echo $LN`s/>=.*,</>={new_version},</" '
"setup.py && rm setup.py.bk".format(
module=module, new_version=new_version
)
def _create_commit_or_amend(components):
for component in components:
commit_cmd = 'git commit -m "installation: bump shared modules"'
if amend:
commit_cmd = "git commit --amend --no-edit"

files_to_commit = ["setup.py"]
if os.path.exists(get_srcdir(component) + os.sep + "requirements.txt"):
files_to_commit.append("requirements.txt")
run_command(
f"git add {' '.join(files_to_commit)} && {commit_cmd}", component,
)
run_command(update_setup_py_dep_cmd, component)

def _commit_and_publish_version_bumps(components):
"""Create commit and version bump PRs for all specified components."""
def _push_to_origin(components):
for component in components:
has_changes = run_command(
"git status --porcelain --untracked-files=no | "
"grep setup.py || echo",
component=component,
return_output=True,
branch = run_command(
"git branch --show-current", component, return_output=True
)
run_command(
f"git push --force origin {branch}", component,
)
if has_changes:
branch_name = datetime.datetime.now().strftime("version-bump-%Y%m%d")
create_branch = "git checkout -b {}".format(branch_name)
run_command(create_branch, component)
create_commit = (
"git add setup.py && "
"git commit "
'-m "installation: shared packages version bump"'
)
run_command(create_commit, component)
create_pr = "hub pull-request -p --no-edit && hub pr list -L 1"
run_command(create_pr, component)
else:
click.echo("{} has no changes, skipping.".format(component))

cluster_components_with_shared_modules = set(
COMPONENTS_USING_SHARED_MODULE_DB
).union(set(COMPONENTS_USING_SHARED_MODULE_COMMONS))

components = select_components(component)

for module in REPO_LIST_SHARED:
last_version = _fetch_latest_pypi_version(module)
_update_module_in_cluster_components(module, last_version)
click.secho(
"✅ {module} updated to: {last_version}".format(
module=module, last_version=last_version
),
bold=True,
fg="green",
last_version = fetch_latest_pypi_version(module)
update_module_in_cluster_components(
module,
last_version,
components_to_update=components,
use_latest_known_tag=use_latest_known_tag,
force_feature_branch=True,
)

if commit_and_publish:
click.secho("Creating version bump commits and pull requests...")
_commit_and_publish_version_bumps(cluster_components_with_shared_modules)
click.secho(
"\nVersion bump commits and pull requests created ✨", bold=True, fg="green"
)
click.secho(
"PR list 👉 https://github.com/search?q="
"org%3Areanahub+is%3Apr+is%3Aopen&"
"unscoped_q=is%3Apr+is%3Aopen",
fg="green",
)
_create_commit_or_amend(components)
ctx.invoke(git_diff, component=component)
if push:
_push_to_origin(components)


git_commands_list = list(git_commands.commands.values())
123 changes: 123 additions & 0 deletions reana/reana_dev/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
import click

from reana.config import (
COMPONENTS_USING_SHARED_MODULE_COMMONS,
COMPONENTS_USING_SHARED_MODULE_DB,
REPO_LIST_ALL,
REPO_LIST_CLIENT,
REPO_LIST_CLUSTER,
Expand Down Expand Up @@ -335,3 +337,124 @@ def get_prefixed_component_name(component):
:return: Prefixed name.
"""
return "-".join([INSTANCE_NAME, component])


def fetch_latest_pypi_version(package):
"""Fetch latest released version of a package."""
import requests

pypi_rc_info = requests.get("https://pypi.python.org/pypi/{}/json".format(package))
return sorted(pypi_rc_info.json()["releases"].keys())[-1]


def is_last_commit_tagged(package):
"""Check whether the last commit of the module points at a tag."""
tag = run_command("git tag --points-at", package, return_output=True)
return bool(tag)


def is_feature_branch(component):
"""Check whether component current branch is different from master."""
return get_current_branch(get_srcdir(component)) != "master"


def replace_string(
file_=None, find=None, replace=None, line_selector_regex=None, component=None
):
"""Replace the old string with the new one in the specified file.
:param file_: filename where the replacement takes place
(e.g. 'setup.py', 'requirements.txt')
:param find: current string regex
:param replace: new string regex
:param line_selector_regex: grep expression to identify file line of the replacement
:param component: standard component name where to run the command
:type file_: str
:type find: str
:type replace: str
:type line_selector_regex: str
:type component: str
"""
line = ""
if line_selector_regex:
line = run_command(
f'cat {file_} | grep -n -e "{line_selector_regex}" | cut -f1 -d: ',
return_output=True,
)

cmd = (
f"sed -i.bk '{line}s/{find}/{replace}/' {file_} && [ -e {file_}.bk ]"
f" && rm {file_}.bk" # Compatibility with BSD sed
)

run_command(cmd, component)


def update_module_in_cluster_components(
module,
new_version,
components_to_update=None,
use_latest_known_tag=True,
force_feature_branch=False,
):
"""Update the specified module version in all affected components."""
updatable_components = {
"reana-commons": COMPONENTS_USING_SHARED_MODULE_COMMONS + ["reana-client"],
"reana-db": COMPONENTS_USING_SHARED_MODULE_DB,
}[module]

if components_to_update:
components_to_update = set(components_to_update).intersection(
set(updatable_components)
)
else:
components_to_update = updatable_components

if (
components_to_update
and not use_latest_known_tag
and not is_last_commit_tagged(module)
):
click.secho(
f"[ERROR] Last commit of {module} does not point to a tag."
" Use `--use-latest-known-tag` if you want to proceed using the latest"
" known tag.",
err=True,
fg="red",
),
sys.exit(1)

for component in components_to_update:
if force_feature_branch and not is_feature_branch(component):
click.secho(
f"[ERROR] Component {component} current branch is master."
" Must be a feature branch.",
err=True,
fg="red",
),
sys.exit(1)

replace_string(
file_="setup.py",
find=">=.*,<",
replace=f">={new_version},<",
line_selector_regex=f"{module}.*>=",
component=component,
)
if os.path.exists(get_srcdir(component) + os.sep + "requirements.txt"):
replace_string(
file_="requirements.txt",
find="==.*#",
replace=f"=={new_version}\t#",
line_selector_regex=f"{module}.*==",
component=component,
)

if components_to_update:
click.secho(
"✅ {module} updated to: {last_version}".format(
module=module, last_version=new_version
),
bold=True,
fg="green",
)

0 comments on commit 2d3182c

Please sign in to comment.