Skip to content

Commit

Permalink
make: add option to ignore components from build
Browse files Browse the repository at this point in the history 
* Closes reanahub#194.
  • Loading branch information
Diego Rodriguez committed Mar 26, 2020
1 parent 1b9697e commit 3ec6e96
Show file tree
Hide file tree
Showing 3 changed files with 82 additions and 19 deletions.
16 changes: 10 additions & 6 deletions Makefile
View file
Expand Up @@ -54,6 +54,7 @@ help:
@echo ' VENV_NAME Which Python virtual environment name to use? [default=reana]'
@echo ' SERVER_URL Setting a customized REANA Server hostname? [e.g. "https://example.org"; default is Minikube IP]'
@echo ' CLUSTER_FLAGS Which values need to be passed to Helm? [e.g. "debug.enabled=true,ui.enabled=true"; no flags are passed by default]'
@echo ' EXCLUDE_COMPONENTS Which REANA components should be excluded from the build? [e.g. reana-ui,reana-message-broker]'
@echo
@echo 'Examples:'
@echo
Expand All @@ -63,19 +64,22 @@ help:
@echo ' # Example 2: build and deploy REANA in production mode'
@echo ' $$ make build deploy'
@echo
@echo ' # Example 3: build and deploy REANA in development mode (with live code changes and debugging)'
@echo ' # Example 3: build REANA except REANA-UI and REANA-Message-Broker'
@echo ' $$ EXCLUDE_COMPONENTS=r-ui,r-m-broker make build'
@echo
@echo ' # Example 4: build and deploy REANA in development mode (with live code changes and debugging)'
@echo ' $$ CLUSTER_FLAGS=debug.enabled=true make build deploy'
@echo
@echo ' # Example 4: build and deploy REANA with a custom hostname including REANA-UI'
@echo ' # Example 5: build and deploy REANA with a custom hostname including REANA-UI'
@echo ' $$ CLUSTER_FLAGS=ui.enabled=true SERVER_URL=https://example.org make build deploy'
@echo
@echo ' # Example 5: run one small demo example to verify the build'
@echo ' # Example 6: run one small demo example to verify the build'
@echo ' $$ DEMO=reana-demo-helloworld make example'
@echo
@echo ' # Example 6: run several small examples to verify the build'
@echo ' # Example 7: run several small examples to verify the build'
@echo ' $$ make example'
@echo
@echo ' # Example 7: perform full CI build-and-test cycle'
@echo ' # Example 8: perform full CI build-and-test cycle'
@echo ' $$ make ci'
@echo
@echo ' # Example 8: perform full CI build-and-test cycle in an independent cluster'
Expand Down Expand Up @@ -125,7 +129,7 @@ build: # Build REANA client and cluster components.
fi && \
pip check && \
reana-dev git-submodule --update && \
reana-dev docker-build -b DEBUG=${DEBUG}
reana-dev docker-build -b DEBUG=${DEBUG} $(addprefix --exclude-components , ${EXCLUDE_COMPONENTS})

deploy: # Deploy/redeploy previously built REANA cluster.
@echo -e "\033[1;32m[$$(date +%Y-%m-%dT%H:%M:%S)]\033[1;33m reana:\033[0m\033[1m make deploy\033[0m"
Expand Down
78 changes: 65 additions & 13 deletions reana/cli.py
View file
Expand Up @@ -326,15 +326,26 @@ def find_standard_component_name(short_component_name):
:raise: exception in case more than one is found
"""
output = []
for component in REPO_LIST_ALL:
component_short_name = shorten_component_name(component)
if component_short_name == short_component_name:
output.append(component)
if len(output) == 1:
return output[0]
raise Exception('Component name {0} cannot be uniquely mapped.'.format(
'short_component_name'))
def _is_standard_name(component_name):
"""Detect whether the provided component name is already standard."""
prefixes = component_name.split('-')[:-1]
return all([len(n) > 1 for n in prefixes])

if _is_standard_name(short_component_name):
standard_component_name = short_component_name
else:
output = []
for component in REPO_LIST_ALL:
component_short_name = shorten_component_name(component)
if component_short_name == short_component_name:
output.append(component)
if len(output) == 1:
standard_component_name = output[0]
else:
raise Exception('Component name {0} cannot be uniquely '
'mapped.'.format(short_component_name))

return standard_component_name


def find_reana_srcdir():
Expand Down Expand Up @@ -426,7 +437,7 @@ def get_current_commit(srcdir):
shell=True).decode().rstrip('\r\n')


def select_components(components):
def select_components(components, exclude_components=None):
"""Return expanded and unified component name list based on input values.
:param components: A list of component name that may consist of:
Expand All @@ -443,7 +454,9 @@ def select_components(components):
to include several runable REANA demo examples;
* (7) special value 'ALL' that will expand to include
all REANA repositories.
:param exclude_components: A list of components to exclude.
:type components: list
:type exclude_components: list
:return: Unique standard component names.
:rtype: list
Expand Down Expand Up @@ -476,9 +489,40 @@ def select_components(components):
else:
display_message('Ignoring unknown component {0}.'.format(
component))

if exclude_components:
output = exclude_components_from_selection(output, exclude_components)

return list(output)


def exclude_components_from_selection(selection, exclude_components):
"""Exclude list of components from list of selections.
:param selection: List of selected components in standard naming form.
:param exclude_components: List of components to exclude, either in short
or standard naming form.
:type selection: set
:type exclude_components: list
:return: Set of selected components without ``exclude_components``, all in
standard naming form.
:rtype: set
"""
st_exclude_components = \
[find_standard_component_name(c) for c in exclude_components]
non_existing_exclude_components = \
set(st_exclude_components).difference(selection)
if non_existing_exclude_components:
display_message('Unkown component(s) to exclude: {}'.format(
non_existing_exclude_components))
sys.exit(1)

click.secho(
'Excluding component(s) {}'.format(st_exclude_components), fg='yellow')
return selection.difference(st_exclude_components)


def select_workflow_engines(workflow_engines):
"""Return known workflow engine names that REANA supports.
Expand Down Expand Up @@ -1177,6 +1221,8 @@ def git_push(component): # noqa: D301
'\'0.5.1-3-g75ae5ce\'')
@click.option('--component', '-c', multiple=True, default=['CLUSTER'],
help='Which components? [name|CLUSTER]')
@click.option('--exclude-components', default='',
help='Which components to exclude from build? [c1,c2,c3]')
@click.option('--build-arg', '-b', default='', multiple=True,
help='Any build arguments? (e.g. `-b DEBUG=1`)')
@click.option('--no-cache', is_flag=True)
Expand All @@ -1185,8 +1231,10 @@ def git_push(component): # noqa: D301
@click.option('-q', '--quiet', is_flag=True,
help='Suppress the build output and print image ID on success')
@cli.command(name='docker-build')
def docker_build(user, tag, component, build_arg,
no_cache, output_component_versions, quiet): # noqa: D301
@click.pass_context
def docker_build(ctx, user, tag, component, build_arg,
no_cache, output_component_versions, quiet,
exclude_components): # noqa: D301
"""Build REANA component images.
\b
Expand All @@ -1205,6 +1253,7 @@ def docker_build(user, tag, component, build_arg,
to include several runable REANA demo examples;
* (7) special value 'ALL' that will expand to include
all REANA repositories.
:param exclude_components: List of components to exclude from the build.
:param user: DockerHub organisation or user name. [default=reanahub]
:param tag: Docker image tag to generate. Default 'latest'. Use 'auto' to
generate git-tag-based value such as '0.5.1-3-g75ae5ce'.
Expand All @@ -1214,14 +1263,17 @@ def docker_build(user, tag, component, build_arg,
tags. Useful when using `--tag auto` since every REANA component
will have a different tag.
:type component: str
:type exclude_components: str
:type user: str
:type tag: str
:type build_arg: str
:type no_cache: bool
:type output_component_versions: File
:type quiet: bool
"""
components = select_components(component)
if exclude_components:
exclude_components = exclude_components.split(',')
components = select_components(component, exclude_components)
built_components_versions_tags = []
for component in components:
component_tag = tag
Expand Down
7 changes: 7 additions & 0 deletions tests/test_cli.py
View file
Expand Up @@ -89,6 +89,12 @@ def test_select_components():
output_obtained = select_components(input_value)
assert output_obtained.sort() == output_expected.sort()

num_excluded = 2
exclude_components = REPO_LIST_CLUSTER[:num_excluded]
output_obtained = select_components(REPO_LIST_CLUSTER, exclude_components)
assert len(output_obtained) == (len(REPO_LIST_CLUSTER) - num_excluded)
assert not set(exclude_components).intersection(output_obtained)


def test_select_workflow_engines():
"""Tests for select_workflow_engines()."""
Expand All @@ -115,6 +121,7 @@ def test_find_standard_component_name():
('reana', 'reana'),
('r-server', 'reana-server'),
('r-j-controller', 'reana-job-controller'),
('reana-ui', 'reana-ui')
):
output_obtained = find_standard_component_name(input_value)
assert output_obtained == output_expected
Expand Down

0 comments on commit 3ec6e96

Please sign in to comment.