From 3ec6e962cbfc53de86b7a5c9820f15c538347952 Mon Sep 17 00:00:00 2001 From: Diego Rodriguez Date: Wed, 25 Mar 2020 15:27:45 +0100 Subject: [PATCH] make: add option to ignore components from build * Closes #194. --- Makefile | 16 ++++++---- reana/cli.py | 78 +++++++++++++++++++++++++++++++++++++++-------- tests/test_cli.py | 7 +++++ 3 files changed, 82 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 842f173e..25f68921 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,7 @@ help: @echo ' VENV_NAME Which Python virtual environment name to use? [default=reana]' @echo ' SERVER_URL Setting a customized REANA Server hostname? [e.g. "https://example.org"; default is Minikube IP]' @echo ' CLUSTER_FLAGS Which values need to be passed to Helm? [e.g. "debug.enabled=true,ui.enabled=true"; no flags are passed by default]' + @echo ' EXCLUDE_COMPONENTS Which REANA components should be excluded from the build? [e.g. reana-ui,reana-message-broker]' @echo @echo 'Examples:' @echo @@ -63,19 +64,22 @@ help: @echo ' # Example 2: build and deploy REANA in production mode' @echo ' $$ make build deploy' @echo - @echo ' # Example 3: build and deploy REANA in development mode (with live code changes and debugging)' + @echo ' # Example 3: build REANA except REANA-UI and REANA-Message-Broker' + @echo ' $$ EXCLUDE_COMPONENTS=r-ui,r-m-broker make build' + @echo + @echo ' # Example 4: build and deploy REANA in development mode (with live code changes and debugging)' @echo ' $$ CLUSTER_FLAGS=debug.enabled=true make build deploy' @echo - @echo ' # Example 4: build and deploy REANA with a custom hostname including REANA-UI' + @echo ' # Example 5: build and deploy REANA with a custom hostname including REANA-UI' @echo ' $$ CLUSTER_FLAGS=ui.enabled=true SERVER_URL=https://example.org make build deploy' @echo - @echo ' # Example 5: run one small demo example to verify the build' + @echo ' # Example 6: run one small demo example to verify the build' @echo ' $$ DEMO=reana-demo-helloworld make example' @echo - @echo ' # Example 6: run several small examples to verify the build' + @echo ' # Example 7: run several small examples to verify the build' @echo ' $$ make example' @echo - @echo ' # Example 7: perform full CI build-and-test cycle' + @echo ' # Example 8: perform full CI build-and-test cycle' @echo ' $$ make ci' @echo @echo ' # Example 8: perform full CI build-and-test cycle in an independent cluster' @@ -125,7 +129,7 @@ build: # Build REANA client and cluster components. fi && \ pip check && \ reana-dev git-submodule --update && \ - reana-dev docker-build -b DEBUG=${DEBUG} + reana-dev docker-build -b DEBUG=${DEBUG} $(addprefix --exclude-components , ${EXCLUDE_COMPONENTS}) deploy: # Deploy/redeploy previously built REANA cluster. @echo -e "\033[1;32m[$$(date +%Y-%m-%dT%H:%M:%S)]\033[1;33m reana:\033[0m\033[1m make deploy\033[0m" diff --git a/reana/cli.py b/reana/cli.py index 9e22b131..4234fa4a 100644 --- a/reana/cli.py +++ b/reana/cli.py @@ -326,15 +326,26 @@ def find_standard_component_name(short_component_name): :raise: exception in case more than one is found """ - output = [] - for component in REPO_LIST_ALL: - component_short_name = shorten_component_name(component) - if component_short_name == short_component_name: - output.append(component) - if len(output) == 1: - return output[0] - raise Exception('Component name {0} cannot be uniquely mapped.'.format( - 'short_component_name')) + def _is_standard_name(component_name): + """Detect whether the provided component name is already standard.""" + prefixes = component_name.split('-')[:-1] + return all([len(n) > 1 for n in prefixes]) + + if _is_standard_name(short_component_name): + standard_component_name = short_component_name + else: + output = [] + for component in REPO_LIST_ALL: + component_short_name = shorten_component_name(component) + if component_short_name == short_component_name: + output.append(component) + if len(output) == 1: + standard_component_name = output[0] + else: + raise Exception('Component name {0} cannot be uniquely ' + 'mapped.'.format(short_component_name)) + + return standard_component_name def find_reana_srcdir(): @@ -426,7 +437,7 @@ def get_current_commit(srcdir): shell=True).decode().rstrip('\r\n') -def select_components(components): +def select_components(components, exclude_components=None): """Return expanded and unified component name list based on input values. :param components: A list of component name that may consist of: @@ -443,7 +454,9 @@ def select_components(components): to include several runable REANA demo examples; * (7) special value 'ALL' that will expand to include all REANA repositories. + :param exclude_components: A list of components to exclude. :type components: list + :type exclude_components: list :return: Unique standard component names. :rtype: list @@ -476,9 +489,40 @@ def select_components(components): else: display_message('Ignoring unknown component {0}.'.format( component)) + + if exclude_components: + output = exclude_components_from_selection(output, exclude_components) + return list(output) +def exclude_components_from_selection(selection, exclude_components): + """Exclude list of components from list of selections. + + :param selection: List of selected components in standard naming form. + :param exclude_components: List of components to exclude, either in short + or standard naming form. + :type selection: set + :type exclude_components: list + + :return: Set of selected components without ``exclude_components``, all in + standard naming form. + :rtype: set + """ + st_exclude_components = \ + [find_standard_component_name(c) for c in exclude_components] + non_existing_exclude_components = \ + set(st_exclude_components).difference(selection) + if non_existing_exclude_components: + display_message('Unkown component(s) to exclude: {}'.format( + non_existing_exclude_components)) + sys.exit(1) + + click.secho( + 'Excluding component(s) {}'.format(st_exclude_components), fg='yellow') + return selection.difference(st_exclude_components) + + def select_workflow_engines(workflow_engines): """Return known workflow engine names that REANA supports. @@ -1177,6 +1221,8 @@ def git_push(component): # noqa: D301 '\'0.5.1-3-g75ae5ce\'') @click.option('--component', '-c', multiple=True, default=['CLUSTER'], help='Which components? [name|CLUSTER]') +@click.option('--exclude-components', default='', + help='Which components to exclude from build? [c1,c2,c3]') @click.option('--build-arg', '-b', default='', multiple=True, help='Any build arguments? (e.g. `-b DEBUG=1`)') @click.option('--no-cache', is_flag=True) @@ -1185,8 +1231,10 @@ def git_push(component): # noqa: D301 @click.option('-q', '--quiet', is_flag=True, help='Suppress the build output and print image ID on success') @cli.command(name='docker-build') -def docker_build(user, tag, component, build_arg, - no_cache, output_component_versions, quiet): # noqa: D301 +@click.pass_context +def docker_build(ctx, user, tag, component, build_arg, + no_cache, output_component_versions, quiet, + exclude_components): # noqa: D301 """Build REANA component images. \b @@ -1205,6 +1253,7 @@ def docker_build(user, tag, component, build_arg, to include several runable REANA demo examples; * (7) special value 'ALL' that will expand to include all REANA repositories. + :param exclude_components: List of components to exclude from the build. :param user: DockerHub organisation or user name. [default=reanahub] :param tag: Docker image tag to generate. Default 'latest'. Use 'auto' to generate git-tag-based value such as '0.5.1-3-g75ae5ce'. @@ -1214,6 +1263,7 @@ def docker_build(user, tag, component, build_arg, tags. Useful when using `--tag auto` since every REANA component will have a different tag. :type component: str + :type exclude_components: str :type user: str :type tag: str :type build_arg: str @@ -1221,7 +1271,9 @@ def docker_build(user, tag, component, build_arg, :type output_component_versions: File :type quiet: bool """ - components = select_components(component) + if exclude_components: + exclude_components = exclude_components.split(',') + components = select_components(component, exclude_components) built_components_versions_tags = [] for component in components: component_tag = tag diff --git a/tests/test_cli.py b/tests/test_cli.py index b1112dc5..cd87cf8f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -89,6 +89,12 @@ def test_select_components(): output_obtained = select_components(input_value) assert output_obtained.sort() == output_expected.sort() + num_excluded = 2 + exclude_components = REPO_LIST_CLUSTER[:num_excluded] + output_obtained = select_components(REPO_LIST_CLUSTER, exclude_components) + assert len(output_obtained) == (len(REPO_LIST_CLUSTER) - num_excluded) + assert not set(exclude_components).intersection(output_obtained) + def test_select_workflow_engines(): """Tests for select_workflow_engines().""" @@ -115,6 +121,7 @@ def test_find_standard_component_name(): ('reana', 'reana'), ('r-server', 'reana-server'), ('r-j-controller', 'reana-job-controller'), + ('reana-ui', 'reana-ui') ): output_obtained = find_standard_component_name(input_value) assert output_obtained == output_expected