From c118dbae3c02c3d42e4b0ec48f9d67f37fd056f5 Mon Sep 17 00:00:00 2001 From: Francisco Enrique Cordova Gonzalez Date: Mon, 30 Jul 2018 09:33:57 -0700 Subject: [PATCH] cli: matrix execution (#420) Adds support for matrix execution. Fixes #362 --- .popper.yml | 31 ++++++++-------- .travis.yml | 2 +- ci/test-matrix-execution.sh | 30 ++++++++++++++++ cli/popper/commands/cmd_run.py | 66 ++++++++++++++++++++++++++-------- 4 files changed, 99 insertions(+), 30 deletions(-) create mode 100755 ci/test-matrix-execution.sh diff --git a/.popper.yml b/.popper.yml index c1defd330..ee4cc40a3 100644 --- a/.popper.yml +++ b/.popper.yml @@ -10,24 +10,25 @@ pipelines: path: ci stages: - setup - - test-default - - test-init - - test-require - - test-existing - - test-run + - test-add + - test-archive + - test-badge - test-ci - - test-stages - - test-env - test-cleanup - - test-workflow + - test-defaults + - test-env + - test-env-vars + - test-existing + - test-init + - test-matrix-execution - test-metadata - - test-add - - test-search + - test-mv - test-rm - - test-archive - - test-reset - - test-env-vars - - test-badge + - test-run + - test-search + - test-stages + - test-workflow + - teardown popperized: - github/popperized -badge-server-url: http://badges.falsifiable.us +badge-server-url: http://badges.falsifiable.us \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index b8e2e6be1..91b8fedb4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,5 +8,5 @@ script: - export PATH=$PATH:$PWD/cli/bin - popper run - USE_VIRTUALENV=1 popper run - - docker run --rm -e CI=1 -e POPPER_FIGSHARE_API_TOKEN="${POPPER_FIGSHARE_API_TOKEN}" -e POPPER_ZENODO_API_TOKEN="${POPPER_ZENODO_API_TOKEN}" -e POPPER_GITHUB_API_TOKEN="${POPPER_GITHUB_API_TOKEN}" -v `pwd`:/repo -v /var/run/docker.sock:/var/run/docker.sock --workdir=/repo falsifiable/popper:alpine-3.4 run --skip=test-env,test-badge + - docker run --rm -e CI=1 -e POPPER_FIGSHARE_API_TOKEN="${POPPER_FIGSHARE_API_TOKEN}" -e POPPER_ZENODO_API_TOKEN="${POPPER_ZENODO_API_TOKEN}" -e POPPER_GITHUB_API_TOKEN="${POPPER_GITHUB_API_TOKEN}" -v `pwd`:/repo -v /var/run/docker.sock:/var/run/docker.sock --workdir=/repo falsifiable/popper:alpine-3.4 run --skip=test-env,test-badge,test-matrix-execution - envs/push_images diff --git a/ci/test-matrix-execution.sh b/ci/test-matrix-execution.sh new file mode 100755 index 000000000..cabe59b2b --- /dev/null +++ b/ci/test-matrix-execution.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -ex + +source common-setup.sh + +init_test +# test matrix execution + +popper init mypipe --stages=one,two + +popper env-vars mypipe --add key1=val1 --add key2=val2 +popper env-vars mypipe --add key2=new_val_2 --add key3=val3 + +popper env mypipe --add alpine-3.4 +popper env mypipe --add debian-9 + +popper run + +test -f pipelines/mypipe/popper/host/0/one.sh.out +test -f pipelines/mypipe/popper/host/1/one.sh.out +test -f pipelines/mypipe/popper/alpine-3.4/0/one.sh.out +test -f pipelines/mypipe/popper/alpine-3.4/1/one.sh.out +test -f pipelines/mypipe/popper/debian-9/0/one.sh.out +test -f pipelines/mypipe/popper/debian-9/1/one.sh.out +test -f pipelines/mypipe/popper/host/0/one.sh.err +test -f pipelines/mypipe/popper/host/1/one.sh.err +test -f pipelines/mypipe/popper/alpine-3.4/0/one.sh.err +test -f pipelines/mypipe/popper/alpine-3.4/1/one.sh.err +test -f pipelines/mypipe/popper/debian-9/0/one.sh.err +test -f pipelines/mypipe/popper/debian-9/1/one.sh.err \ No newline at end of file diff --git a/cli/popper/commands/cmd_run.py b/cli/popper/commands/cmd_run.py index 4ad628de2..091b1ab1a 100755 --- a/cli/popper/commands/cmd_run.py +++ b/cli/popper/commands/cmd_run.py @@ -94,10 +94,13 @@ def cli(ctx, pipeline, timeout, skip, ignore_errors, output, for pipe_n, pipe_d in pipelines.items(): for env in pipe_d.get('envs', ['host']): - status = run_pipeline(project_root, pipe_n, pipe_d, env, timeout, - skip, ignore_errors, output) + args = [project_root, pipe_n, pipe_d, env, timeout, skip, + ignore_errors, output] + executions = get_executions_for_pipeline(pipe_d.get('vars')) + status = run_pipeline(*args, executions=executions) if status == 'FAIL' and not ignore_errors: break + if not len(pipelines): pu.info("No pipelines to execute") sys.exit(0) @@ -111,6 +114,26 @@ def cli(ctx, pipeline, timeout, skip, ignore_errors, output, pu.fail("Failed to execute pipeline") +def get_executions_for_pipeline(env_vars): + executions = [] + if env_vars: + for env_var in env_vars: + executions.append(env_var) + else: + executions.append("") + return executions + + +def set_env_vars(env_vars): + for env_var in env_vars: + os.environ[env_var] = env_vars[env_var] + + +def unset_env_vars(env_vars): + for env_var in env_vars: + del os.environ[env_var] + + def get_pipelines_to_execute(cwd, pipe_n, project_pipelines): pipelines = {} if pipe_n: @@ -222,11 +245,13 @@ def check_requirements(pipe_n, pipeline, requirement_level): def run_in_docker(project_root, pipe_n, pipe_d, env, timeout, skip, - ignore_errors, output_dir): + ignore_errors, output_dir, env_vars): abs_path = '{}/{}'.format(project_root, pipe_d['path']) docker_cmd = 'docker run --rm -v {0}:{0}'.format(project_root) docker_cmd += ' --workdir={}'.format(abs_path) + docker_cmd += ''.join([' -e {0}="{1}"'.format(k, env_vars[k]) + for k in env_vars]) if env_vars else '' if '/' in env: img = env @@ -366,24 +391,37 @@ def execute(stage, timeout, output_dir, bar=None): def run_pipeline(project_root, pipe_n, pipe_d, env, timeout, - skip, ignore_errors, output_dir): + skip, ignore_errors, output_dir, executions): timeout_parsed = pu.parse_timeout(timeout) skip_list = skip.split(',') if skip else [] click.echo('Executing pipeline: {}'.format(pipe_n)) - if os.path.isfile('/.dockerenv'): - return run_on_host(project_root, pipe_n, pipe_d, skip_list, - timeout_parsed, output_dir) - - if env != 'host': - return run_in_docker(project_root, pipe_n, pipe_d, env, timeout, skip, - ignore_errors, '{}/{}'.format( - output_dir, env.replace('/', '_'))) + status = 'SUCCESS' + for number_of_run, env_vars in enumerate(executions): + set_env_vars(env_vars) + if os.path.isfile('/.dockerenv'): + status = run_on_host(project_root, pipe_n, pipe_d, skip_list, + timeout_parsed, output_dir) + elif env != 'host': + status = \ + run_in_docker(project_root, pipe_n, pipe_d, env, + timeout, skip, ignore_errors, '{}/{}/{}'. + format(output_dir, env.replace('/', '_'), + number_of_run if env_vars else ""), + env_vars) + else: + status = \ + run_on_host(project_root, pipe_n, pipe_d, skip_list, + timeout_parsed, os.path. join(output_dir, 'host', + str(number_of_run) + if env_vars else "")) + unset_env_vars(env_vars) + if status == 'FAIL' and not ignore_errors: + break - return run_on_host(project_root, pipe_n, pipe_d, skip_list, - timeout_parsed, os.path.join(output_dir, 'host')) + return status class Unbuffered(object):