diff --git a/.ansible-lint b/.ansible-lint index 31331b1..d835026 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -4,3 +4,4 @@ exclude_paths: skip_list: - galaxy[no-changelog] + - meta-runtime[unsupported-version] diff --git a/.github/workflows/test-collection.yml b/.github/workflows/test-collection.yml index 50da212..e672abc 100644 --- a/.github/workflows/test-collection.yml +++ b/.github/workflows/test-collection.yml @@ -7,6 +7,20 @@ jobs: test: name: Test github role runs-on: ubuntu-latest + strategy: + matrix: + name: + - none_default + - single_default + - input_default + - none_custom_registry + - single_custom_registry + - input_custom_registry + - none_custom_kayobe_argument + - single_custom_kayobe_argument + - input_custom_kayobe_argument + - reference + steps: - name: Check out the codebase. uses: actions/checkout@v3 @@ -19,13 +33,13 @@ jobs: - name: Install Ansible run: pip3 install ansible - - name: Test the playbook. - run: ansible-playbook tests/test.yml + - name: Test the playbook ${{ matrix.name }} + run: "ansible-playbook -i tests/inventory/hosts.yml tests/test.yml --limit ${{ matrix.name }}" env: ANSIBLE_FORCE_COLOR: '1' - - name: Upload workflows produced + - name: Upload workflows produced ${{ matrix.name }} uses: actions/upload-artifact@v3 with: - name: github_kayobe_workflows - path: tests/.github/workflows + name: "${{ format('github_kayobe_workflows_{0}', matrix.name) }}" + path: "${{ format('tests/.github/{0}', matrix.name) }}" \ No newline at end of file diff --git a/roles/github/README.md b/roles/github/README.md index cc81f79..90d9292 100644 --- a/roles/github/README.md +++ b/roles/github/README.md @@ -36,23 +36,25 @@ The following variables can be used to make small adjustments to the composition `github_output_directory`: control the location where the workflows shall be written to. +`github_environment_selector`: control the type of environment support the workflows should be generated with. Either `single` for fixed environment or `input` whereby the environment is controlled at `workflow_dispatch`. No environment is the default by setting `github_environment_selector` to no value or `Null`. + +`github_kayobe_environments`: list of environments the workflows should target. Only has effect when `github_environment_selector` is `input` or `single`. + `github_runs_on`: control which runner can accept this workflow. See GitHub for more information on [runs-on](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on). `github_image_name`: name of the kayobe image defaults to `kayobe`. `github_image_tag`: tag used to select kayobe image defaults to `latest` -`github_registry_username`: username used to authenticate with the docker registry. - -`github_registry_password`: password used to authenticate with the docker registry. +`github_registry`: dictionary containing keys that correspond to `url`, `username`, `password` and `share` for the registry to be used by the workflows. Defaults to `ghcr.io` and uses the actors token to login. The key `share` is to indiciate if the registry is shared between environments. `github_kayobe_base_image`: select the base image used when building the kayobe docker image. Default is `quay.io/centos/centos:stream8` supports OpenStack Wallaby, Xena and Yoga. Zed and higher would require `quay.io/rockylinux/rockylinux:9`. -`github_kayobe_arguments`: a dictionary of arguments that can be used to override the default arguments found within `vars/main.yml`. For example if you wanted to change the value of `KAYOBE_ENVIRONMENT` from its default of `production` you can simply add `KAYOBE_ENVIRONMENT` to this dictionary and it will take precedence over the defaults. +`github_kayobe_arguments`: a dictionary of arguments that can be used to override the default arguments found within `vars/main.yml`. For example if you wanted to change the value of `KAYOBE_AUTOMATION_PR_TITLE` from its default, you can do by simply adding `KAYOBE_AUTOMATION_PR_TITLE` to this dictionary and it will take precedence over the default. `github_*_hook:` see section [Template Hooks](#template-hooks) for information about this variables -`github_buildx_enable`: In some deployments the build kayobe docker image workflow has had difficulties successfully pushing the image to container registries such as Pulp if buildx has been used. It situations where failure to push images is been experienced a user might wish to disable buildx. Buildx is enabled by default. +`github_buildx_enable`: In some deployments the build kayobe docker image workflow has had difficulties successfully pushing the image to container registries such as Pulp if buildx has been used. It situations where failure to push images is been experienced a user might wish to disable buildx. Buildx is disabled by default. `github_buildx_inline_config`: provide configuration parameters to buildx. Useful for connecting to insecure docker registry. @@ -141,7 +143,7 @@ The following example playbook will generate a series of `reference` workflows w - name: Write Kayobe Automation Workflows for GitHub hosts: localhost roles: - - stackhpc.kayobe_automation_workflows.github + - stackhpc.kayobe_workflows.github ``` License diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 660be0c..bfd185f 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -1,17 +1,17 @@ --- github_output_directory: .github/workflows -github_runs_on: self-hosted +github_environment_selector: -github_registry_url: ghcr.io +github_kayobe_environments: [] -github_registry_username: !unsafe "${{ github.actor }}" +github_runs_on: [self-hosted] -github_registry_password: !unsafe "${{ secrets.GITHUB_TOKEN }}" +github_registry: {} github_image_name: kayobe -github_image_tag: latest +github_image_tag: !unsafe "${{ needs.prepare-runner.outputs.openstack_release }}-latest" github_kayobe_base_image: "quay.io/centos/centos:stream8" @@ -23,7 +23,7 @@ github_kayobe_hook: "" github_final_hook: "" -github_buildx_enabled: true +github_buildx_enabled: false github_buildx_inline_config: "" @@ -36,26 +36,36 @@ github_tempest_test_suites: | - tempest-full github_kayobe_limit_input: | - kayobeLimit: + kayobe_limit: description: | The ansible limit to use when running kayobe playbooks. github_kayobe_tags_input: | - kayobeTags: + kayobe_tags: description: | The ansible tags to use when running kayobe playbooks. github_kolla_limit_input: | - kollaLimit: + kolla_limit: description: | The ansible limit to use for kolla-ansible playbooks. github_kolla_tags_input: | - kollaTags: + kolla_tags: description: | The ansible tags to use when running kolla-ansible playbooks. +github_kayobe_environment_input: | + kayobe_environment: + description: | + Select the environment the kayobe workflow shall target. + type: choice + required: true + default: '{{ github_kayobe_environments | first }}' + options: {{ github_kayobe_environments }} + github_workflows: + - "{{ github_prepare_runner }}" - "{{ github_build_kayobe_image }}" - "{{ github_run_kolla_config_diff }}" - "{{ github_run_infra_vm_host_configure }}" @@ -79,6 +89,10 @@ github_workflows: - "{{ github_run_seed_vm_provision }}" - "{{ github_run_tempest }}" +github_prepare_runner: + file_name: prepare-runner.yml + use_bespoke: true + github_build_kayobe_image: file_name: build-kayobe-docker-image.yml use_bespoke: true @@ -225,3 +239,4 @@ github_run_tempest: file_name: run-tempest.yml use_bespoke: true test_suites: "{{ github_tempest_test_suites }}" + concurrency_group: tempest diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index 9df472b..75fb06a 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -1,3 +1,9 @@ +<%- if github_environment_selector == 'input' -%> +<%- set github_runs_on = github_runs_on + ['${{ matrix.environment }}'] -%> +<%- set _ = github_default_registry.update({"url": "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" }) -%> +<%- set _ = github_default_registry.update({"username": "${{ vars[format('{0}_REGISTRY_USERNAME', matrix.environment)] }}" }) -%> +<%- set _ = github_default_registry.update({"password": "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" }) -%> +<%- endif -%> name: %% format_file_name(workflow.file_name, is_title=true) %% on: @@ -8,13 +14,21 @@ env: KAYOBE_USER_GID: 1000 jobs: + prepare-runner: + uses: ./.github/workflows/prepare-runner.yml %% format_file_name(workflow.file_name) %%: + <%- if github_environment_selector == 'input' and (github_registry.share | default(github_default_registry.share)) is false +%> + strategy: + matrix: + environment: %% github_kayobe_environments %% + <%- endif +%> runs-on: %% github_runs_on %% container: image: docker:24.0-git permissions: contents: read - packages: %% 'write' if github_registry_url == 'ghcr.io' else 'none' %% + packages: %% 'write' if (github_registry.url | default(github_default_registry.url)) == 'ghcr.io' else 'none' %% + needs: prepare-runner steps: <% if github_checkout_hook | length >= 1 %> %% github_checkout_hook | indent(width=6, first=false) %% @@ -28,9 +42,9 @@ jobs: - name: Log in to the Container registry uses: docker/login-action@v2 with: - registry: %% github_registry_url %% - username: %% github_registry_username %% - password: %% github_registry_password %% + registry: %% github_registry.url | default(github_default_registry.url) %% + username: %% github_registry.username | default(github_default_registry.username) %% + password: %% github_registry.password | default(github_default_registry.password) %% <% if github_buildx_enabled %> - name: Set up Docker Buildx @@ -60,8 +74,8 @@ jobs: BASE_IMAGE=%% github_kayobe_base_image %% push: true tags: | - %% github_registry_url %%/%% github_image_name %%:latest - %% github_registry_url %%/%% github_image_name %%:${{ github.sha }} + %% github_registry.url | default(github_default_registry.url) %%/%% github_image_name %%:%% github_image_tag %% + %% github_registry.url | default(github_default_registry.url) %%/%% github_image_name %%:${{ github.sha }} <% if not github_buildx_enable_provenance %> provenance: false <% endif %> diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index af2391a..6fbe206 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -1,21 +1,36 @@ -<% include "header.yml.j2" +%> - +<%- if github_environment_selector == 'input' -%> +<%- set github_runs_on = github_runs_on + ['${{ inputs.kayobe_environment }}'] -%> +<%- set _ = workflow.update({"concurrency_group": "format('{0}-{1}', " + workflow.concurrency_group + ", '${{ inputs.kayobe_environment }}')" }) -%> +<%- set _ = github_default_registry.update({"url": "${{ vars[format('{0}_REGISTRY_URL', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_registry.update({"username": "${{ vars[format('{0}_REGISTRY_USERNAME', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_registry.update({"password": "${{ secrets[format('{0}_REGISTRY_PASSWORD', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> +<%- endif -%> +<%- if github_environment_selector == 'single' -%> +<%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": github_kayobe_environments | first}) -%> +<%- endif -%> +%% lookup('template', 'header.yml.j2') %% jobs: + prepare-runner: + uses: ./.github/workflows/prepare-runner.yml %% format_file_name(workflow.file_name) %%: runs-on: %% github_runs_on %% permissions: contents: %% 'write' if 'KAYOBE_AUTOMATION_PR_TYPE' in workflow.arguments | flatten else 'read' %% - packages: %% 'read' if github_registry_url == 'ghcr.io' else 'none' %% + packages: %% 'read' if (github_registry.url | default(github_default_registry.url)) == 'ghcr.io' else 'none' %% pull-requests: %% 'write' if 'KAYOBE_AUTOMATION_PR_TYPE' in workflow.arguments | flatten else 'none' %% container: - image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% + image: %% github_registry.url | default(github_default_registry.url) %%/%% github_image_name %%:%% github_image_tag %% credentials: - username: %% github_registry_username %% - password: %% github_registry_password %% + username: %% github_registry.username | default(github_default_registry.username) %% + password: %% github_registry.password | default(github_default_registry.password) %% concurrency: group: %% workflow.concurrency_group %% cancel-in-progress: false timeout-minutes: %% github_timeout %% + needs: prepare-runner steps: <% if github_checkout_hook | length >= 1 %> %% github_checkout_hook | indent(width=6, first=false) %% @@ -37,9 +52,11 @@ jobs: /src/.automation/pipeline/%% workflow.file_name[4:-4] %%.sh <%- if workflow.arguments is defined +%> env: - KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' - KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' +<% if github_environment_selector is not none %> + KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT %%' +<% endif %> + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: "%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%" + KAYOBE_VAULT_PASSWORD: "%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%" <%- for argument in workflow.arguments | flatten +%> %% argument %%: '%% github_kayobe_arguments[argument] | default(github_default_kayobe_arguments[argument]) %%' <%- endfor +%> diff --git a/roles/github/templates/header.yml.j2 b/roles/github/templates/header.yml.j2 index 966135d..ca601f6 100644 --- a/roles/github/templates/header.yml.j2 +++ b/roles/github/templates/header.yml.j2 @@ -1,17 +1,30 @@ -name: %% format_file_name(workflow.file_name, is_title=true) %% +{%- macro format_file_name(file_name, is_title=false, is_subtitle=false) -%} + {%- set formatted_name = file_name | splitext | first -%} + {%- if is_title -%} + {%- set formatted_name = formatted_name | replace('-', ' ') | title | regex_replace('Vm','VM') -%} + {%- endif -%} + {%- if is_subtitle -%} + {%- set formatted_name = formatted_name | replace('-', ' ') | capitalize | regex_replace('vm','VM') -%} + {%- endif -%} + {{ formatted_name }} +{%- endmacro -%} +name: {{ format_file_name(workflow.file_name, is_title=true) }} on: -<%- if workflow.trigger is defined +%> - <%- for trigger_name in workflow.trigger.keys() +%> - <%- if trigger_name == 'schedule' +%> +{%- if workflow.trigger is defined +%} + {%- for trigger_name in workflow.trigger.keys() +%} + {%- if trigger_name == 'schedule' +%} schedule: - - cron: '%% workflow.trigger['schedule']['cron'] %%' - <%- elif trigger_name == 'workflow_dispatch' +%> + - cron: '{{ workflow.trigger['schedule']['cron'] }}' + {%- elif trigger_name == 'workflow_dispatch' +%} workflow_dispatch: - <%- if workflow.trigger['workflow_dispatch'] is not none +%> + {%- if workflow.trigger['workflow_dispatch'] is not none +%} inputs: - %% workflow.trigger['workflow_dispatch'] | flatten | join('') | indent(6) | trim %% - <%- endif +%> - <%- endif +%> - <%- endfor +%> -<%- endif +%> + {{ workflow.trigger['workflow_dispatch'] | flatten | join('') | indent(6) | trim }} + {%- if github_environment_selector == 'input' +%} + {{ github_kayobe_environment_input | flatten | join('') | indent(6) | trim }} + {%- endif +%} + {%- endif +%} + {%- endif +%} + {%- endfor +%} +{%- endif -%} diff --git a/roles/github/templates/prepare-runner.yml.j2 b/roles/github/templates/prepare-runner.yml.j2 new file mode 100644 index 0000000..a87463d --- /dev/null +++ b/roles/github/templates/prepare-runner.yml.j2 @@ -0,0 +1,28 @@ +name: %% format_file_name(workflow.file_name, is_title=true) %% + +on: + workflow_call: + outputs: + openstack_release: + description: "The version of OpenStack/Kayobe to be used by the runner." + value: ${{ jobs.prepare-runner.outputs.openstack_release }} + +jobs: + prepare-runner: + runs-on: %% github_runs_on %% + container: + image: alpine:latest + permissions: + contents: read + packages: read + outputs: + openstack_release: ${{ steps.openstack_release.outputs.openstack_release }} + steps: + - name: Checkout kayobe config + uses: actions/checkout@v3 + + - name: Extract OpenStack Release + id: openstack_release + run: | + BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) + echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT diff --git a/roles/github/templates/run-config-diff.yml.j2 b/roles/github/templates/run-config-diff.yml.j2 index fefe9c4..e6f3096 100644 --- a/roles/github/templates/run-config-diff.yml.j2 +++ b/roles/github/templates/run-config-diff.yml.j2 @@ -1,3 +1,13 @@ +<%- if github_environment_selector == 'input' -%> +<%- set _ = github_default_registry.update({"url": "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" }) -%> +<%- set _ = github_default_registry.update({"username": "${{ vars[format('{0}_REGISTRY_USERNAME', matrix.environment)] }}" }) -%> +<%- set _ = github_default_registry.update({"password": "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" }) -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', matrix.environment)] }}" }) -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', matrix.environment)] }}" }) -%> +<%- endif -%> +<%- if github_environment_selector is not none -%> +<%- set github_runs_on = github_runs_on + ['${{ matrix.environment }}'] -%> +<%- endif -%> name: %% format_file_name(workflow.file_name, is_title=true) %% concurrency: @@ -8,16 +18,24 @@ on: pull_request: jobs: + prepare-runner: + uses: ./.github/workflows/prepare-runner.yml %% format_file_name(workflow.file_name) %%: + <%- if github_environment_selector is not none +%> + strategy: + matrix: + environment: %% github_kayobe_environments %% + <%- endif +%> runs-on: %% github_runs_on %% permissions: contents: read - packages: %% 'read' if github_registry_url == 'ghcr.io' else 'none' %% + packages: %% 'read' if (github_registry.url | default(github_default_registry.url)) == 'ghcr.io' else 'none' %% container: - image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% + image: %% github_registry.url | default(github_default_registry.url) %%/%% github_image_name %%:%% github_image_tag %% credentials: - username: %% github_registry_username %% - password: %% github_registry_password %% + username: %% github_registry.username | default(github_default_registry.username) %% + password: %% github_registry.password | default(github_default_registry.password) %% + needs: prepare-runner steps: <% if github_checkout_hook | length >= 1 %> %% github_checkout_hook | indent(width=6, first=false) %% @@ -42,9 +60,10 @@ jobs: run: | sudo -E -u stack bash -c '/src/.automation/pipeline/config-diff.sh ${{ github.event.pull_request.base.sha }}' env: - KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' - KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' +<% if github_environment_selector is not none %> + KAYOBE_ENVIRONMENT: '${{ matrix.environment }}' +<% endif %> + KAYOBE_VAULT_PASSWORD: "%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%" HOME: '%% github_kayobe_arguments.HOME | default(github_default_kayobe_arguments.HOME) %%' - name: Show summary of changes diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index e5bbb1d..dc07a68 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -1,37 +1,36 @@ -name: %% format_file_name(workflow.file_name, is_title=true) %% - -on: - workflow_dispatch: - inputs: - testSuite: - description: | - The list of tests to run. This should be a name of file under - .automation.conf/tempest/load-lists. Defaults to running - default test suite. - required: false - default: 'default' - type: choice - options: - %% workflow.test_suites | flatten | join('') | indent(10) | trim %% - tempestPattern: - description: | - Limit tests to this regex. Takes precedence over testSuite. - +<%- if github_environment_selector == 'input' -%> +<%- set github_runs_on = github_runs_on + ['${{ inputs.kayobe_environment }}'] -%> +<%- set _ = workflow.update({"concurrency_group": "format('{0}-{1}', " + workflow.concurrency_group + ", '${{ inputs.kayobe_environment }}')" }) -%> +<%- set _ = github_default_registry.update({"url": "${{ vars[format('{0}_REGISTRY_URL', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_registry.update({"username": "${{ vars[format('{0}_REGISTRY_USERNAME', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_registry.update({"password": "${{ secrets[format('{0}_REGISTRY_PASSWORD', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_kayobe_arguments.update({"TEMPEST_OPENRC": "${{ secrets[format('{0}_TEMPEST_OPENRC', inputs.kayobe_environment)] }}" }) -%> +<%- endif -%> +<%- if github_environment_selector == 'single' -%> +<%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": github_kayobe_environments[0] }) -%> +<%- endif -%> +%% lookup('template', 'tempest-header.yml.j2') %% jobs: + prepare-runner: + uses: ./.github/workflows/prepare-runner.yml %% format_file_name(workflow.file_name) %%: runs-on: %% github_runs_on %% permissions: contents: read - packages: %% 'read' if github_registry_url == 'ghcr.io' else 'none' %% + packages: %% 'read' if (github_registry.url | default(github_default_registry.url)) == 'ghcr.io' else 'none' %% container: - image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% + image: %% github_registry.url | default(github_default_registry.url) %%/%% github_image_name %%:%% github_image_tag %% credentials: - username: %% github_registry_username %% - password: %% github_registry_password %% + username: %% github_registry.username | default(github_default_registry.username) %% + password: %% github_registry.password | default(github_default_registry.password) %% concurrency: - group: tempest + group: %% workflow.concurrency_group %% cancel-in-progress: false timeout-minutes: %% github_timeout %% + needs: prepare-runner steps: <% if github_checkout_hook | length >= 1 %> %% github_checkout_hook | indent(width=6, first=false) %% @@ -48,34 +47,18 @@ jobs: <% if github_kayobe_hook | length >= 1 %> %% github_kayobe_hook | indent(width=6, first=false) %% <% endif %> - - name: Run kayobe control host bootstrap - run: | - source /src/.automation/functions && - kayobe_install - env: - KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' - KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' - HOME: '%% github_kayobe_arguments.HOME | default(github_default_kayobe_arguments.HOME) %%' - USER: stack - - - name: Symlink kolla into /src/etc - run: | - mkdir -p /src/etc && - ln -s /stack/kayobe-automation-env/src/kayobe-config/etc/kolla /src/etc - - - name: Run ${{ github.event.inputs.testSuite }} tempest testsuite + - name: Run ${{ github.event.inputs.test_suite }} tempest testsuite run: | /src/.automation/pipeline/tempest.sh -e ansible_user=stack env: - KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' - KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' +<% if github_environment_selector is not none %> + KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT %%' +<% endif %> + KAYOBE_VAULT_PASSWORD: "%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%" + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: "%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%" KAYOBE_AUTOMATION_TEMPEST_LOADLIST: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_TEMPEST_LOADLIST | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_TEMPEST_LOADLIST) %%' TEMPEST_PATTERN: '%% github_kayobe_arguments.TEMPEST_PATTERN | default(github_default_kayobe_arguments.TEMPEST_PATTERN) %%' - KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY: '%% github_registry_url %%' - KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_USERNAME: '%% github_registry_username %%' - KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_PASSWORD: '%% github_registry_password %%' + TEMPEST_OPENRC: "%% github_kayobe_arguments.TEMPEST_OPENRC | default(github_default_kayobe_arguments.TEMPEST_OPENRC) %%" HOME: '%% github_kayobe_arguments.HOME | default(github_default_kayobe_arguments.HOME) %%' - name: Print stdout diff --git a/roles/github/templates/tempest-header.yml.j2 b/roles/github/templates/tempest-header.yml.j2 new file mode 100644 index 0000000..16d9a87 --- /dev/null +++ b/roles/github/templates/tempest-header.yml.j2 @@ -0,0 +1,31 @@ +{%- macro format_file_name(file_name, is_title=false, is_subtitle=false) -%} + {%- set formatted_name = file_name | splitext | first -%} + {%- if is_title -%} + {%- set formatted_name = formatted_name | replace('-', ' ') | title | regex_replace('Vm','VM') -%} + {%- endif -%} + {%- if is_subtitle -%} + {%- set formatted_name = formatted_name | replace('-', ' ') | capitalize | regex_replace('vm','VM') -%} + {%- endif -%} + {{ formatted_name }} +{%- endmacro -%} +name: {{ format_file_name(workflow.file_name, is_title=true) }} + +on: + workflow_dispatch: + inputs: + test_suite: + description: | + The list of tests to run. This should be a name of file under + .automation.conf/tempest/load-lists. Defaults to running + default test suite. + required: false + default: 'default' + type: choice + options: + {{ workflow.test_suites | flatten | join('') | indent(10) | trim }} + tempest_pattern: + description: | + Limit tests to this regex. Takes precedence over testSuite. + {%- if github_environment_selector == 'input' +%} + {{ github_kayobe_environment_input | flatten | join('') | indent(6) | trim }} + {%- endif +%} diff --git a/roles/github/vars/main.yml b/roles/github/vars/main.yml index d9f54b4..f4be3ec 100644 --- a/roles/github/vars/main.yml +++ b/roles/github/vars/main.yml @@ -1,12 +1,17 @@ --- +github_default_registry: + url: ghcr.io + username: !unsafe "${{ github.actor }}" + password: !unsafe "${{ secrets.GITHUB_TOKEN }}" + share: false + github_default_kayobe_arguments: - KAYOBE_ENVIRONMENT: production KAYOBE_VAULT_PASSWORD: !unsafe "${{ secrets.KAYOBE_VAULT_PASSWORD }}" KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: !unsafe "${{ secrets.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY }}" - KOLLA_LIMIT: !unsafe "${{ github.event.inputs.kollaLimit }}" - KOLLA_TAGS: !unsafe "${{ github.event.inputs.kollaTags }}" - KAYOBE_TAGS: !unsafe "${{ github.event.inputs.kayobeTags }}" - KAYOBE_LIMIT: !unsafe "${{ github.event.inputs.kayobeLimit }}" + KOLLA_LIMIT: !unsafe "${{ github.event.inputs.kolla_limit }}" + KOLLA_TAGS: !unsafe "${{ github.event.inputs.kolla_tags }}" + KAYOBE_TAGS: !unsafe "${{ github.event.inputs.kayobe_tags }}" + KAYOBE_LIMIT: !unsafe "${{ github.event.inputs.kayobe_limit }}" KAYOBE_AUTOMATION_PR_TARGET_BRANCH: !unsafe ${{ github.event.ref }} KAYOBE_AUTOMATION_PR_REMOTE: !unsafe https://${KAYOBE_AUTOMATION_PR_GITHUB_USER}:${KAYOBE_AUTOMATION_PR_AUTH_TOKEN}@github.com/${{ github.repository }} KAYOBE_AUTOMATION_PR_GITHUB_USER: !unsafe ${{ github.actor }} @@ -14,8 +19,9 @@ github_default_kayobe_arguments: KAYOBE_AUTOMATION_PR_TYPE: github KAYOBE_AUTOMATION_PR_TITLE: !unsafe "[kayobe-automation] ${{ github.workflow }} #${{ github.run_id }}" KAYOBE_AUTOMATION_PR_URL: !unsafe https://api.github.com/repos/${{ github.repository }}/pulls - KAYOBE_AUTOMATION_TEMPEST_LOADLIST: !unsafe "${{ github.event.inputs.testSuite }}" - TEMPEST_PATTERN: !unsafe "${{ github.event.inputs.tempestPattern }}" + KAYOBE_AUTOMATION_TEMPEST_LOADLIST: !unsafe "${{ github.event.inputs.test_suite }}" + TEMPEST_PATTERN: !unsafe "${{ github.event.inputs.tempest_pattern }}" + TEMPEST_OPENRC: !unsafe "${{ secrets.TEMPEST_OPENRC }}" HOME: /stack github_kayobe_dispatch_inputs: diff --git a/tests/inventory/host_vars/input_custom_kayobe_argument.yml b/tests/inventory/host_vars/input_custom_kayobe_argument.yml new file mode 100644 index 0000000..55bd119 --- /dev/null +++ b/tests/inventory/host_vars/input_custom_kayobe_argument.yml @@ -0,0 +1,9 @@ +--- +github_output_directory: .github/input_custom_kayobe_argument + +github_environment_selector: input + +github_kayobe_environments: [prod-gb-01, test-gb-01] + +github_kayobe_arguments: + KAYOBE_VAULT_PASSWORD: !unsafe "${{ secrets.SUPER_SECRET_KAYOBE_VAULT_PASSWORD }}" diff --git a/tests/inventory/host_vars/input_custom_registry.yml b/tests/inventory/host_vars/input_custom_registry.yml new file mode 100644 index 0000000..71c0222 --- /dev/null +++ b/tests/inventory/host_vars/input_custom_registry.yml @@ -0,0 +1,9 @@ +--- +github_output_directory: .github/input_custom_registry + +github_environment_selector: input + +github_kayobe_environments: [prod-gb-01, test-gb-01] + +github_registry: + url: "pulp.example.com" diff --git a/tests/inventory/host_vars/input_default.yml b/tests/inventory/host_vars/input_default.yml new file mode 100644 index 0000000..4dacf23 --- /dev/null +++ b/tests/inventory/host_vars/input_default.yml @@ -0,0 +1,6 @@ +--- +github_output_directory: .github/input_default + +github_environment_selector: input + +github_kayobe_environments: [prod-gb-01, test-gb-01] diff --git a/tests/inventory/host_vars/none_custom_kayobe_argument.yml b/tests/inventory/host_vars/none_custom_kayobe_argument.yml new file mode 100644 index 0000000..8a877c2 --- /dev/null +++ b/tests/inventory/host_vars/none_custom_kayobe_argument.yml @@ -0,0 +1,5 @@ +--- +github_output_directory: .github/none_custom_kayobe_argument + +github_kayobe_arguments: + KAYOBE_VAULT_PASSWORD: !unsafe "${{ secrets.SUPER_SECRET_KAYOBE_VAULT_PASSWORD }}" diff --git a/tests/inventory/host_vars/none_custom_registry.yml b/tests/inventory/host_vars/none_custom_registry.yml new file mode 100644 index 0000000..fbbcd40 --- /dev/null +++ b/tests/inventory/host_vars/none_custom_registry.yml @@ -0,0 +1,5 @@ +--- +github_output_directory: .github/none_custom_registry + +github_registry: + url: "pulp.example.com" diff --git a/tests/inventory/host_vars/none_default.yml b/tests/inventory/host_vars/none_default.yml new file mode 100644 index 0000000..697a314 --- /dev/null +++ b/tests/inventory/host_vars/none_default.yml @@ -0,0 +1,2 @@ +--- +github_output_directory: .github/none_default diff --git a/tests/inventory/host_vars/reference.yml b/tests/inventory/host_vars/reference.yml new file mode 100644 index 0000000..0733538 --- /dev/null +++ b/tests/inventory/host_vars/reference.yml @@ -0,0 +1,22 @@ +--- +github_output_directory: ".github/reference" + +github_environment_selector: input + +github_kayobe_environments: + - production-gb-01 + - testing-gb-01 + - production-de-01 + - testing-de-01 + +github_runs_on: + - kayobe + - openstack + +github_registry: + url: pulp.infra.os.example.cloud + username: admin + password: ${{ secrets.REGISTRY_PASSWORD }} + +github_kayobe_arguments: + KAYOBE_VAULT_PASSWORD: !unsafe "${{ secrets.KAYOBE_VAULT_PASSWORD }}" diff --git a/tests/inventory/host_vars/single_custom_kayobe_argument.yml b/tests/inventory/host_vars/single_custom_kayobe_argument.yml new file mode 100644 index 0000000..85fe3d0 --- /dev/null +++ b/tests/inventory/host_vars/single_custom_kayobe_argument.yml @@ -0,0 +1,9 @@ +--- +github_output_directory: .github/single_custom_kayobe_argument + +github_environment_selector: single + +github_kayobe_environments: [prod-gb-01] + +github_kayobe_arguments: + KAYOBE_VAULT_PASSWORD: !unsafe "${{ secrets.SUPER_SECRET_KAYOBE_VAULT_PASSWORD }}" diff --git a/tests/inventory/host_vars/single_custom_registry.yml b/tests/inventory/host_vars/single_custom_registry.yml new file mode 100644 index 0000000..eec4514 --- /dev/null +++ b/tests/inventory/host_vars/single_custom_registry.yml @@ -0,0 +1,9 @@ +--- +github_output_directory: .github/single_custom_registry + +github_environment_selector: single + +github_kayobe_environments: [prod-gb-01] + +github_registry: + url: "pulp.example.com" diff --git a/tests/inventory/host_vars/single_default.yml b/tests/inventory/host_vars/single_default.yml new file mode 100644 index 0000000..d99c840 --- /dev/null +++ b/tests/inventory/host_vars/single_default.yml @@ -0,0 +1,6 @@ +--- +github_output_directory: .github/single_default + +github_environment_selector: single + +github_kayobe_environments: [prod-gb-01] diff --git a/tests/inventory/hosts.yml b/tests/inventory/hosts.yml new file mode 100644 index 0000000..930c925 --- /dev/null +++ b/tests/inventory/hosts.yml @@ -0,0 +1,23 @@ +--- +tests_cases: + hosts: + none_default: + ansible_connection: local + single_default: + ansible_connection: local + input_default: + ansible_connection: local + none_custom_registry: + ansible_connection: local + single_custom_registry: + ansible_connection: local + input_custom_registry: + ansible_connection: local + none_custom_kayobe_argument: + ansible_connection: local + single_custom_kayobe_argument: + ansible_connection: local + input_custom_kayobe_argument: + ansible_connection: local + reference: + ansible_connection: local diff --git a/tests/test.yml b/tests/test.yml index 59fe65b..17926f5 100644 --- a/tests/test.yml +++ b/tests/test.yml @@ -1,5 +1,5 @@ --- - name: Test `stackhpc.kayobe_automation` - hosts: localhost + hosts: all roles: - github