From bb0fea6f07d49983eac4ea68d178e8aff3ca4500 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 28 Sep 2023 11:39:36 +0100 Subject: [PATCH 01/55] feat: add environment variables --- roles/github/README.md | 4 ++++ roles/github/defaults/main.yml | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/roles/github/README.md b/roles/github/README.md index cc81f79..2afc370 100644 --- a/roles/github/README.md +++ b/roles/github/README.md @@ -36,6 +36,10 @@ The following variables can be used to make small adjustments to the composition `github_output_directory`: control the location where the workflows shall be written to. +`github_environment_type`: control the type of environment support the workflows should be generated with. Either `none` for single environment deployment or `kayobe` for a kayobe based environment workflows. + +`github_kayobe_environments`: list of environments the workflows should target. Only has effect when `github_environment_type` is not `none`. + `github_runs_on`: control which runner can accept this workflow. See GitHub for more information on [runs-on](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on). `github_image_name`: name of the kayobe image defaults to `kayobe`. diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 660be0c..1e5f506 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -1,6 +1,10 @@ --- github_output_directory: .github/workflows +github_environment_type: none + +github_kayobe_environments: none + github_runs_on: self-hosted github_registry_url: ghcr.io @@ -55,6 +59,15 @@ github_kolla_tags_input: | description: | The ansible tags to use when running kolla-ansible playbooks. +github_kayobe_environment_input: | + kayobeEnvironment: + description: | + Select the environment the kayobe workflow shall target. + type: choice + required: true + default: 'production' + options: + github_workflows: - "{{ github_build_kayobe_image }}" - "{{ github_run_kolla_config_diff }}" From 21686bbc11e8986c6f6091e17134ce9ee18696e9 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 28 Sep 2023 11:41:01 +0100 Subject: [PATCH 02/55] feat: add `concurrency_group` to `tempest` --- roles/github/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 1e5f506..6b05cfb 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -238,3 +238,4 @@ github_run_tempest: file_name: run-tempest.yml use_bespoke: true test_suites: "{{ github_tempest_test_suites }}" + concurrency_group: tempest From 4759aafe62142d9dbe3230d50bbd9b5c39b083d0 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 28 Sep 2023 11:43:50 +0100 Subject: [PATCH 03/55] fix: use `snake_case` for `inputs` --- roles/github/defaults/main.yml | 10 +++++----- roles/github/vars/main.yml | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 6b05cfb..28809a6 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -40,27 +40,27 @@ github_tempest_test_suites: | - tempest-full github_kayobe_limit_input: | - kayobeLimit: + kayobe_limit: description: | The ansible limit to use when running kayobe playbooks. github_kayobe_tags_input: | - kayobeTags: + kayobe_tags: description: | The ansible tags to use when running kayobe playbooks. github_kolla_limit_input: | - kollaLimit: + kolla_limit: description: | The ansible limit to use for kolla-ansible playbooks. github_kolla_tags_input: | - kollaTags: + kolla_tags: description: | The ansible tags to use when running kolla-ansible playbooks. github_kayobe_environment_input: | - kayobeEnvironment: + kayobe_environment: description: | Select the environment the kayobe workflow shall target. type: choice diff --git a/roles/github/vars/main.yml b/roles/github/vars/main.yml index d9f54b4..abb19e1 100644 --- a/roles/github/vars/main.yml +++ b/roles/github/vars/main.yml @@ -3,10 +3,10 @@ github_default_kayobe_arguments: KAYOBE_ENVIRONMENT: production KAYOBE_VAULT_PASSWORD: !unsafe "${{ secrets.KAYOBE_VAULT_PASSWORD }}" KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: !unsafe "${{ secrets.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY }}" - KOLLA_LIMIT: !unsafe "${{ github.event.inputs.kollaLimit }}" - KOLLA_TAGS: !unsafe "${{ github.event.inputs.kollaTags }}" - KAYOBE_TAGS: !unsafe "${{ github.event.inputs.kayobeTags }}" - KAYOBE_LIMIT: !unsafe "${{ github.event.inputs.kayobeLimit }}" + KOLLA_LIMIT: !unsafe "${{ github.event.inputs.kolla_limit }}" + KOLLA_TAGS: !unsafe "${{ github.event.inputs.kolla_tags }}" + KAYOBE_TAGS: !unsafe "${{ github.event.inputs.kayobe_tags }}" + KAYOBE_LIMIT: !unsafe "${{ github.event.inputs.kayob_limit }}" KAYOBE_AUTOMATION_PR_TARGET_BRANCH: !unsafe ${{ github.event.ref }} KAYOBE_AUTOMATION_PR_REMOTE: !unsafe https://${KAYOBE_AUTOMATION_PR_GITHUB_USER}:${KAYOBE_AUTOMATION_PR_AUTH_TOKEN}@github.com/${{ github.repository }} KAYOBE_AUTOMATION_PR_GITHUB_USER: !unsafe ${{ github.actor }} From 99dfc11c21702fb78b1a4fb7d3e6bdcd1a81382a Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 28 Sep 2023 11:54:51 +0100 Subject: [PATCH 04/55] feat!: add multiple environment support to tempest WARNING: this also carries a change to remove the bootstrap prior to tempest running. This has been done to reduce the time taken to start and finish a tempest run. This will require a copy of `public_openrc.sh` to GitHub secrets. --- roles/github/templates/run-tempest.yml.j2 | 40 +++++++++++------------ roles/github/vars/main.yml | 1 + 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index e5bbb1d..baf6fa3 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -1,3 +1,13 @@ +<%- if github_environment_type == 'kayobe' -%> +<%- set github_runs_on = github_runs_on + ['${{ inputs.kayobe_environment }}'] -%> +<%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', inputs.kayobe_environment)] }}" -%> +<%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', inputs.kayobe_environment)] }}" -%> +<%- set _ = workflow.update({"concurrency_group": "format('{0}-{1}', " + workflow.concurrency_group + ", '${{ inputs.kayobe_environment }}')" }) -%> +<%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> +<%- set _ = github_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_kayobe_arguments.update({"TEMPEST_OPENRC": "${{ secrets[format('{0}_TEMPEST_OPENRC', inputs.kayobe_environment)] }}" }) -%> +<%- endif +%> name: %% format_file_name(workflow.file_name, is_title=true) %% on: @@ -16,6 +26,9 @@ on: tempestPattern: description: | Limit tests to this regex. Takes precedence over testSuite. + <%- if github_environment_type == 'kayobe' +%> + %% github_kayobe_environment_input | flatten | join('') | indent(6) | trim %% %% github_kayobe_environments %% + <%- endif +%> jobs: %% format_file_name(workflow.file_name) %%: @@ -29,7 +42,7 @@ jobs: username: %% github_registry_username %% password: %% github_registry_password %% concurrency: - group: tempest + group: %% workflow.concurrency_group %% cancel-in-progress: false timeout-minutes: %% github_timeout %% steps: @@ -48,34 +61,19 @@ jobs: <% if github_kayobe_hook | length >= 1 %> %% github_kayobe_hook | indent(width=6, first=false) %% <% endif %> - - name: Run kayobe control host bootstrap - run: | - source /src/.automation/functions && - kayobe_install - env: - KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' - KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' - HOME: '%% github_kayobe_arguments.HOME | default(github_default_kayobe_arguments.HOME) %%' - USER: stack - - - name: Symlink kolla into /src/etc - run: | - mkdir -p /src/etc && - ln -s /stack/kayobe-automation-env/src/kayobe-config/etc/kolla /src/etc - - name: Run ${{ github.event.inputs.testSuite }} tempest testsuite run: | /src/.automation/pipeline/tempest.sh -e ansible_user=stack env: KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' - KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' + KAYOBE_VAULT_PASSWORD: "%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%" + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: "%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%" KAYOBE_AUTOMATION_TEMPEST_LOADLIST: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_TEMPEST_LOADLIST | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_TEMPEST_LOADLIST) %%' TEMPEST_PATTERN: '%% github_kayobe_arguments.TEMPEST_PATTERN | default(github_default_kayobe_arguments.TEMPEST_PATTERN) %%' - KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY: '%% github_registry_url %%' + TEMPEST_OPENRC: "%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%" + KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY: "%% github_registry_url %%" KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_USERNAME: '%% github_registry_username %%' - KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_PASSWORD: '%% github_registry_password %%' + KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_PASSWORD: "%% github_registry_password %%" HOME: '%% github_kayobe_arguments.HOME | default(github_default_kayobe_arguments.HOME) %%' - name: Print stdout diff --git a/roles/github/vars/main.yml b/roles/github/vars/main.yml index abb19e1..3edaf93 100644 --- a/roles/github/vars/main.yml +++ b/roles/github/vars/main.yml @@ -16,6 +16,7 @@ github_default_kayobe_arguments: KAYOBE_AUTOMATION_PR_URL: !unsafe https://api.github.com/repos/${{ github.repository }}/pulls KAYOBE_AUTOMATION_TEMPEST_LOADLIST: !unsafe "${{ github.event.inputs.testSuite }}" TEMPEST_PATTERN: !unsafe "${{ github.event.inputs.tempestPattern }}" + TEMPEST_OPENRC: !unsafe "${{ secrets.TEMPEST_OPENRC }}" HOME: /stack github_kayobe_dispatch_inputs: From 51c6b8cd04bc58daf48b53967204f615e5a3fe02 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 28 Sep 2023 11:58:07 +0100 Subject: [PATCH 05/55] fix: use `snake_case` for `tempest` vars --- roles/github/templates/run-tempest.yml.j2 | 6 +++--- roles/github/vars/main.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index baf6fa3..36cf99d 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -13,7 +13,7 @@ name: %% format_file_name(workflow.file_name, is_title=true) %% on: workflow_dispatch: inputs: - testSuite: + test_suite: description: | The list of tests to run. This should be a name of file under .automation.conf/tempest/load-lists. Defaults to running @@ -23,7 +23,7 @@ on: type: choice options: %% workflow.test_suites | flatten | join('') | indent(10) | trim %% - tempestPattern: + tempest_pattern: description: | Limit tests to this regex. Takes precedence over testSuite. <%- if github_environment_type == 'kayobe' +%> @@ -61,7 +61,7 @@ jobs: <% if github_kayobe_hook | length >= 1 %> %% github_kayobe_hook | indent(width=6, first=false) %% <% endif %> - - name: Run ${{ github.event.inputs.testSuite }} tempest testsuite + - name: Run ${{ github.event.inputs.test_suite }} tempest testsuite run: | /src/.automation/pipeline/tempest.sh -e ansible_user=stack env: diff --git a/roles/github/vars/main.yml b/roles/github/vars/main.yml index 3edaf93..9972dd3 100644 --- a/roles/github/vars/main.yml +++ b/roles/github/vars/main.yml @@ -14,8 +14,8 @@ github_default_kayobe_arguments: KAYOBE_AUTOMATION_PR_TYPE: github KAYOBE_AUTOMATION_PR_TITLE: !unsafe "[kayobe-automation] ${{ github.workflow }} #${{ github.run_id }}" KAYOBE_AUTOMATION_PR_URL: !unsafe https://api.github.com/repos/${{ github.repository }}/pulls - KAYOBE_AUTOMATION_TEMPEST_LOADLIST: !unsafe "${{ github.event.inputs.testSuite }}" - TEMPEST_PATTERN: !unsafe "${{ github.event.inputs.tempestPattern }}" + KAYOBE_AUTOMATION_TEMPEST_LOADLIST: !unsafe "${{ github.event.inputs.test_suite }}" + TEMPEST_PATTERN: !unsafe "${{ github.event.inputs.tempest_pattern }}" TEMPEST_OPENRC: !unsafe "${{ secrets.TEMPEST_OPENRC }}" HOME: /stack From 1d0afd2656b2cec4a6323780d98a7d19cdf15316 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 28 Sep 2023 12:00:17 +0100 Subject: [PATCH 06/55] feat: add environment support to `generic` --- roles/github/templates/generic.yml.j2 | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index af2391a..4653450 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -1,4 +1,13 @@ <% include "header.yml.j2" +%> +<%- if github_environment_type == 'kayobe' -%> +<%- set github_runs_on = github_runs_on + ['${{ inputs.kayobe_environment }}'] -%> +<%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', inputs.kayobe_environment)] }}" -%> +<%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', inputs.kayobe_environment)] }}" -%> +<%- set _ = workflow.update({"concurrency_group": "format('{0}-{1}', " + workflow.concurrency_group + ", '${{ inputs.kayobe_environment }}')" }) -%> +<%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> +<%- set _ = github_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', inputs.kayobe_environment)] }}" }) -%> +<%- endif +%> jobs: %% format_file_name(workflow.file_name) %%: @@ -38,8 +47,8 @@ jobs: <%- if workflow.arguments is defined +%> env: KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' - KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: "%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%" + KAYOBE_VAULT_PASSWORD: "%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%" <%- for argument in workflow.arguments | flatten +%> %% argument %%: '%% github_kayobe_arguments[argument] | default(github_default_kayobe_arguments[argument]) %%' <%- endfor +%> From c42b8682123f7d133deb3d13bafa765b405a53a4 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 28 Sep 2023 12:03:01 +0100 Subject: [PATCH 07/55] feat: add environment support to `build-kayobe` --- .../github/templates/build-kayobe-docker-image.yml.j2 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index 9df472b..d9b004a 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -1,3 +1,8 @@ +<%- if github_environment_type == 'kayobe' -%> +<%- set github_runs_on = github_runs_on + ['${{ matrix.environment }}'] -%> +<%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" -%> +<%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" -%> +<%- endif +%> name: %% format_file_name(workflow.file_name, is_title=true) %% on: @@ -9,6 +14,11 @@ env: jobs: %% format_file_name(workflow.file_name) %%: + <%- if github_environment_type == 'kayobe' +%> + strategy: + matrix: + environment: %% github_kayobe_environments %% + <%- endif +%> runs-on: %% github_runs_on %% container: image: docker:24.0-git From f17aaea8bbb9481e158e9fed74d44d486ed0ed5e Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 28 Sep 2023 12:05:15 +0100 Subject: [PATCH 08/55] feat: add environment support to `config-diff` --- roles/github/templates/run-config-diff.yml.j2 | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/roles/github/templates/run-config-diff.yml.j2 b/roles/github/templates/run-config-diff.yml.j2 index fefe9c4..844c252 100644 --- a/roles/github/templates/run-config-diff.yml.j2 +++ b/roles/github/templates/run-config-diff.yml.j2 @@ -1,3 +1,11 @@ +<%- if github_environment_type == 'kayobe' -%> +<%- set github_runs_on = github_runs_on + ['${{ matrix.environment }}'] -%> +<%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" -%> +<%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" -%> +<%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ matrix.environment }}'}) -%> +<%- set _ = github_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', matrix.environment)] }}" }) -%> +<%- set _ = github_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', matrix.environment)] }}" }) -%> +<%- endif +%> name: %% format_file_name(workflow.file_name, is_title=true) %% concurrency: @@ -9,6 +17,11 @@ on: jobs: %% format_file_name(workflow.file_name) %%: + <%- if github_environment_type == 'kayobe' +%> + strategy: + matrix: + environment: %% github_kayobe_environments %% + <%- endif +%> runs-on: %% github_runs_on %% permissions: contents: read @@ -43,8 +56,8 @@ jobs: sudo -E -u stack bash -c '/src/.automation/pipeline/config-diff.sh ${{ github.event.pull_request.base.sha }}' env: KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' - KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' + KAYOBE_VAULT_PASSWORD: "%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%" + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: "%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%" HOME: '%% github_kayobe_arguments.HOME | default(github_default_kayobe_arguments.HOME) %%' - name: Show summary of changes From cc800ca5c1bb2ad325d0468e4ac432c5d3a72629 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 28 Sep 2023 12:10:17 +0100 Subject: [PATCH 09/55] fix: add missing `environment` input --- roles/github/templates/header.yml.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/github/templates/header.yml.j2 b/roles/github/templates/header.yml.j2 index 966135d..6b16526 100644 --- a/roles/github/templates/header.yml.j2 +++ b/roles/github/templates/header.yml.j2 @@ -11,6 +11,9 @@ on: <%- if workflow.trigger['workflow_dispatch'] is not none +%> inputs: %% workflow.trigger['workflow_dispatch'] | flatten | join('') | indent(6) | trim %% + <%- if github_environment_type == 'kayobe' +%> + %% github_kayobe_environment_input | flatten | join('') | indent(6) | trim %% %% github_kayobe_environments %% + <%- endif +%> <%- endif +%> <%- endif +%> <%- endfor +%> From 266741ce055f45befc757c969a99b266fa3a413c Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 28 Sep 2023 12:20:24 +0100 Subject: [PATCH 10/55] fix: remove unwanted whitespace --- roles/github/templates/generic.yml.j2 | 2 +- roles/github/templates/run-tempest.yml.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 4653450..1b847cd 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -7,7 +7,7 @@ <%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> <%- set _ = github_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', inputs.kayobe_environment)] }}" }) -%> <%- set _ = github_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', inputs.kayobe_environment)] }}" }) -%> -<%- endif +%> +<%- endif -%> jobs: %% format_file_name(workflow.file_name) %%: diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index 36cf99d..b26b1d8 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -7,7 +7,7 @@ <%- set _ = github_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', inputs.kayobe_environment)] }}" }) -%> <%- set _ = github_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', inputs.kayobe_environment)] }}" }) -%> <%- set _ = github_kayobe_arguments.update({"TEMPEST_OPENRC": "${{ secrets[format('{0}_TEMPEST_OPENRC', inputs.kayobe_environment)] }}" }) -%> -<%- endif +%> +<%- endif -%> name: %% format_file_name(workflow.file_name, is_title=true) %% on: From cdc046f4c9c746b223c7d8a27d0230dfe7e2e3a8 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 28 Sep 2023 12:59:38 +0100 Subject: [PATCH 11/55] fix: cannot concatenate between str and list Role was failing if the user had not expanded `github_runs_on` to contain multiple entries in list. This is because the default value was a string which cannot be concatenated with a list. Fix: set `github_runs_on` to a list by default. --- roles/github/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 28809a6..8e5ddf6 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -3,9 +3,9 @@ github_output_directory: .github/workflows github_environment_type: none -github_kayobe_environments: none +github_kayobe_environments: [] -github_runs_on: self-hosted +github_runs_on: [self-hosted] github_registry_url: ghcr.io From 63164b79439edd75b4bca1a643ee05f307c0d07d Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 28 Sep 2023 13:01:52 +0100 Subject: [PATCH 12/55] fix: add missing newline --- roles/github/templates/generic.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 1b847cd..b527a75 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -1,4 +1,3 @@ -<% include "header.yml.j2" +%> <%- if github_environment_type == 'kayobe' -%> <%- set github_runs_on = github_runs_on + ['${{ inputs.kayobe_environment }}'] -%> <%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', inputs.kayobe_environment)] }}" -%> @@ -8,6 +7,7 @@ <%- set _ = github_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', inputs.kayobe_environment)] }}" }) -%> <%- set _ = github_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', inputs.kayobe_environment)] }}" }) -%> <%- endif -%> +<% include "header.yml.j2" +%> jobs: %% format_file_name(workflow.file_name) %%: From a145abec23f05109a6fc39b87d5c09634bf87e4d Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 28 Sep 2023 13:08:48 +0100 Subject: [PATCH 13/55] feat: test both single and multiple environments --- .github/workflows/test-collection.yml | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-collection.yml b/.github/workflows/test-collection.yml index 50da212..33695dc 100644 --- a/.github/workflows/test-collection.yml +++ b/.github/workflows/test-collection.yml @@ -19,13 +19,24 @@ jobs: - name: Install Ansible run: pip3 install ansible - - name: Test the playbook. - run: ansible-playbook tests/test.yml + - name: Test the playbook [single environment]. + run: "ansible-playbook tests/test.yml -e \"{'github_output_directory': '.github/workflows/single_environment'}\"" env: ANSIBLE_FORCE_COLOR: '1' - - name: Upload workflows produced + - name: Test the playbook [multiple environments]. + run: "ansible-playbook tests/test.yml -e \"{'github_output_directory': '.github/workflows/multiple_environments', 'github_environment_type': 'kayobe', 'github_kayobe_environments': [production, staging]}\"" + env: + ANSIBLE_FORCE_COLOR: '1' + + - name: Upload workflows produced [single environment] + uses: actions/upload-artifact@v3 + with: + name: github_kayobe_workflows_single_environment + path: tests/.github/workflows/single_environment + + - name: Upload workflows produced [multiple environment] uses: actions/upload-artifact@v3 with: - name: github_kayobe_workflows - path: tests/.github/workflows + name: github_kayobe_workflows_multiple_environments + path: tests/.github/workflows/multiple_environments From 742d01ed11121db8ece73bb5625bfeedaed02acd Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 28 Sep 2023 13:17:18 +0100 Subject: [PATCH 14/55] fix: remove excess newlines --- roles/github/templates/build-kayobe-docker-image.yml.j2 | 2 +- roles/github/templates/run-config-diff.yml.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index d9b004a..4e8e219 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -2,7 +2,7 @@ <%- set github_runs_on = github_runs_on + ['${{ matrix.environment }}'] -%> <%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" -%> <%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" -%> -<%- endif +%> +<%- endif -%> name: %% format_file_name(workflow.file_name, is_title=true) %% on: diff --git a/roles/github/templates/run-config-diff.yml.j2 b/roles/github/templates/run-config-diff.yml.j2 index 844c252..5d6e7f6 100644 --- a/roles/github/templates/run-config-diff.yml.j2 +++ b/roles/github/templates/run-config-diff.yml.j2 @@ -5,7 +5,7 @@ <%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ matrix.environment }}'}) -%> <%- set _ = github_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', matrix.environment)] }}" }) -%> <%- set _ = github_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', matrix.environment)] }}" }) -%> -<%- endif +%> +<%- endif -%> name: %% format_file_name(workflow.file_name, is_title=true) %% concurrency: From c47bd9f326c9cb4c1930ddeec1c9a0c12deda188 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 28 Sep 2023 17:43:52 +0100 Subject: [PATCH 15/55] fix: support `ghcr.io` with multiple environments --- .github/workflows/test-collection.yml | 22 +++++++++++++++++++ .../build-kayobe-docker-image.yml.j2 | 2 ++ roles/github/templates/generic.yml.j2 | 6 +++-- roles/github/templates/run-config-diff.yml.j2 | 2 ++ roles/github/templates/run-tempest.yml.j2 | 6 +++-- 5 files changed, 34 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-collection.yml b/.github/workflows/test-collection.yml index 33695dc..7050c2a 100644 --- a/.github/workflows/test-collection.yml +++ b/.github/workflows/test-collection.yml @@ -29,14 +29,36 @@ jobs: env: ANSIBLE_FORCE_COLOR: '1' + - name: Test the playbook [single environment with custom registry]. + run: "ansible-playbook tests/test.yml -e \"{'github_registry_url': 'registry.example.com', 'github_output_directory': '.github/workflows/single_environment_registry'}\"" + env: + ANSIBLE_FORCE_COLOR: '1' + + - name: Test the playbook [multiple environments with custom registry]. + run: "ansible-playbook tests/test.yml -e \"{'github_registry_url': 'registry.example.com', 'github_output_directory': '.github/workflows/multiple_environments_registry', 'github_environment_type': 'kayobe', 'github_kayobe_environments': [production, staging]}\"" + env: + ANSIBLE_FORCE_COLOR: '1' + - name: Upload workflows produced [single environment] uses: actions/upload-artifact@v3 with: name: github_kayobe_workflows_single_environment path: tests/.github/workflows/single_environment + - name: Upload workflows produced [single environment with custom registry] + uses: actions/upload-artifact@v3 + with: + name: github_kayobe_workflows_single_environment_registry + path: tests/.github/workflows/single_environment_registry + - name: Upload workflows produced [multiple environment] uses: actions/upload-artifact@v3 with: name: github_kayobe_workflows_multiple_environments path: tests/.github/workflows/multiple_environments + + - name: Upload workflows produced [multiple environment with custom registry] + uses: actions/upload-artifact@v3 + with: + name: github_kayobe_workflows_multiple_environments + path: tests/.github/workflows/multiple_environments_registry diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index 4e8e219..d803aae 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -1,8 +1,10 @@ <%- if github_environment_type == 'kayobe' -%> <%- set github_runs_on = github_runs_on + ['${{ matrix.environment }}'] -%> +<%- if github_registry_url != 'ghcr.io' -%> <%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" -%> <%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" -%> <%- endif -%> +<%- endif -%> name: %% format_file_name(workflow.file_name, is_title=true) %% on: diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index b527a75..8d843ce 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -1,7 +1,9 @@ <%- if github_environment_type == 'kayobe' -%> <%- set github_runs_on = github_runs_on + ['${{ inputs.kayobe_environment }}'] -%> -<%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', inputs.kayobe_environment)] }}" -%> -<%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', inputs.kayobe_environment)] }}" -%> +<%- if github_registry_url != 'ghcr.io' -%> +<%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" -%> +<%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" -%> +<%- endif -%> <%- set _ = workflow.update({"concurrency_group": "format('{0}-{1}', " + workflow.concurrency_group + ", '${{ inputs.kayobe_environment }}')" }) -%> <%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> <%- set _ = github_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', inputs.kayobe_environment)] }}" }) -%> diff --git a/roles/github/templates/run-config-diff.yml.j2 b/roles/github/templates/run-config-diff.yml.j2 index 5d6e7f6..2c217c3 100644 --- a/roles/github/templates/run-config-diff.yml.j2 +++ b/roles/github/templates/run-config-diff.yml.j2 @@ -1,7 +1,9 @@ <%- if github_environment_type == 'kayobe' -%> <%- set github_runs_on = github_runs_on + ['${{ matrix.environment }}'] -%> +<%- if github_registry_url != 'ghcr.io' -%> <%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" -%> <%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" -%> +<%- endif -%> <%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ matrix.environment }}'}) -%> <%- set _ = github_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', matrix.environment)] }}" }) -%> <%- set _ = github_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', matrix.environment)] }}" }) -%> diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index b26b1d8..6e82e73 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -1,7 +1,9 @@ <%- if github_environment_type == 'kayobe' -%> <%- set github_runs_on = github_runs_on + ['${{ inputs.kayobe_environment }}'] -%> -<%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', inputs.kayobe_environment)] }}" -%> -<%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', inputs.kayobe_environment)] }}" -%> +<%- if github_registry_url != 'ghcr.io' -%> +<%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" -%> +<%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" -%> +<%- endif -%> <%- set _ = workflow.update({"concurrency_group": "format('{0}-{1}', " + workflow.concurrency_group + ", '${{ inputs.kayobe_environment }}')" }) -%> <%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> <%- set _ = github_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', inputs.kayobe_environment)] }}" }) -%> From c3ff8924ce0d4dae6efc3b0027d9b88509421a28 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 28 Sep 2023 17:49:43 +0100 Subject: [PATCH 16/55] fix: artifact name collision --- .github/workflows/test-collection.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-collection.yml b/.github/workflows/test-collection.yml index 7050c2a..573f3ea 100644 --- a/.github/workflows/test-collection.yml +++ b/.github/workflows/test-collection.yml @@ -60,5 +60,5 @@ jobs: - name: Upload workflows produced [multiple environment with custom registry] uses: actions/upload-artifact@v3 with: - name: github_kayobe_workflows_multiple_environments + name: github_kayobe_workflows_multiple_environments_registry path: tests/.github/workflows/multiple_environments_registry From 61cce9d3f5dba040964bc41a289586bd44088fed Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 28 Sep 2023 18:03:56 +0100 Subject: [PATCH 17/55] feat: test registry support with custom password --- .github/workflows/test-collection.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-collection.yml b/.github/workflows/test-collection.yml index 573f3ea..0154dc8 100644 --- a/.github/workflows/test-collection.yml +++ b/.github/workflows/test-collection.yml @@ -30,12 +30,12 @@ jobs: ANSIBLE_FORCE_COLOR: '1' - name: Test the playbook [single environment with custom registry]. - run: "ansible-playbook tests/test.yml -e \"{'github_registry_url': 'registry.example.com', 'github_output_directory': '.github/workflows/single_environment_registry'}\"" + run: "ansible-playbook tests/test.yml -e \"{'github_registry_url': 'registry.example.com', 'github_registry_password': '${{ secrets.REGISTRY_PASSWORD }}', 'github_output_directory': '.github/workflows/single_environment_registry'}\"" env: ANSIBLE_FORCE_COLOR: '1' - name: Test the playbook [multiple environments with custom registry]. - run: "ansible-playbook tests/test.yml -e \"{'github_registry_url': 'registry.example.com', 'github_output_directory': '.github/workflows/multiple_environments_registry', 'github_environment_type': 'kayobe', 'github_kayobe_environments': [production, staging]}\"" + run: "ansible-playbook tests/test.yml -e \"{'github_registry_url': 'registry.example.com', 'github_registry_password': '${{ secrets.REGISTRY_PASSWORD }}', 'github_output_directory': '.github/workflows/multiple_environments_registry', 'github_environment_type': 'kayobe', 'github_kayobe_environments': [production, staging]}\"" env: ANSIBLE_FORCE_COLOR: '1' From e59e242575b99b770fb35457211824f235b557c6 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 28 Sep 2023 18:08:16 +0100 Subject: [PATCH 18/55] feat: use unsafe map --- .github/workflows/test-collection.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-collection.yml b/.github/workflows/test-collection.yml index 0154dc8..58c5846 100644 --- a/.github/workflows/test-collection.yml +++ b/.github/workflows/test-collection.yml @@ -30,12 +30,12 @@ jobs: ANSIBLE_FORCE_COLOR: '1' - name: Test the playbook [single environment with custom registry]. - run: "ansible-playbook tests/test.yml -e \"{'github_registry_url': 'registry.example.com', 'github_registry_password': '${{ secrets.REGISTRY_PASSWORD }}', 'github_output_directory': '.github/workflows/single_environment_registry'}\"" + run: "ansible-playbook tests/test.yml -e \"{'github_registry_url': 'registry.example.com', 'github_registry_password': '!unsafe ${{ secrets.REGISTRY_PASSWORD }}', 'github_output_directory': '.github/workflows/single_environment_registry'}\"" env: ANSIBLE_FORCE_COLOR: '1' - name: Test the playbook [multiple environments with custom registry]. - run: "ansible-playbook tests/test.yml -e \"{'github_registry_url': 'registry.example.com', 'github_registry_password': '${{ secrets.REGISTRY_PASSWORD }}', 'github_output_directory': '.github/workflows/multiple_environments_registry', 'github_environment_type': 'kayobe', 'github_kayobe_environments': [production, staging]}\"" + run: "ansible-playbook tests/test.yml -e \"{'github_registry_url': 'registry.example.com', 'github_registry_password': !unsafe ${{ secrets.REGISTRY_PASSWORD }}', 'github_output_directory': '.github/workflows/multiple_environments_registry', 'github_environment_type': 'kayobe', 'github_kayobe_environments': [production, staging]}\"" env: ANSIBLE_FORCE_COLOR: '1' From 566cfefe881c86ec2b40c14686083d90e0c8d011 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 28 Sep 2023 18:13:33 +0100 Subject: [PATCH 19/55] fix: use plaintext password in test --- .github/workflows/test-collection.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-collection.yml b/.github/workflows/test-collection.yml index 58c5846..4c4a06c 100644 --- a/.github/workflows/test-collection.yml +++ b/.github/workflows/test-collection.yml @@ -30,12 +30,12 @@ jobs: ANSIBLE_FORCE_COLOR: '1' - name: Test the playbook [single environment with custom registry]. - run: "ansible-playbook tests/test.yml -e \"{'github_registry_url': 'registry.example.com', 'github_registry_password': '!unsafe ${{ secrets.REGISTRY_PASSWORD }}', 'github_output_directory': '.github/workflows/single_environment_registry'}\"" + run: "ansible-playbook tests/test.yml -e \"{'github_registry_url': 'registry.example.com', 'github_registry_password': 'password123', 'github_output_directory': '.github/workflows/single_environment_registry'}\"" env: ANSIBLE_FORCE_COLOR: '1' - name: Test the playbook [multiple environments with custom registry]. - run: "ansible-playbook tests/test.yml -e \"{'github_registry_url': 'registry.example.com', 'github_registry_password': !unsafe ${{ secrets.REGISTRY_PASSWORD }}', 'github_output_directory': '.github/workflows/multiple_environments_registry', 'github_environment_type': 'kayobe', 'github_kayobe_environments': [production, staging]}\"" + run: "ansible-playbook tests/test.yml -e \"{'github_registry_url': 'registry.example.com', 'github_registry_password': password123', 'github_output_directory': '.github/workflows/multiple_environments_registry', 'github_environment_type': 'kayobe', 'github_kayobe_environments': [production, staging]}\"" env: ANSIBLE_FORCE_COLOR: '1' From 3a8128af9f9af7b156e8a6bfea213618e28fece7 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 3 Oct 2023 11:33:57 +0100 Subject: [PATCH 20/55] fix: use `github_environment_selector` --- roles/github/README.md | 2 +- roles/github/defaults/main.yml | 2 +- roles/github/templates/build-kayobe-docker-image.yml.j2 | 4 ++-- roles/github/templates/generic.yml.j2 | 2 +- roles/github/templates/header.yml.j2 | 2 +- roles/github/templates/run-config-diff.yml.j2 | 4 ++-- roles/github/templates/run-tempest.yml.j2 | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/roles/github/README.md b/roles/github/README.md index 2afc370..d796016 100644 --- a/roles/github/README.md +++ b/roles/github/README.md @@ -36,7 +36,7 @@ The following variables can be used to make small adjustments to the composition `github_output_directory`: control the location where the workflows shall be written to. -`github_environment_type`: control the type of environment support the workflows should be generated with. Either `none` for single environment deployment or `kayobe` for a kayobe based environment workflows. +`github_environment_selector`: control the type of environment support the workflows should be generated with. Either `none` for no environment, `single` for fixed environment or `input` whereby the environment is controlled at `workflow_dispatch` `github_kayobe_environments`: list of environments the workflows should target. Only has effect when `github_environment_type` is not `none`. diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 8e5ddf6..84465ab 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -1,7 +1,7 @@ --- github_output_directory: .github/workflows -github_environment_type: none +github_environment_selector: Null # Null | single | input github_kayobe_environments: [] diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index d803aae..23816a5 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -1,4 +1,4 @@ -<%- if github_environment_type == 'kayobe' -%> +<%- if github_environment_selector == 'input' -%> <%- set github_runs_on = github_runs_on + ['${{ matrix.environment }}'] -%> <%- if github_registry_url != 'ghcr.io' -%> <%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" -%> @@ -16,7 +16,7 @@ env: jobs: %% format_file_name(workflow.file_name) %%: - <%- if github_environment_type == 'kayobe' +%> + <%- if github_environment_type == 'input' +%> strategy: matrix: environment: %% github_kayobe_environments %% diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 8d843ce..a9fb272 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -1,4 +1,4 @@ -<%- if github_environment_type == 'kayobe' -%> +<%- if github_environment_selector == 'input' -%> <%- set github_runs_on = github_runs_on + ['${{ inputs.kayobe_environment }}'] -%> <%- if github_registry_url != 'ghcr.io' -%> <%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" -%> diff --git a/roles/github/templates/header.yml.j2 b/roles/github/templates/header.yml.j2 index 6b16526..5f95031 100644 --- a/roles/github/templates/header.yml.j2 +++ b/roles/github/templates/header.yml.j2 @@ -11,7 +11,7 @@ on: <%- if workflow.trigger['workflow_dispatch'] is not none +%> inputs: %% workflow.trigger['workflow_dispatch'] | flatten | join('') | indent(6) | trim %% - <%- if github_environment_type == 'kayobe' +%> + <%- if github_environment_selector == 'input' +%> %% github_kayobe_environment_input | flatten | join('') | indent(6) | trim %% %% github_kayobe_environments %% <%- endif +%> <%- endif +%> diff --git a/roles/github/templates/run-config-diff.yml.j2 b/roles/github/templates/run-config-diff.yml.j2 index 2c217c3..cb8c7b3 100644 --- a/roles/github/templates/run-config-diff.yml.j2 +++ b/roles/github/templates/run-config-diff.yml.j2 @@ -1,4 +1,4 @@ -<%- if github_environment_type == 'kayobe' -%> +<%- if github_environment_selector is not none -%> <%- set github_runs_on = github_runs_on + ['${{ matrix.environment }}'] -%> <%- if github_registry_url != 'ghcr.io' -%> <%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" -%> @@ -19,7 +19,7 @@ on: jobs: %% format_file_name(workflow.file_name) %%: - <%- if github_environment_type == 'kayobe' +%> + <%- if github_environment_type == 'input' +%> strategy: matrix: environment: %% github_kayobe_environments %% diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index 6e82e73..0f25ba3 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -1,4 +1,4 @@ -<%- if github_environment_type == 'kayobe' -%> +<%- if github_environment_selector == 'input' -%> <%- set github_runs_on = github_runs_on + ['${{ inputs.kayobe_environment }}'] -%> <%- if github_registry_url != 'ghcr.io' -%> <%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" -%> From 980a27c233efc3458e34541e41594a6d0c7bec4e Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 3 Oct 2023 11:46:20 +0100 Subject: [PATCH 21/55] fix: improve templating of `inputs` --- roles/github/defaults/main.yml | 4 ++-- roles/github/templates/header.yml.j2 | 2 +- roles/github/templates/run-tempest.yml.j2 | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 84465ab..f06bec9 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -65,8 +65,8 @@ github_kayobe_environment_input: | Select the environment the kayobe workflow shall target. type: choice required: true - default: 'production' - options: + default: '{{ github_kayobe_environments | first }}' + options: {{ github_kayobe_environments }} github_workflows: - "{{ github_build_kayobe_image }}" diff --git a/roles/github/templates/header.yml.j2 b/roles/github/templates/header.yml.j2 index 5f95031..9a8deee 100644 --- a/roles/github/templates/header.yml.j2 +++ b/roles/github/templates/header.yml.j2 @@ -12,7 +12,7 @@ on: inputs: %% workflow.trigger['workflow_dispatch'] | flatten | join('') | indent(6) | trim %% <%- if github_environment_selector == 'input' +%> - %% github_kayobe_environment_input | flatten | join('') | indent(6) | trim %% %% github_kayobe_environments %% + %% github_kayobe_environment_input | flatten | join('') | indent(6) | trim %% <%- endif +%> <%- endif +%> <%- endif +%> diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index 0f25ba3..52352a6 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -29,7 +29,7 @@ on: description: | Limit tests to this regex. Takes precedence over testSuite. <%- if github_environment_type == 'kayobe' +%> - %% github_kayobe_environment_input | flatten | join('') | indent(6) | trim %% %% github_kayobe_environments %% + %% github_kayobe_environment_input | flatten | join('') | indent(6) | trim %% <%- endif +%> jobs: From a05b5024ff9d9cd8b88b701039e2f4f81cf70ada Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 3 Oct 2023 11:54:12 +0100 Subject: [PATCH 22/55] feat: only pass `KAYOBE_ENVIRONMENT` if required --- roles/github/templates/generic.yml.j2 | 2 ++ roles/github/templates/run-config-diff.yml.j2 | 2 ++ roles/github/templates/run-tempest.yml.j2 | 2 ++ 3 files changed, 6 insertions(+) diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index a9fb272..fed81c7 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -48,7 +48,9 @@ jobs: /src/.automation/pipeline/%% workflow.file_name[4:-4] %%.sh <%- if workflow.arguments is defined +%> env: +<% if github_environment_selector is not none %> KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' +<% endif %> KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: "%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%" KAYOBE_VAULT_PASSWORD: "%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%" <%- for argument in workflow.arguments | flatten +%> diff --git a/roles/github/templates/run-config-diff.yml.j2 b/roles/github/templates/run-config-diff.yml.j2 index cb8c7b3..6523a1e 100644 --- a/roles/github/templates/run-config-diff.yml.j2 +++ b/roles/github/templates/run-config-diff.yml.j2 @@ -57,7 +57,9 @@ jobs: run: | sudo -E -u stack bash -c '/src/.automation/pipeline/config-diff.sh ${{ github.event.pull_request.base.sha }}' env: +<% if github_environment_selector is not none %> KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' +<% endif %> KAYOBE_VAULT_PASSWORD: "%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%" KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: "%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%" HOME: '%% github_kayobe_arguments.HOME | default(github_default_kayobe_arguments.HOME) %%' diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index 52352a6..cd51627 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -67,7 +67,9 @@ jobs: run: | /src/.automation/pipeline/tempest.sh -e ansible_user=stack env: +<% if github_environment_selector is not none %> KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' +<% endif %> KAYOBE_VAULT_PASSWORD: "%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%" KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: "%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%" KAYOBE_AUTOMATION_TEMPEST_LOADLIST: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_TEMPEST_LOADLIST | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_TEMPEST_LOADLIST) %%' From ab277bc5a728d3918d5804f4745f33a215d88493 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 3 Oct 2023 12:05:04 +0100 Subject: [PATCH 23/55] feat: allow arguments to be overriden with mult env --- roles/github/templates/generic.yml.j2 | 6 +++--- roles/github/templates/run-config-diff.yml.j2 | 9 ++++----- roles/github/templates/run-tempest.yml.j2 | 8 ++++---- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index fed81c7..6fa6997 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -5,9 +5,9 @@ <%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" -%> <%- endif -%> <%- set _ = workflow.update({"concurrency_group": "format('{0}-{1}', " + workflow.concurrency_group + ", '${{ inputs.kayobe_environment }}')" }) -%> -<%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> -<%- set _ = github_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', inputs.kayobe_environment)] }}" }) -%> -<%- set _ = github_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', inputs.kayobe_environment)] }}" }) -%> <%- endif -%> <% include "header.yml.j2" +%> diff --git a/roles/github/templates/run-config-diff.yml.j2 b/roles/github/templates/run-config-diff.yml.j2 index 6523a1e..16dc6ab 100644 --- a/roles/github/templates/run-config-diff.yml.j2 +++ b/roles/github/templates/run-config-diff.yml.j2 @@ -4,9 +4,8 @@ <%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" -%> <%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" -%> <%- endif -%> -<%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ matrix.environment }}'}) -%> -<%- set _ = github_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', matrix.environment)] }}" }) -%> -<%- set _ = github_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', matrix.environment)] }}" }) -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', matrix.environment)] }}" }) -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', matrix.environment)] }}" }) -%> <%- endif -%> name: %% format_file_name(workflow.file_name, is_title=true) %% @@ -19,7 +18,7 @@ on: jobs: %% format_file_name(workflow.file_name) %%: - <%- if github_environment_type == 'input' +%> + <%- if github_environment_type is not none +%> strategy: matrix: environment: %% github_kayobe_environments %% @@ -58,7 +57,7 @@ jobs: sudo -E -u stack bash -c '/src/.automation/pipeline/config-diff.sh ${{ github.event.pull_request.base.sha }}' env: <% if github_environment_selector is not none %> - KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' + KAYOBE_ENVIRONMENT: '${{ matrix.environment }}' <% endif %> KAYOBE_VAULT_PASSWORD: "%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%" KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: "%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%" diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index cd51627..2e0fd38 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -5,10 +5,10 @@ <%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" -%> <%- endif -%> <%- set _ = workflow.update({"concurrency_group": "format('{0}-{1}', " + workflow.concurrency_group + ", '${{ inputs.kayobe_environment }}')" }) -%> -<%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> -<%- set _ = github_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', inputs.kayobe_environment)] }}" }) -%> -<%- set _ = github_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', inputs.kayobe_environment)] }}" }) -%> -<%- set _ = github_kayobe_arguments.update({"TEMPEST_OPENRC": "${{ secrets[format('{0}_TEMPEST_OPENRC', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_kayobe_arguments.update({"TEMPEST_OPENRC": "${{ secrets[format('{0}_TEMPEST_OPENRC', inputs.kayobe_environment)] }}" }) -%> <%- endif -%> name: %% format_file_name(workflow.file_name, is_title=true) %% From a7fcbe33f1cde58d0271f41a39adeea488e971e7 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 3 Oct 2023 12:53:45 +0100 Subject: [PATCH 24/55] fix: improve `single` environment support --- roles/github/templates/build-kayobe-docker-image.yml.j2 | 2 +- roles/github/templates/generic.yml.j2 | 7 +++++-- roles/github/templates/run-config-diff.yml.j2 | 4 +++- roles/github/templates/run-tempest.yml.j2 | 9 ++++++--- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index 23816a5..796062f 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -16,7 +16,7 @@ env: jobs: %% format_file_name(workflow.file_name) %%: - <%- if github_environment_type == 'input' +%> + <%- if github_environment_selector == 'input' +%> strategy: matrix: environment: %% github_kayobe_environments %% diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 6fa6997..39948a6 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -1,14 +1,17 @@ <%- if github_environment_selector == 'input' -%> <%- set github_runs_on = github_runs_on + ['${{ inputs.kayobe_environment }}'] -%> <%- if github_registry_url != 'ghcr.io' -%> -<%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" -%> -<%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" -%> +<%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', inputs.kayobe_environment)] }}" -%> +<%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', inputs.kayobe_environment)] }}" -%> <%- endif -%> <%- set _ = workflow.update({"concurrency_group": "format('{0}-{1}', " + workflow.concurrency_group + ", '${{ inputs.kayobe_environment }}')" }) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', inputs.kayobe_environment)] }}" }) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', inputs.kayobe_environment)] }}" }) -%> <%- endif -%> +<%- if github_environment_selector == 'single' -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": github_kayobe_environments[0] }) -%> +<%- endif -%> <% include "header.yml.j2" +%> jobs: diff --git a/roles/github/templates/run-config-diff.yml.j2 b/roles/github/templates/run-config-diff.yml.j2 index 16dc6ab..ba2a61c 100644 --- a/roles/github/templates/run-config-diff.yml.j2 +++ b/roles/github/templates/run-config-diff.yml.j2 @@ -1,5 +1,4 @@ <%- if github_environment_selector is not none -%> -<%- set github_runs_on = github_runs_on + ['${{ matrix.environment }}'] -%> <%- if github_registry_url != 'ghcr.io' -%> <%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" -%> <%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" -%> @@ -7,6 +6,9 @@ <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', matrix.environment)] }}" }) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', matrix.environment)] }}" }) -%> <%- endif -%> +<%- if github_environment_selector == 'input' -%> +<%- set github_runs_on = github_runs_on + ['${{ matrix.environment }}'] -%> +<%- endif -%> name: %% format_file_name(workflow.file_name, is_title=true) %% concurrency: diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index 2e0fd38..a1a3d79 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -1,8 +1,8 @@ <%- if github_environment_selector == 'input' -%> <%- set github_runs_on = github_runs_on + ['${{ inputs.kayobe_environment }}'] -%> <%- if github_registry_url != 'ghcr.io' -%> -<%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" -%> -<%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" -%> +<%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', inputs.kayobe_environment)] }}" -%> +<%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', inputs.kayobe_environment)] }}" -%> <%- endif -%> <%- set _ = workflow.update({"concurrency_group": "format('{0}-{1}', " + workflow.concurrency_group + ", '${{ inputs.kayobe_environment }}')" }) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> @@ -10,6 +10,9 @@ <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', inputs.kayobe_environment)] }}" }) -%> <%- set _ = github_default_kayobe_arguments.update({"TEMPEST_OPENRC": "${{ secrets[format('{0}_TEMPEST_OPENRC', inputs.kayobe_environment)] }}" }) -%> <%- endif -%> +<%- if github_environment_selector == 'single' -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": github_kayobe_environments[0] }) -%> +<%- endif -%> name: %% format_file_name(workflow.file_name, is_title=true) %% on: @@ -28,7 +31,7 @@ on: tempest_pattern: description: | Limit tests to this regex. Takes precedence over testSuite. - <%- if github_environment_type == 'kayobe' +%> + <%- if github_environment_selector == 'kayobe' +%> %% github_kayobe_environment_input | flatten | join('') | indent(6) | trim %% <%- endif +%> From ef730a4d3169aa8dcc49fcb93ecdf7e4bde5833e Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 3 Oct 2023 16:01:32 +0100 Subject: [PATCH 25/55] fix: customise `TEMPEST_OPENRC` --- roles/github/templates/run-tempest.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index a1a3d79..eea913b 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -77,7 +77,7 @@ jobs: KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: "%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%" KAYOBE_AUTOMATION_TEMPEST_LOADLIST: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_TEMPEST_LOADLIST | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_TEMPEST_LOADLIST) %%' TEMPEST_PATTERN: '%% github_kayobe_arguments.TEMPEST_PATTERN | default(github_default_kayobe_arguments.TEMPEST_PATTERN) %%' - TEMPEST_OPENRC: "%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%" + TEMPEST_OPENRC: "%% github_kayobe_arguments.TEMPEST_OPENRC | default(github_default_kayobe_arguments.TEMPEST_OPENRC) %%" KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY: "%% github_registry_url %%" KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_USERNAME: '%% github_registry_username %%' KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_PASSWORD: "%% github_registry_password %%" From 0229ce11021e2e6dcdd6469a04b42ab445db29d1 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 4 Oct 2023 13:48:25 +0100 Subject: [PATCH 26/55] feat: allow registry attributes to be overridden --- roles/github/README.md | 4 +--- roles/github/defaults/main.yml | 6 +----- .../build-kayobe-docker-image.yml.j2 | 19 ++++++++--------- roles/github/templates/generic.yml.j2 | 18 +++++++--------- roles/github/templates/run-config-diff.yml.j2 | 15 +++++++------ roles/github/templates/run-tempest.yml.j2 | 21 +++++++++---------- roles/github/vars/main.yml | 5 +++++ 7 files changed, 41 insertions(+), 47 deletions(-) diff --git a/roles/github/README.md b/roles/github/README.md index d796016..42a0bfb 100644 --- a/roles/github/README.md +++ b/roles/github/README.md @@ -46,9 +46,7 @@ The following variables can be used to make small adjustments to the composition `github_image_tag`: tag used to select kayobe image defaults to `latest` -`github_registry_username`: username used to authenticate with the docker registry. - -`github_registry_password`: password used to authenticate with the docker registry. +`github_registry`: dictionary containing keys that correspond to `url`, `username` and `password` for the registry to be used by the workflows. Defaults to `ghcr.io` and uses the actors token to login. `github_kayobe_base_image`: select the base image used when building the kayobe docker image. Default is `quay.io/centos/centos:stream8` supports OpenStack Wallaby, Xena and Yoga. Zed and higher would require `quay.io/rockylinux/rockylinux:9`. diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index f06bec9..4a4094c 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -7,11 +7,7 @@ github_kayobe_environments: [] github_runs_on: [self-hosted] -github_registry_url: ghcr.io - -github_registry_username: !unsafe "${{ github.actor }}" - -github_registry_password: !unsafe "${{ secrets.GITHUB_TOKEN }}" +github_registry: {} github_image_name: kayobe diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index 796062f..44ab817 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -1,9 +1,8 @@ <%- if github_environment_selector == 'input' -%> <%- set github_runs_on = github_runs_on + ['${{ matrix.environment }}'] -%> -<%- if github_registry_url != 'ghcr.io' -%> -<%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" -%> -<%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" -%> -<%- endif -%> +<%- set _ = github_default_registry.update({"url": "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" }) -%> +<%- set _ = github_default_registry.update({"username": "${{ vars[format('{0}_REGISTRY_USERNAME', matrix.environment)] }}" }) -%> +<%- set _ = github_default_registry.update({"password": "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" }) -%> <%- endif -%> name: %% format_file_name(workflow.file_name, is_title=true) %% @@ -26,7 +25,7 @@ jobs: image: docker:24.0-git permissions: contents: read - packages: %% 'write' if github_registry_url == 'ghcr.io' else 'none' %% + packages: %% 'write' if (github_registry.url | default(github_default_registry.url)) == 'ghcr.io' else 'none' %% steps: <% if github_checkout_hook | length >= 1 %> %% github_checkout_hook | indent(width=6, first=false) %% @@ -40,9 +39,9 @@ jobs: - name: Log in to the Container registry uses: docker/login-action@v2 with: - registry: %% github_registry_url %% - username: %% github_registry_username %% - password: %% github_registry_password %% + registry: %% github_registry.url | default(github_default_registry.url) %% + username: %% github_registry.username | default(github_default_registry.username) %% + password: %% github_registry.password | default(github_default_registry.password) %% <% if github_buildx_enabled %> - name: Set up Docker Buildx @@ -72,8 +71,8 @@ jobs: BASE_IMAGE=%% github_kayobe_base_image %% push: true tags: | - %% github_registry_url %%/%% github_image_name %%:latest - %% github_registry_url %%/%% github_image_name %%:${{ github.sha }} + %% github_registry.url | default(github_default_registry.url) %%/%% github_image_name %%:latest + %% github_registry.url | default(github_default_registry.url) %%/%% github_image_name %%:${{ github.sha }} <% if not github_buildx_enable_provenance %> provenance: false <% endif %> diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 39948a6..65439ac 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -1,15 +1,13 @@ <%- if github_environment_selector == 'input' -%> <%- set github_runs_on = github_runs_on + ['${{ inputs.kayobe_environment }}'] -%> -<%- if github_registry_url != 'ghcr.io' -%> -<%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', inputs.kayobe_environment)] }}" -%> -<%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', inputs.kayobe_environment)] }}" -%> -<%- endif -%> <%- set _ = workflow.update({"concurrency_group": "format('{0}-{1}', " + workflow.concurrency_group + ", '${{ inputs.kayobe_environment }}')" }) -%> +<%- set _ = github_default_registry.update({"url": "${{ vars[format('{0}_REGISTRY_URL', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_registry.update({"username": "${{ vars[format('{0}_REGISTRY_USERNAME', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_registry.update({"password": "${{ secrets[format('{0}_REGISTRY_PASSWORD', inputs.kayobe_environment)] }}" }) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', inputs.kayobe_environment)] }}" }) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', inputs.kayobe_environment)] }}" }) -%> -<%- endif -%> -<%- if github_environment_selector == 'single' -%> +<%- elif github_environment_selector == 'single' -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": github_kayobe_environments[0] }) -%> <%- endif -%> <% include "header.yml.j2" +%> @@ -19,13 +17,13 @@ jobs: runs-on: %% github_runs_on %% permissions: contents: %% 'write' if 'KAYOBE_AUTOMATION_PR_TYPE' in workflow.arguments | flatten else 'read' %% - packages: %% 'read' if github_registry_url == 'ghcr.io' else 'none' %% + packages: %% 'read' if (github_registry.url | default(github_default_registry.url)) == 'ghcr.io' else 'none' %% pull-requests: %% 'write' if 'KAYOBE_AUTOMATION_PR_TYPE' in workflow.arguments | flatten else 'none' %% container: - image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% + image: %% github_registry.url | default(github_default_registry.url) %%/%% github_image_name %%:%% github_image_tag %% credentials: - username: %% github_registry_username %% - password: %% github_registry_password %% + username: %% github_registry.username | default(github_default_registry.username) %% + password: %% github_registry.password | default(github_default_registry.password) %% concurrency: group: %% workflow.concurrency_group %% cancel-in-progress: false diff --git a/roles/github/templates/run-config-diff.yml.j2 b/roles/github/templates/run-config-diff.yml.j2 index ba2a61c..a9935ff 100644 --- a/roles/github/templates/run-config-diff.yml.j2 +++ b/roles/github/templates/run-config-diff.yml.j2 @@ -1,8 +1,7 @@ <%- if github_environment_selector is not none -%> -<%- if github_registry_url != 'ghcr.io' -%> -<%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" -%> -<%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" -%> -<%- endif -%> +<%- set _ = github_default_registry.update({"url": "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" }) -%> +<%- set _ = github_default_registry.update({"username": "${{ vars[format('{0}_REGISTRY_USERNAME', matrix.environment)] }}" }) -%> +<%- set _ = github_default_registry.update({"password": "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" }) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', matrix.environment)] }}" }) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', matrix.environment)] }}" }) -%> <%- endif -%> @@ -28,12 +27,12 @@ jobs: runs-on: %% github_runs_on %% permissions: contents: read - packages: %% 'read' if github_registry_url == 'ghcr.io' else 'none' %% + packages: %% 'read' if (github_registry.url | default(github_default_registry.url)) == 'ghcr.io' else 'none' %% container: - image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% + image: %% github_registry.url | default(github_default_registry.url) %%/%% github_image_name %%:%% github_image_tag %% credentials: - username: %% github_registry_username %% - password: %% github_registry_password %% + username: %% github_registry.username | default(github_default_registry.username) %% + password: %% github_registry.password | default(github_default_registry.password) %% steps: <% if github_checkout_hook | length >= 1 %> %% github_checkout_hook | indent(width=6, first=false) %% diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index eea913b..a457c1d 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -1,10 +1,9 @@ <%- if github_environment_selector == 'input' -%> <%- set github_runs_on = github_runs_on + ['${{ inputs.kayobe_environment }}'] -%> -<%- if github_registry_url != 'ghcr.io' -%> -<%- set github_registry_url = "${{ vars[format('{0}_REGISTRY_URL', inputs.kayobe_environment)] }}" -%> -<%- set github_registry_password = "${{ secrets[format('{0}_REGISTRY_PASSWORD', inputs.kayobe_environment)] }}" -%> -<%- endif -%> <%- set _ = workflow.update({"concurrency_group": "format('{0}-{1}', " + workflow.concurrency_group + ", '${{ inputs.kayobe_environment }}')" }) -%> +<%- set _ = github_default_registry.update({"url": "${{ vars[format('{0}_REGISTRY_URL', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_registry.update({"username": "${{ vars[format('{0}_REGISTRY_USERNAME', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_registry.update({"password": "${{ secrets[format('{0}_REGISTRY_PASSWORD', inputs.kayobe_environment)] }}" }) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', inputs.kayobe_environment)] }}" }) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', inputs.kayobe_environment)] }}" }) -%> @@ -40,12 +39,12 @@ jobs: runs-on: %% github_runs_on %% permissions: contents: read - packages: %% 'read' if github_registry_url == 'ghcr.io' else 'none' %% + packages: %% 'read' if (github_registry.url | default(github_default_registry.url)) == 'ghcr.io' else 'none' %% container: - image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% + image: %% github_registry.url | default(github_default_registry.url) %%/%% github_image_name %%:%% github_image_tag %% credentials: - username: %% github_registry_username %% - password: %% github_registry_password %% + username: %% github_registry.username | default(github_default_registry.username) %% + password: %% github_registry.password | default(github_default_registry.password) %% concurrency: group: %% workflow.concurrency_group %% cancel-in-progress: false @@ -78,9 +77,9 @@ jobs: KAYOBE_AUTOMATION_TEMPEST_LOADLIST: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_TEMPEST_LOADLIST | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_TEMPEST_LOADLIST) %%' TEMPEST_PATTERN: '%% github_kayobe_arguments.TEMPEST_PATTERN | default(github_default_kayobe_arguments.TEMPEST_PATTERN) %%' TEMPEST_OPENRC: "%% github_kayobe_arguments.TEMPEST_OPENRC | default(github_default_kayobe_arguments.TEMPEST_OPENRC) %%" - KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY: "%% github_registry_url %%" - KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_USERNAME: '%% github_registry_username %%' - KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_PASSWORD: "%% github_registry_password %%" + KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY: "%% github_registry.url | default(github_default_registry.url) %%" + KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_USERNAME: '%% github_registry.username | default(github_default_registry.username) %%' + KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_PASSWORD: "%% github_registry.password | default(github_default_registry.password) %%" HOME: '%% github_kayobe_arguments.HOME | default(github_default_kayobe_arguments.HOME) %%' - name: Print stdout diff --git a/roles/github/vars/main.yml b/roles/github/vars/main.yml index 9972dd3..d7ac192 100644 --- a/roles/github/vars/main.yml +++ b/roles/github/vars/main.yml @@ -1,4 +1,9 @@ --- +github_default_registry: + url: ghcr.io + username: !unsafe "${{ github.actor }}" + password: !unsafe "${{ secrets.GITHUB_TOKEN }}" + github_default_kayobe_arguments: KAYOBE_ENVIRONMENT: production KAYOBE_VAULT_PASSWORD: !unsafe "${{ secrets.KAYOBE_VAULT_PASSWORD }}" From ba886a113872b6fde0afafeddbb01aba0f62ef5b Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 4 Oct 2023 13:51:34 +0100 Subject: [PATCH 27/55] feat: improve testing of `github` role --- .github/workflows/test-collection.yml | 57 +++++++------------ .../input_custom_kayobe_argument.yml | 9 +++ .../host_vars/input_custom_registry.yml | 9 +++ tests/inventory/host_vars/input_default.yml | 6 ++ .../host_vars/none_custom_kayobe_argument.yml | 5 ++ .../host_vars/none_custom_registry.yml | 7 +++ tests/inventory/host_vars/none_default.yml | 2 + tests/inventory/host_vars/reference.yml | 22 +++++++ .../single_custom_kayobe_argument.yml | 9 +++ .../host_vars/single_custom_registry.yml | 9 +++ tests/inventory/host_vars/single_default.yml | 6 ++ tests/inventory/hosts.yml | 23 ++++++++ tests/test.yml | 2 +- 13 files changed, 127 insertions(+), 39 deletions(-) create mode 100644 tests/inventory/host_vars/input_custom_kayobe_argument.yml create mode 100644 tests/inventory/host_vars/input_custom_registry.yml create mode 100644 tests/inventory/host_vars/input_default.yml create mode 100644 tests/inventory/host_vars/none_custom_kayobe_argument.yml create mode 100644 tests/inventory/host_vars/none_custom_registry.yml create mode 100644 tests/inventory/host_vars/none_default.yml create mode 100644 tests/inventory/host_vars/reference.yml create mode 100644 tests/inventory/host_vars/single_custom_kayobe_argument.yml create mode 100644 tests/inventory/host_vars/single_custom_registry.yml create mode 100644 tests/inventory/host_vars/single_default.yml create mode 100644 tests/inventory/hosts.yml diff --git a/.github/workflows/test-collection.yml b/.github/workflows/test-collection.yml index 4c4a06c..ef6e69e 100644 --- a/.github/workflows/test-collection.yml +++ b/.github/workflows/test-collection.yml @@ -7,6 +7,20 @@ jobs: test: name: Test github role runs-on: ubuntu-latest + strategy: + matrix: + name: + - none_default + - single_default + - input_default + - none_custom_registry + - single_custom_registry + - input_custom_registry + - none_custom_kayobe_argument + - single_custom_kayobe_argument + - input_custom_kayobe_argument + - reference + steps: - name: Check out the codebase. uses: actions/checkout@v3 @@ -19,46 +33,13 @@ jobs: - name: Install Ansible run: pip3 install ansible - - name: Test the playbook [single environment]. - run: "ansible-playbook tests/test.yml -e \"{'github_output_directory': '.github/workflows/single_environment'}\"" - env: - ANSIBLE_FORCE_COLOR: '1' - - - name: Test the playbook [multiple environments]. - run: "ansible-playbook tests/test.yml -e \"{'github_output_directory': '.github/workflows/multiple_environments', 'github_environment_type': 'kayobe', 'github_kayobe_environments': [production, staging]}\"" - env: - ANSIBLE_FORCE_COLOR: '1' - - - name: Test the playbook [single environment with custom registry]. - run: "ansible-playbook tests/test.yml -e \"{'github_registry_url': 'registry.example.com', 'github_registry_password': 'password123', 'github_output_directory': '.github/workflows/single_environment_registry'}\"" - env: - ANSIBLE_FORCE_COLOR: '1' - - - name: Test the playbook [multiple environments with custom registry]. - run: "ansible-playbook tests/test.yml -e \"{'github_registry_url': 'registry.example.com', 'github_registry_password': password123', 'github_output_directory': '.github/workflows/multiple_environments_registry', 'github_environment_type': 'kayobe', 'github_kayobe_environments': [production, staging]}\"" + - name: Test the playbook ${{ matrix.name }} + run: "ansible-playbook tests/test.yml --limit ${{ matrix.name }}" env: ANSIBLE_FORCE_COLOR: '1' - - name: Upload workflows produced [single environment] - uses: actions/upload-artifact@v3 - with: - name: github_kayobe_workflows_single_environment - path: tests/.github/workflows/single_environment - - - name: Upload workflows produced [single environment with custom registry] - uses: actions/upload-artifact@v3 - with: - name: github_kayobe_workflows_single_environment_registry - path: tests/.github/workflows/single_environment_registry - - - name: Upload workflows produced [multiple environment] - uses: actions/upload-artifact@v3 - with: - name: github_kayobe_workflows_multiple_environments - path: tests/.github/workflows/multiple_environments - - - name: Upload workflows produced [multiple environment with custom registry] + - name: Upload workflows produced ${{ matrix.name }} uses: actions/upload-artifact@v3 with: - name: github_kayobe_workflows_multiple_environments_registry - path: tests/.github/workflows/multiple_environments_registry + name: format("github_kayobe_workflows_{0}", ${{ matrix.name }} + path: format("tests/.github/{0}", ${{ matrix.name }}) diff --git a/tests/inventory/host_vars/input_custom_kayobe_argument.yml b/tests/inventory/host_vars/input_custom_kayobe_argument.yml new file mode 100644 index 0000000..55bd119 --- /dev/null +++ b/tests/inventory/host_vars/input_custom_kayobe_argument.yml @@ -0,0 +1,9 @@ +--- +github_output_directory: .github/input_custom_kayobe_argument + +github_environment_selector: input + +github_kayobe_environments: [prod-gb-01, test-gb-01] + +github_kayobe_arguments: + KAYOBE_VAULT_PASSWORD: !unsafe "${{ secrets.SUPER_SECRET_KAYOBE_VAULT_PASSWORD }}" diff --git a/tests/inventory/host_vars/input_custom_registry.yml b/tests/inventory/host_vars/input_custom_registry.yml new file mode 100644 index 0000000..71c0222 --- /dev/null +++ b/tests/inventory/host_vars/input_custom_registry.yml @@ -0,0 +1,9 @@ +--- +github_output_directory: .github/input_custom_registry + +github_environment_selector: input + +github_kayobe_environments: [prod-gb-01, test-gb-01] + +github_registry: + url: "pulp.example.com" diff --git a/tests/inventory/host_vars/input_default.yml b/tests/inventory/host_vars/input_default.yml new file mode 100644 index 0000000..4dacf23 --- /dev/null +++ b/tests/inventory/host_vars/input_default.yml @@ -0,0 +1,6 @@ +--- +github_output_directory: .github/input_default + +github_environment_selector: input + +github_kayobe_environments: [prod-gb-01, test-gb-01] diff --git a/tests/inventory/host_vars/none_custom_kayobe_argument.yml b/tests/inventory/host_vars/none_custom_kayobe_argument.yml new file mode 100644 index 0000000..8a877c2 --- /dev/null +++ b/tests/inventory/host_vars/none_custom_kayobe_argument.yml @@ -0,0 +1,5 @@ +--- +github_output_directory: .github/none_custom_kayobe_argument + +github_kayobe_arguments: + KAYOBE_VAULT_PASSWORD: !unsafe "${{ secrets.SUPER_SECRET_KAYOBE_VAULT_PASSWORD }}" diff --git a/tests/inventory/host_vars/none_custom_registry.yml b/tests/inventory/host_vars/none_custom_registry.yml new file mode 100644 index 0000000..29cfa1f --- /dev/null +++ b/tests/inventory/host_vars/none_custom_registry.yml @@ -0,0 +1,7 @@ +--- +github_output_directory: .github/none_custom_registry + +github_registry_url: "pulp.example.com" + +github_registry: + url: "pulp.example.com" diff --git a/tests/inventory/host_vars/none_default.yml b/tests/inventory/host_vars/none_default.yml new file mode 100644 index 0000000..697a314 --- /dev/null +++ b/tests/inventory/host_vars/none_default.yml @@ -0,0 +1,2 @@ +--- +github_output_directory: .github/none_default diff --git a/tests/inventory/host_vars/reference.yml b/tests/inventory/host_vars/reference.yml new file mode 100644 index 0000000..0733538 --- /dev/null +++ b/tests/inventory/host_vars/reference.yml @@ -0,0 +1,22 @@ +--- +github_output_directory: ".github/reference" + +github_environment_selector: input + +github_kayobe_environments: + - production-gb-01 + - testing-gb-01 + - production-de-01 + - testing-de-01 + +github_runs_on: + - kayobe + - openstack + +github_registry: + url: pulp.infra.os.example.cloud + username: admin + password: ${{ secrets.REGISTRY_PASSWORD }} + +github_kayobe_arguments: + KAYOBE_VAULT_PASSWORD: !unsafe "${{ secrets.KAYOBE_VAULT_PASSWORD }}" diff --git a/tests/inventory/host_vars/single_custom_kayobe_argument.yml b/tests/inventory/host_vars/single_custom_kayobe_argument.yml new file mode 100644 index 0000000..85fe3d0 --- /dev/null +++ b/tests/inventory/host_vars/single_custom_kayobe_argument.yml @@ -0,0 +1,9 @@ +--- +github_output_directory: .github/single_custom_kayobe_argument + +github_environment_selector: single + +github_kayobe_environments: [prod-gb-01] + +github_kayobe_arguments: + KAYOBE_VAULT_PASSWORD: !unsafe "${{ secrets.SUPER_SECRET_KAYOBE_VAULT_PASSWORD }}" diff --git a/tests/inventory/host_vars/single_custom_registry.yml b/tests/inventory/host_vars/single_custom_registry.yml new file mode 100644 index 0000000..eec4514 --- /dev/null +++ b/tests/inventory/host_vars/single_custom_registry.yml @@ -0,0 +1,9 @@ +--- +github_output_directory: .github/single_custom_registry + +github_environment_selector: single + +github_kayobe_environments: [prod-gb-01] + +github_registry: + url: "pulp.example.com" diff --git a/tests/inventory/host_vars/single_default.yml b/tests/inventory/host_vars/single_default.yml new file mode 100644 index 0000000..b45d3b2 --- /dev/null +++ b/tests/inventory/host_vars/single_default.yml @@ -0,0 +1,6 @@ +--- +github_output_directory: .github/single_default + +github_environment_selector: single + +github_kayobe_environments: [prod-gb-01] \ No newline at end of file diff --git a/tests/inventory/hosts.yml b/tests/inventory/hosts.yml new file mode 100644 index 0000000..021cb44 --- /dev/null +++ b/tests/inventory/hosts.yml @@ -0,0 +1,23 @@ +--- +tests_cases: + hosts: + none_default: + ansible_connection: local + single_default: + ansible_connection: local + input_default: + ansible_connection: local + none_custom_registry: + ansible_connection: local + single_custom_registry: + ansible_connection: local + input_custom_registry: + ansible_connection: local + none_custom_kayobe_argument: + ansible_connection: local + single_custom_kayobe_argument: + ansible_connection: local + input_custom_kayobe_argument: + ansible_connection: local + reference: + ansible_connection: local \ No newline at end of file diff --git a/tests/test.yml b/tests/test.yml index 59fe65b..17926f5 100644 --- a/tests/test.yml +++ b/tests/test.yml @@ -1,5 +1,5 @@ --- - name: Test `stackhpc.kayobe_automation` - hosts: localhost + hosts: all roles: - github From b440bf1ff01caa225b2825b468ba999b358b7d77 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 4 Oct 2023 13:56:51 +0100 Subject: [PATCH 28/55] fix: add `inventory` to playbook --- .github/workflows/test-collection.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-collection.yml b/.github/workflows/test-collection.yml index ef6e69e..c8a683c 100644 --- a/.github/workflows/test-collection.yml +++ b/.github/workflows/test-collection.yml @@ -34,7 +34,7 @@ jobs: run: pip3 install ansible - name: Test the playbook ${{ matrix.name }} - run: "ansible-playbook tests/test.yml --limit ${{ matrix.name }}" + run: "ansible-playbook -i tests/inventory/hosts.yml tests/test.yml --limit ${{ matrix.name }}" env: ANSIBLE_FORCE_COLOR: '1' From 3ce414113ad26680cbf95e1e0c7fa2217615e807 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 4 Oct 2023 14:03:02 +0100 Subject: [PATCH 29/55] fix: `format` func call --- .github/workflows/test-collection.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-collection.yml b/.github/workflows/test-collection.yml index c8a683c..31069ca 100644 --- a/.github/workflows/test-collection.yml +++ b/.github/workflows/test-collection.yml @@ -41,5 +41,5 @@ jobs: - name: Upload workflows produced ${{ matrix.name }} uses: actions/upload-artifact@v3 with: - name: format("github_kayobe_workflows_{0}", ${{ matrix.name }} - path: format("tests/.github/{0}", ${{ matrix.name }}) + name: ${{ format("github_kayobe_workflows_{0}", matrix.name) }} + path: ${{ format("tests/.github/{0}", matrix.name) }} From 860f674c8ee78ccbe76e1c7fa605b90b4506bb71 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 4 Oct 2023 15:33:29 +0100 Subject: [PATCH 30/55] fix: `format` syntax --- .github/workflows/test-collection.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-collection.yml b/.github/workflows/test-collection.yml index 31069ca..e672abc 100644 --- a/.github/workflows/test-collection.yml +++ b/.github/workflows/test-collection.yml @@ -41,5 +41,5 @@ jobs: - name: Upload workflows produced ${{ matrix.name }} uses: actions/upload-artifact@v3 with: - name: ${{ format("github_kayobe_workflows_{0}", matrix.name) }} - path: ${{ format("tests/.github/{0}", matrix.name) }} + name: "${{ format('github_kayobe_workflows_{0}', matrix.name) }}" + path: "${{ format('tests/.github/{0}', matrix.name) }}" \ No newline at end of file From 8203621c30f92d1f51b03d1e2255a1481085a806 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 4 Oct 2023 15:42:56 +0100 Subject: [PATCH 31/55] fix: linting issues --- .ansible-lint | 1 + tests/inventory/host_vars/single_default.yml | 2 +- tests/inventory/hosts.yml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.ansible-lint b/.ansible-lint index 31331b1..d835026 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -4,3 +4,4 @@ exclude_paths: skip_list: - galaxy[no-changelog] + - meta-runtime[unsupported-version] diff --git a/tests/inventory/host_vars/single_default.yml b/tests/inventory/host_vars/single_default.yml index b45d3b2..d99c840 100644 --- a/tests/inventory/host_vars/single_default.yml +++ b/tests/inventory/host_vars/single_default.yml @@ -3,4 +3,4 @@ github_output_directory: .github/single_default github_environment_selector: single -github_kayobe_environments: [prod-gb-01] \ No newline at end of file +github_kayobe_environments: [prod-gb-01] diff --git a/tests/inventory/hosts.yml b/tests/inventory/hosts.yml index 021cb44..930c925 100644 --- a/tests/inventory/hosts.yml +++ b/tests/inventory/hosts.yml @@ -20,4 +20,4 @@ tests_cases: input_custom_kayobe_argument: ansible_connection: local reference: - ansible_connection: local \ No newline at end of file + ansible_connection: local From 780a2359132cb53900e450a09b9f68884dfee7c2 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 13 Oct 2023 17:09:36 +0100 Subject: [PATCH 32/55] feat: add support for selecting `release` --- roles/github/defaults/main.yml | 7 ++++- .../build-kayobe-docker-image.yml.j2 | 3 +++ roles/github/templates/generic.yml.j2 | 3 +++ roles/github/templates/prepare-runner.yml.j2 | 27 +++++++++++++++++++ roles/github/templates/run-config-diff.yml.j2 | 7 ++--- roles/github/templates/run-tempest.yml.j2 | 3 +++ 6 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 roles/github/templates/prepare-runner.yml.j2 diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 4a4094c..3f2e672 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -11,7 +11,7 @@ github_registry: {} github_image_name: kayobe -github_image_tag: latest +github_image_tag: "${{ needs.prepare-runner.outputs.openstack_release }}-latest" github_kayobe_base_image: "quay.io/centos/centos:stream8" @@ -65,6 +65,7 @@ github_kayobe_environment_input: | options: {{ github_kayobe_environments }} github_workflows: + - "{{ github_prepare_runner }}" - "{{ github_build_kayobe_image }}" - "{{ github_run_kolla_config_diff }}" - "{{ github_run_infra_vm_host_configure }}" @@ -88,6 +89,10 @@ github_workflows: - "{{ github_run_seed_vm_provision }}" - "{{ github_run_tempest }}" +github_prepare_runner: + file_name: prepate-runner.yml + use_bespoke: true + github_build_kayobe_image: file_name: build-kayobe-docker-image.yml use_bespoke: true diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index 44ab817..41800e9 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -14,6 +14,8 @@ env: KAYOBE_USER_GID: 1000 jobs: + prepare-runner: + uses: ./.github/workflows/prepare-runner.yml %% format_file_name(workflow.file_name) %%: <%- if github_environment_selector == 'input' +%> strategy: @@ -26,6 +28,7 @@ jobs: permissions: contents: read packages: %% 'write' if (github_registry.url | default(github_default_registry.url)) == 'ghcr.io' else 'none' %% + needs: prepare-runner steps: <% if github_checkout_hook | length >= 1 %> %% github_checkout_hook | indent(width=6, first=false) %% diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 65439ac..5456f43 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -13,6 +13,8 @@ <% include "header.yml.j2" +%> jobs: + prepare-runner: + uses: ./.github/workflows/prepare-runner.yml %% format_file_name(workflow.file_name) %%: runs-on: %% github_runs_on %% permissions: @@ -28,6 +30,7 @@ jobs: group: %% workflow.concurrency_group %% cancel-in-progress: false timeout-minutes: %% github_timeout %% + needs: prepare-runner steps: <% if github_checkout_hook | length >= 1 %> %% github_checkout_hook | indent(width=6, first=false) %% diff --git a/roles/github/templates/prepare-runner.yml.j2 b/roles/github/templates/prepare-runner.yml.j2 new file mode 100644 index 0000000..efe8625 --- /dev/null +++ b/roles/github/templates/prepare-runner.yml.j2 @@ -0,0 +1,27 @@ +name: %% format_file_name(workflow.file_name, is_title=true) %% + +on: + workflow_call: + outputs: + openstack_release: + description: "The version of OpenStack/Kayobe to be used by the runner." + value: ${{ jobs.prepare-runner.outputs.openstack_release }} + +jobs: + prepare-runner: + runs-on: %% github_runs_on %% + container: + image: alpine:latest + permissions: + contents: read + packages: read + outputs: + openstack_release: ${{ steps.openstack_release.outputs.series }} + steps: + - name: Checkout kayobe config + uses: actions/checkout@v3 + + - name: Extract OpenStack Release + id: openstack_release + run: | + echo "series=$(awk -F'kayobe@' '{print $NF; exit}' requirements.txt | awk -F'/' '{print $NF; exit}')" >> $GITHUB_OUTPUT diff --git a/roles/github/templates/run-config-diff.yml.j2 b/roles/github/templates/run-config-diff.yml.j2 index a9935ff..7ef6538 100644 --- a/roles/github/templates/run-config-diff.yml.j2 +++ b/roles/github/templates/run-config-diff.yml.j2 @@ -1,11 +1,9 @@ -<%- if github_environment_selector is not none -%> +<%- if github_environment_selector == 'single' -%> <%- set _ = github_default_registry.update({"url": "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" }) -%> <%- set _ = github_default_registry.update({"username": "${{ vars[format('{0}_REGISTRY_USERNAME', matrix.environment)] }}" }) -%> <%- set _ = github_default_registry.update({"password": "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" }) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', matrix.environment)] }}" }) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', matrix.environment)] }}" }) -%> -<%- endif -%> -<%- if github_environment_selector == 'input' -%> <%- set github_runs_on = github_runs_on + ['${{ matrix.environment }}'] -%> <%- endif -%> name: %% format_file_name(workflow.file_name, is_title=true) %% @@ -18,6 +16,8 @@ on: pull_request: jobs: + prepare-runner: + uses: ./.github/workflows/prepare-runner.yml %% format_file_name(workflow.file_name) %%: <%- if github_environment_type is not none +%> strategy: @@ -33,6 +33,7 @@ jobs: credentials: username: %% github_registry.username | default(github_default_registry.username) %% password: %% github_registry.password | default(github_default_registry.password) %% + needs: prepare-runner steps: <% if github_checkout_hook | length >= 1 %> %% github_checkout_hook | indent(width=6, first=false) %% diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index a457c1d..0213018 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -35,6 +35,8 @@ on: <%- endif +%> jobs: + prepare-runner: + uses: ./.github/workflows/prepare-runner.yml %% format_file_name(workflow.file_name) %%: runs-on: %% github_runs_on %% permissions: @@ -49,6 +51,7 @@ jobs: group: %% workflow.concurrency_group %% cancel-in-progress: false timeout-minutes: %% github_timeout %% + needs: prepare-runner steps: <% if github_checkout_hook | length >= 1 %> %% github_checkout_hook | indent(width=6, first=false) %% From f979b701fc3ae882628b14c0746dfc5096afcbd8 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 13 Oct 2023 17:20:28 +0100 Subject: [PATCH 33/55] fix: `prepate` -> `prepare` --- roles/github/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 3f2e672..a3fabde 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -90,7 +90,7 @@ github_workflows: - "{{ github_run_tempest }}" github_prepare_runner: - file_name: prepate-runner.yml + file_name: prepare-runner.yml use_bespoke: true github_build_kayobe_image: From 26d9b644da38482278d740b48dd70c43c094be4f Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 13 Oct 2023 17:48:32 +0100 Subject: [PATCH 34/55] fix: add `github_image_tag` --- roles/github/templates/build-kayobe-docker-image.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index 41800e9..3943cf4 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -74,7 +74,7 @@ jobs: BASE_IMAGE=%% github_kayobe_base_image %% push: true tags: | - %% github_registry.url | default(github_default_registry.url) %%/%% github_image_name %%:latest + %% github_registry.url | default(github_default_registry.url) %%/%% github_image_name %%:%% github_image_tag %% %% github_registry.url | default(github_default_registry.url) %%/%% github_image_name %%:${{ github.sha }} <% if not github_buildx_enable_provenance %> provenance: false From 20f2d6de5072b4c1e5a9557b43620cdfe94e7f5d Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 13 Oct 2023 17:55:05 +0100 Subject: [PATCH 35/55] fix: single environment config diff --- roles/github/templates/build-kayobe-docker-image.yml.j2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index 3943cf4..5cb20b2 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -1,8 +1,12 @@ <%- if github_environment_selector == 'input' -%> -<%- set github_runs_on = github_runs_on + ['${{ matrix.environment }}'] -%> <%- set _ = github_default_registry.update({"url": "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" }) -%> <%- set _ = github_default_registry.update({"username": "${{ vars[format('{0}_REGISTRY_USERNAME', matrix.environment)] }}" }) -%> <%- set _ = github_default_registry.update({"password": "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" }) -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', matrix.environment)] }}" }) -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', matrix.environment)] }}" }) -%> +<%- endif -%> +<%- if github_environment_selector is not none -%> +<%- set github_runs_on = github_runs_on + ['${{ matrix.environment }}'] -%> <%- endif -%> name: %% format_file_name(workflow.file_name, is_title=true) %% From 5617be6adc386b6d2ce9ab37dc26ef70b82b1b92 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 13 Oct 2023 18:02:28 +0100 Subject: [PATCH 36/55] Revert "fix: single environment config diff" This reverts commit 20f2d6de5072b4c1e5a9557b43620cdfe94e7f5d. --- roles/github/templates/build-kayobe-docker-image.yml.j2 | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index 5cb20b2..3943cf4 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -1,12 +1,8 @@ <%- if github_environment_selector == 'input' -%> +<%- set github_runs_on = github_runs_on + ['${{ matrix.environment }}'] -%> <%- set _ = github_default_registry.update({"url": "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" }) -%> <%- set _ = github_default_registry.update({"username": "${{ vars[format('{0}_REGISTRY_USERNAME', matrix.environment)] }}" }) -%> <%- set _ = github_default_registry.update({"password": "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" }) -%> -<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', matrix.environment)] }}" }) -%> -<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', matrix.environment)] }}" }) -%> -<%- endif -%> -<%- if github_environment_selector is not none -%> -<%- set github_runs_on = github_runs_on + ['${{ matrix.environment }}'] -%> <%- endif -%> name: %% format_file_name(workflow.file_name, is_title=true) %% From 53a7d0eddcd0f611f6ecf12c6f8c3f696dfc3fd9 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 13 Oct 2023 18:03:20 +0100 Subject: [PATCH 37/55] fix: single environment config diff --- roles/github/templates/run-config-diff.yml.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/github/templates/run-config-diff.yml.j2 b/roles/github/templates/run-config-diff.yml.j2 index 7ef6538..5c946d4 100644 --- a/roles/github/templates/run-config-diff.yml.j2 +++ b/roles/github/templates/run-config-diff.yml.j2 @@ -1,9 +1,11 @@ -<%- if github_environment_selector == 'single' -%> +<%- if github_environment_selector == 'input' -%> <%- set _ = github_default_registry.update({"url": "${{ vars[format('{0}_REGISTRY_URL', matrix.environment)] }}" }) -%> <%- set _ = github_default_registry.update({"username": "${{ vars[format('{0}_REGISTRY_USERNAME', matrix.environment)] }}" }) -%> <%- set _ = github_default_registry.update({"password": "${{ secrets[format('{0}_REGISTRY_PASSWORD', matrix.environment)] }}" }) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', matrix.environment)] }}" }) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', matrix.environment)] }}" }) -%> +<%- endif -%> +<%- if github_environment_selector is not none -%> <%- set github_runs_on = github_runs_on + ['${{ matrix.environment }}'] -%> <%- endif -%> name: %% format_file_name(workflow.file_name, is_title=true) %% From 51a9b1788325c5a4369e094d7b518bb482c82d9a Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Sun, 15 Oct 2023 14:37:55 +0100 Subject: [PATCH 38/55] typo: add missing `e` --- roles/github/vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/vars/main.yml b/roles/github/vars/main.yml index d7ac192..53b83c8 100644 --- a/roles/github/vars/main.yml +++ b/roles/github/vars/main.yml @@ -11,7 +11,7 @@ github_default_kayobe_arguments: KOLLA_LIMIT: !unsafe "${{ github.event.inputs.kolla_limit }}" KOLLA_TAGS: !unsafe "${{ github.event.inputs.kolla_tags }}" KAYOBE_TAGS: !unsafe "${{ github.event.inputs.kayobe_tags }}" - KAYOBE_LIMIT: !unsafe "${{ github.event.inputs.kayob_limit }}" + KAYOBE_LIMIT: !unsafe "${{ github.event.inputs.kayobe_limit }}" KAYOBE_AUTOMATION_PR_TARGET_BRANCH: !unsafe ${{ github.event.ref }} KAYOBE_AUTOMATION_PR_REMOTE: !unsafe https://${KAYOBE_AUTOMATION_PR_GITHUB_USER}:${KAYOBE_AUTOMATION_PR_AUTH_TOKEN}@github.com/${{ github.repository }} KAYOBE_AUTOMATION_PR_GITHUB_USER: !unsafe ${{ github.actor }} From 02931bc5a078e5049ce6eb7be30279e89937e481 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Sun, 15 Oct 2023 17:21:05 +0100 Subject: [PATCH 39/55] fix: `github_environment_selector` value --- roles/github/templates/run-tempest.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index 0213018..c1e6d13 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -30,7 +30,7 @@ on: tempest_pattern: description: | Limit tests to this regex. Takes precedence over testSuite. - <%- if github_environment_selector == 'kayobe' +%> + <%- if github_environment_selector == 'input' +%> %% github_kayobe_environment_input | flatten | join('') | indent(6) | trim %% <%- endif +%> From 9777ab2e7c4caf066691ebf5e6d7166e132ae70e Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Sun, 15 Oct 2023 17:21:31 +0100 Subject: [PATCH 40/55] fix: mark `github_image_tag` as `unsafe` --- roles/github/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index a3fabde..ccd619b 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -11,7 +11,7 @@ github_registry: {} github_image_name: kayobe -github_image_tag: "${{ needs.prepare-runner.outputs.openstack_release }}-latest" +github_image_tag: !unsafe "${{ needs.prepare-runner.outputs.openstack_release }}-latest" github_kayobe_base_image: "quay.io/centos/centos:stream8" From ee23057306fffe9c1106b8cd71e8285173a73980 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 17 Oct 2023 09:29:43 +0100 Subject: [PATCH 41/55] feat: add support for sharing registry --- roles/github/README.md | 2 +- roles/github/templates/build-kayobe-docker-image.yml.j2 | 2 +- roles/github/vars/main.yml | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/github/README.md b/roles/github/README.md index 42a0bfb..c80d86d 100644 --- a/roles/github/README.md +++ b/roles/github/README.md @@ -46,7 +46,7 @@ The following variables can be used to make small adjustments to the composition `github_image_tag`: tag used to select kayobe image defaults to `latest` -`github_registry`: dictionary containing keys that correspond to `url`, `username` and `password` for the registry to be used by the workflows. Defaults to `ghcr.io` and uses the actors token to login. +`github_registry`: dictionary containing keys that correspond to `url`, `username`, `password` and `share` for the registry to be used by the workflows. Defaults to `ghcr.io` and uses the actors token to login. The key `share` is to indiciate if the registry is shared between environments. `github_kayobe_base_image`: select the base image used when building the kayobe docker image. Default is `quay.io/centos/centos:stream8` supports OpenStack Wallaby, Xena and Yoga. Zed and higher would require `quay.io/rockylinux/rockylinux:9`. diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index 3943cf4..75fb06a 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -17,7 +17,7 @@ jobs: prepare-runner: uses: ./.github/workflows/prepare-runner.yml %% format_file_name(workflow.file_name) %%: - <%- if github_environment_selector == 'input' +%> + <%- if github_environment_selector == 'input' and (github_registry.share | default(github_default_registry.share)) is false +%> strategy: matrix: environment: %% github_kayobe_environments %% diff --git a/roles/github/vars/main.yml b/roles/github/vars/main.yml index 53b83c8..ca2a8d9 100644 --- a/roles/github/vars/main.yml +++ b/roles/github/vars/main.yml @@ -3,6 +3,7 @@ github_default_registry: url: ghcr.io username: !unsafe "${{ github.actor }}" password: !unsafe "${{ secrets.GITHUB_TOKEN }}" + share: false github_default_kayobe_arguments: KAYOBE_ENVIRONMENT: production From 612c1b79181fae6ac780dbcd03880a7b1c617bd0 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 23 Oct 2023 08:37:02 +0000 Subject: [PATCH 42/55] fix: remove unused variable from `tests` --- tests/inventory/host_vars/none_custom_registry.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/inventory/host_vars/none_custom_registry.yml b/tests/inventory/host_vars/none_custom_registry.yml index 29cfa1f..fbbcd40 100644 --- a/tests/inventory/host_vars/none_custom_registry.yml +++ b/tests/inventory/host_vars/none_custom_registry.yml @@ -1,7 +1,5 @@ --- github_output_directory: .github/none_custom_registry -github_registry_url: "pulp.example.com" - github_registry: url: "pulp.example.com" From 78274602909e35c7c8f736f58ef021e17a66e791 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 23 Oct 2023 14:27:59 +0000 Subject: [PATCH 43/55] feat: remove rally registry settings --- roles/github/templates/run-tempest.yml.j2 | 3 --- 1 file changed, 3 deletions(-) diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index c1e6d13..6b1f35c 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -80,9 +80,6 @@ jobs: KAYOBE_AUTOMATION_TEMPEST_LOADLIST: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_TEMPEST_LOADLIST | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_TEMPEST_LOADLIST) %%' TEMPEST_PATTERN: '%% github_kayobe_arguments.TEMPEST_PATTERN | default(github_default_kayobe_arguments.TEMPEST_PATTERN) %%' TEMPEST_OPENRC: "%% github_kayobe_arguments.TEMPEST_OPENRC | default(github_default_kayobe_arguments.TEMPEST_OPENRC) %%" - KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY: "%% github_registry.url | default(github_default_registry.url) %%" - KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_USERNAME: '%% github_registry.username | default(github_default_registry.username) %%' - KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_PASSWORD: "%% github_registry.password | default(github_default_registry.password) %%" HOME: '%% github_kayobe_arguments.HOME | default(github_default_kayobe_arguments.HOME) %%' - name: Print stdout From 1009a064043fc21f1f8a22cbded6d4801811b74d Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 23 Oct 2023 14:28:58 +0000 Subject: [PATCH 44/55] fix: use newer environment variable --- roles/github/templates/run-config-diff.yml.j2 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/github/templates/run-config-diff.yml.j2 b/roles/github/templates/run-config-diff.yml.j2 index 5c946d4..e6f3096 100644 --- a/roles/github/templates/run-config-diff.yml.j2 +++ b/roles/github/templates/run-config-diff.yml.j2 @@ -21,7 +21,7 @@ jobs: prepare-runner: uses: ./.github/workflows/prepare-runner.yml %% format_file_name(workflow.file_name) %%: - <%- if github_environment_type is not none +%> + <%- if github_environment_selector is not none +%> strategy: matrix: environment: %% github_kayobe_environments %% @@ -64,7 +64,6 @@ jobs: KAYOBE_ENVIRONMENT: '${{ matrix.environment }}' <% endif %> KAYOBE_VAULT_PASSWORD: "%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%" - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: "%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%" HOME: '%% github_kayobe_arguments.HOME | default(github_default_kayobe_arguments.HOME) %%' - name: Show summary of changes From 44ed47f107a7789958c0592e667154df1a25eb4a Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 23 Oct 2023 14:29:39 +0000 Subject: [PATCH 45/55] feat: use `.gitreview` for selecting release --- roles/github/templates/prepare-runner.yml.j2 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/github/templates/prepare-runner.yml.j2 b/roles/github/templates/prepare-runner.yml.j2 index efe8625..a87463d 100644 --- a/roles/github/templates/prepare-runner.yml.j2 +++ b/roles/github/templates/prepare-runner.yml.j2 @@ -16,7 +16,7 @@ jobs: contents: read packages: read outputs: - openstack_release: ${{ steps.openstack_release.outputs.series }} + openstack_release: ${{ steps.openstack_release.outputs.openstack_release }} steps: - name: Checkout kayobe config uses: actions/checkout@v3 @@ -24,4 +24,5 @@ jobs: - name: Extract OpenStack Release id: openstack_release run: | - echo "series=$(awk -F'kayobe@' '{print $NF; exit}' requirements.txt | awk -F'/' '{print $NF; exit}')" >> $GITHUB_OUTPUT + BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) + echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT From 9a47c2632dd6445bd86d7228eaf696311cf06698 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 23 Oct 2023 14:30:19 +0000 Subject: [PATCH 46/55] feat: empty var for `github_environment_selector` --- roles/github/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index ccd619b..005b5e0 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -1,7 +1,7 @@ --- github_output_directory: .github/workflows -github_environment_selector: Null # Null | single | input +github_environment_selector: github_kayobe_environments: [] From 45c95a0be7a277d7af6bdce2d2e4585f93823f73 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 23 Oct 2023 15:37:47 +0100 Subject: [PATCH 47/55] Update roles/github/README.md Co-authored-by: Mark Goddard --- roles/github/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/README.md b/roles/github/README.md index c80d86d..f09865b 100644 --- a/roles/github/README.md +++ b/roles/github/README.md @@ -38,7 +38,7 @@ The following variables can be used to make small adjustments to the composition `github_environment_selector`: control the type of environment support the workflows should be generated with. Either `none` for no environment, `single` for fixed environment or `input` whereby the environment is controlled at `workflow_dispatch` -`github_kayobe_environments`: list of environments the workflows should target. Only has effect when `github_environment_type` is not `none`. +`github_kayobe_environments`: list of environments the workflows should target. Only has effect when `github_environment_selector` is not `none`. `github_runs_on`: control which runner can accept this workflow. See GitHub for more information on [runs-on](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on). From 85612c5906a48df5bc36ff5d0ede6621daf5f573 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 23 Oct 2023 17:39:09 +0000 Subject: [PATCH 48/55] feat: remove `header.yml.j2` --- roles/github/templates/generic.yml.j2 | 21 ++++++++++++++++++++- roles/github/templates/header.yml.j2 | 20 -------------------- 2 files changed, 20 insertions(+), 21 deletions(-) delete mode 100644 roles/github/templates/header.yml.j2 diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 5456f43..2ab1aca 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -10,7 +10,26 @@ <%- elif github_environment_selector == 'single' -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": github_kayobe_environments[0] }) -%> <%- endif -%> -<% include "header.yml.j2" +%> +name: %% format_file_name(workflow.file_name, is_title=true) %% + +on: +<%- if workflow.trigger is defined +%> + <%- for trigger_name in workflow.trigger.keys() +%> + <%- if trigger_name == 'schedule' +%> + schedule: + - cron: '%% workflow.trigger['schedule']['cron'] %%' + <%- elif trigger_name == 'workflow_dispatch' +%> + workflow_dispatch: + <%- if workflow.trigger['workflow_dispatch'] is not none +%> + inputs: + %% workflow.trigger['workflow_dispatch'] | flatten | join('') | indent(6) | trim %% + <%- if github_environment_selector == 'input' +%> + %% github_kayobe_environment_input | flatten | join('') | indent(6) | trim %% + <%- endif +%> + <%- endif +%> + <%- endif +%> + <%- endfor +%> +<%- endif +%> jobs: prepare-runner: diff --git a/roles/github/templates/header.yml.j2 b/roles/github/templates/header.yml.j2 deleted file mode 100644 index 9a8deee..0000000 --- a/roles/github/templates/header.yml.j2 +++ /dev/null @@ -1,20 +0,0 @@ -name: %% format_file_name(workflow.file_name, is_title=true) %% - -on: -<%- if workflow.trigger is defined +%> - <%- for trigger_name in workflow.trigger.keys() +%> - <%- if trigger_name == 'schedule' +%> - schedule: - - cron: '%% workflow.trigger['schedule']['cron'] %%' - <%- elif trigger_name == 'workflow_dispatch' +%> - workflow_dispatch: - <%- if workflow.trigger['workflow_dispatch'] is not none +%> - inputs: - %% workflow.trigger['workflow_dispatch'] | flatten | join('') | indent(6) | trim %% - <%- if github_environment_selector == 'input' +%> - %% github_kayobe_environment_input | flatten | join('') | indent(6) | trim %% - <%- endif +%> - <%- endif +%> - <%- endif +%> - <%- endfor +%> -<%- endif +%> From e0d46e8d2ad720fc199337df209e7785636fd5da Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 24 Oct 2023 09:59:13 +0100 Subject: [PATCH 49/55] feat: remove `KAYOBE_ENVIRONMENT` The variable in `github_default_kayobe_arguments` served no purpose as users most likely want to provide their own environment name. --- roles/github/templates/generic.yml.j2 | 6 ++---- roles/github/templates/run-tempest.yml.j2 | 6 +++--- roles/github/vars/main.yml | 1 - 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 2ab1aca..bb30588 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -4,11 +4,9 @@ <%- set _ = github_default_registry.update({"url": "${{ vars[format('{0}_REGISTRY_URL', inputs.kayobe_environment)] }}" }) -%> <%- set _ = github_default_registry.update({"username": "${{ vars[format('{0}_REGISTRY_USERNAME', inputs.kayobe_environment)] }}" }) -%> <%- set _ = github_default_registry.update({"password": "${{ secrets[format('{0}_REGISTRY_PASSWORD', inputs.kayobe_environment)] }}" }) -%> -<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', inputs.kayobe_environment)] }}" }) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', inputs.kayobe_environment)] }}" }) -%> -<%- elif github_environment_selector == 'single' -%> -<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": github_kayobe_environments[0] }) -%> +<%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> <%- endif -%> name: %% format_file_name(workflow.file_name, is_title=true) %% @@ -72,7 +70,7 @@ jobs: <%- if workflow.arguments is defined +%> env: <% if github_environment_selector is not none %> - KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' + KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT %%' <% endif %> KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: "%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%" KAYOBE_VAULT_PASSWORD: "%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%" diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index 6b1f35c..aa460c8 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -4,13 +4,13 @@ <%- set _ = github_default_registry.update({"url": "${{ vars[format('{0}_REGISTRY_URL', inputs.kayobe_environment)] }}" }) -%> <%- set _ = github_default_registry.update({"username": "${{ vars[format('{0}_REGISTRY_USERNAME', inputs.kayobe_environment)] }}" }) -%> <%- set _ = github_default_registry.update({"password": "${{ secrets[format('{0}_REGISTRY_PASSWORD', inputs.kayobe_environment)] }}" }) -%> -<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> +<%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', inputs.kayobe_environment)] }}" }) -%> <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', inputs.kayobe_environment)] }}" }) -%> <%- set _ = github_default_kayobe_arguments.update({"TEMPEST_OPENRC": "${{ secrets[format('{0}_TEMPEST_OPENRC', inputs.kayobe_environment)] }}" }) -%> <%- endif -%> <%- if github_environment_selector == 'single' -%> -<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": github_kayobe_environments[0] }) -%> +<%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": github_kayobe_environments[0] }) -%> <%- endif -%> name: %% format_file_name(workflow.file_name, is_title=true) %% @@ -73,7 +73,7 @@ jobs: /src/.automation/pipeline/tempest.sh -e ansible_user=stack env: <% if github_environment_selector is not none %> - KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' + KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT %%' <% endif %> KAYOBE_VAULT_PASSWORD: "%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%" KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: "%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%" diff --git a/roles/github/vars/main.yml b/roles/github/vars/main.yml index ca2a8d9..f4be3ec 100644 --- a/roles/github/vars/main.yml +++ b/roles/github/vars/main.yml @@ -6,7 +6,6 @@ github_default_registry: share: false github_default_kayobe_arguments: - KAYOBE_ENVIRONMENT: production KAYOBE_VAULT_PASSWORD: !unsafe "${{ secrets.KAYOBE_VAULT_PASSWORD }}" KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: !unsafe "${{ secrets.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY }}" KOLLA_LIMIT: !unsafe "${{ github.event.inputs.kolla_limit }}" From 5e107cbba928f32bf2ec0f50ce41943b4250f5a2 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 24 Oct 2023 10:48:11 +0000 Subject: [PATCH 50/55] fix: undefined variable for `single` env workflows --- roles/github/templates/generic.yml.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index bb30588..e5fe80a 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -8,6 +8,9 @@ <%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', inputs.kayobe_environment)] }}" }) -%> <%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> <%- endif -%> +<%- if github_environment_selector == 'single' -%> +<%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": github_kayobe_environments | first}) -%> +<%- endif -%> name: %% format_file_name(workflow.file_name, is_title=true) %% on: From 1f242d5ab240a5493be29fa0930ddc5ca001baf8 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 24 Oct 2023 10:50:27 +0000 Subject: [PATCH 51/55] feat: disable `buildx` step by default The `setup-buildx` step within the workflow has been known to cause issue when pushing images to Pulp registry < `3.32.0`. There was a concern surrounding the lack of buildx step potentially leading to a situation where the kayobe-config remains inside the image. However, this does not appear to be the case. --- roles/github/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 005b5e0..bfd185f 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -23,7 +23,7 @@ github_kayobe_hook: "" github_final_hook: "" -github_buildx_enabled: true +github_buildx_enabled: false github_buildx_inline_config: "" From a2dd70e015103081fb2712de0420a0b15f5a810e Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 24 Oct 2023 10:57:46 +0000 Subject: [PATCH 52/55] Revert "feat: remove `header.yml.j2`" This reverts commit 85612c5906a48df5bc36ff5d0ede6621daf5f573. --- roles/github/templates/generic.yml.j2 | 21 +-------------------- roles/github/templates/header.yml.j2 | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 20 deletions(-) create mode 100644 roles/github/templates/header.yml.j2 diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index e5fe80a..5613687 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -11,26 +11,7 @@ <%- if github_environment_selector == 'single' -%> <%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": github_kayobe_environments | first}) -%> <%- endif -%> -name: %% format_file_name(workflow.file_name, is_title=true) %% - -on: -<%- if workflow.trigger is defined +%> - <%- for trigger_name in workflow.trigger.keys() +%> - <%- if trigger_name == 'schedule' +%> - schedule: - - cron: '%% workflow.trigger['schedule']['cron'] %%' - <%- elif trigger_name == 'workflow_dispatch' +%> - workflow_dispatch: - <%- if workflow.trigger['workflow_dispatch'] is not none +%> - inputs: - %% workflow.trigger['workflow_dispatch'] | flatten | join('') | indent(6) | trim %% - <%- if github_environment_selector == 'input' +%> - %% github_kayobe_environment_input | flatten | join('') | indent(6) | trim %% - <%- endif +%> - <%- endif +%> - <%- endif +%> - <%- endfor +%> -<%- endif +%> +<% include "header.yml.j2" +%> jobs: prepare-runner: diff --git a/roles/github/templates/header.yml.j2 b/roles/github/templates/header.yml.j2 new file mode 100644 index 0000000..9a8deee --- /dev/null +++ b/roles/github/templates/header.yml.j2 @@ -0,0 +1,20 @@ +name: %% format_file_name(workflow.file_name, is_title=true) %% + +on: +<%- if workflow.trigger is defined +%> + <%- for trigger_name in workflow.trigger.keys() +%> + <%- if trigger_name == 'schedule' +%> + schedule: + - cron: '%% workflow.trigger['schedule']['cron'] %%' + <%- elif trigger_name == 'workflow_dispatch' +%> + workflow_dispatch: + <%- if workflow.trigger['workflow_dispatch'] is not none +%> + inputs: + %% workflow.trigger['workflow_dispatch'] | flatten | join('') | indent(6) | trim %% + <%- if github_environment_selector == 'input' +%> + %% github_kayobe_environment_input | flatten | join('') | indent(6) | trim %% + <%- endif +%> + <%- endif +%> + <%- endif +%> + <%- endfor +%> +<%- endif +%> From 932b690addf4efc69bf46ba81499fccdc2ae8d0c Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 24 Oct 2023 12:02:30 +0000 Subject: [PATCH 53/55] fix!: backwards compatibility with nested vars Identified in `ansible-core < 2.15` nested vars will not be evaluated within templates that are loaded via `extends` or `includes`. The solution involves using `lookup('template', ...)` to ensure support dating back to `2.12`. Unfortunately `lookup` does not have access to the macro defined in `workflows.yml` nor does it inherit the `block_*_string` and `variable_*_string` from the `ansible.builtin.template` module as defined within `tasks/main.yml`. Also `lookup` does not support setting `block_*_string`` NOTE: this is not ideal due to the duplication and the changing of block/variable markers. Whilst low priority should be improved/replaced in the future. --- roles/github/templates/generic.yml.j2 | 3 +- roles/github/templates/header.yml.j2 | 40 ++++++++++++-------- roles/github/templates/run-tempest.yml.j2 | 23 +---------- roles/github/templates/tempest-header.yml.j2 | 31 +++++++++++++++ 4 files changed, 58 insertions(+), 39 deletions(-) create mode 100644 roles/github/templates/tempest-header.yml.j2 diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 5613687..6fbe206 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -11,8 +11,7 @@ <%- if github_environment_selector == 'single' -%> <%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": github_kayobe_environments | first}) -%> <%- endif -%> -<% include "header.yml.j2" +%> - +%% lookup('template', 'header.yml.j2') %% jobs: prepare-runner: uses: ./.github/workflows/prepare-runner.yml diff --git a/roles/github/templates/header.yml.j2 b/roles/github/templates/header.yml.j2 index 9a8deee..ca601f6 100644 --- a/roles/github/templates/header.yml.j2 +++ b/roles/github/templates/header.yml.j2 @@ -1,20 +1,30 @@ -name: %% format_file_name(workflow.file_name, is_title=true) %% +{%- macro format_file_name(file_name, is_title=false, is_subtitle=false) -%} + {%- set formatted_name = file_name | splitext | first -%} + {%- if is_title -%} + {%- set formatted_name = formatted_name | replace('-', ' ') | title | regex_replace('Vm','VM') -%} + {%- endif -%} + {%- if is_subtitle -%} + {%- set formatted_name = formatted_name | replace('-', ' ') | capitalize | regex_replace('vm','VM') -%} + {%- endif -%} + {{ formatted_name }} +{%- endmacro -%} +name: {{ format_file_name(workflow.file_name, is_title=true) }} on: -<%- if workflow.trigger is defined +%> - <%- for trigger_name in workflow.trigger.keys() +%> - <%- if trigger_name == 'schedule' +%> +{%- if workflow.trigger is defined +%} + {%- for trigger_name in workflow.trigger.keys() +%} + {%- if trigger_name == 'schedule' +%} schedule: - - cron: '%% workflow.trigger['schedule']['cron'] %%' - <%- elif trigger_name == 'workflow_dispatch' +%> + - cron: '{{ workflow.trigger['schedule']['cron'] }}' + {%- elif trigger_name == 'workflow_dispatch' +%} workflow_dispatch: - <%- if workflow.trigger['workflow_dispatch'] is not none +%> + {%- if workflow.trigger['workflow_dispatch'] is not none +%} inputs: - %% workflow.trigger['workflow_dispatch'] | flatten | join('') | indent(6) | trim %% - <%- if github_environment_selector == 'input' +%> - %% github_kayobe_environment_input | flatten | join('') | indent(6) | trim %% - <%- endif +%> - <%- endif +%> - <%- endif +%> - <%- endfor +%> -<%- endif +%> + {{ workflow.trigger['workflow_dispatch'] | flatten | join('') | indent(6) | trim }} + {%- if github_environment_selector == 'input' +%} + {{ github_kayobe_environment_input | flatten | join('') | indent(6) | trim }} + {%- endif +%} + {%- endif +%} + {%- endif +%} + {%- endfor +%} +{%- endif -%} diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index aa460c8..dc07a68 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -12,28 +12,7 @@ <%- if github_environment_selector == 'single' -%> <%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": github_kayobe_environments[0] }) -%> <%- endif -%> -name: %% format_file_name(workflow.file_name, is_title=true) %% - -on: - workflow_dispatch: - inputs: - test_suite: - description: | - The list of tests to run. This should be a name of file under - .automation.conf/tempest/load-lists. Defaults to running - default test suite. - required: false - default: 'default' - type: choice - options: - %% workflow.test_suites | flatten | join('') | indent(10) | trim %% - tempest_pattern: - description: | - Limit tests to this regex. Takes precedence over testSuite. - <%- if github_environment_selector == 'input' +%> - %% github_kayobe_environment_input | flatten | join('') | indent(6) | trim %% - <%- endif +%> - +%% lookup('template', 'tempest-header.yml.j2') %% jobs: prepare-runner: uses: ./.github/workflows/prepare-runner.yml diff --git a/roles/github/templates/tempest-header.yml.j2 b/roles/github/templates/tempest-header.yml.j2 new file mode 100644 index 0000000..16d9a87 --- /dev/null +++ b/roles/github/templates/tempest-header.yml.j2 @@ -0,0 +1,31 @@ +{%- macro format_file_name(file_name, is_title=false, is_subtitle=false) -%} + {%- set formatted_name = file_name | splitext | first -%} + {%- if is_title -%} + {%- set formatted_name = formatted_name | replace('-', ' ') | title | regex_replace('Vm','VM') -%} + {%- endif -%} + {%- if is_subtitle -%} + {%- set formatted_name = formatted_name | replace('-', ' ') | capitalize | regex_replace('vm','VM') -%} + {%- endif -%} + {{ formatted_name }} +{%- endmacro -%} +name: {{ format_file_name(workflow.file_name, is_title=true) }} + +on: + workflow_dispatch: + inputs: + test_suite: + description: | + The list of tests to run. This should be a name of file under + .automation.conf/tempest/load-lists. Defaults to running + default test suite. + required: false + default: 'default' + type: choice + options: + {{ workflow.test_suites | flatten | join('') | indent(10) | trim }} + tempest_pattern: + description: | + Limit tests to this regex. Takes precedence over testSuite. + {%- if github_environment_selector == 'input' +%} + {{ github_kayobe_environment_input | flatten | join('') | indent(6) | trim }} + {%- endif +%} From 51dee17f31b71edcdd84fe26b33aef1d9adba9c4 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 24 Oct 2023 12:18:56 +0000 Subject: [PATCH 54/55] feat: minor changes to `README.md` --- roles/github/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/github/README.md b/roles/github/README.md index f09865b..5f25cdf 100644 --- a/roles/github/README.md +++ b/roles/github/README.md @@ -36,9 +36,9 @@ The following variables can be used to make small adjustments to the composition `github_output_directory`: control the location where the workflows shall be written to. -`github_environment_selector`: control the type of environment support the workflows should be generated with. Either `none` for no environment, `single` for fixed environment or `input` whereby the environment is controlled at `workflow_dispatch` +`github_environment_selector`: control the type of environment support the workflows should be generated with. Either `single` for fixed environment or `input` whereby the environment is controlled at `workflow_dispatch`. No environment is the default by setting `github_environment_selector` to no value or `Null`. -`github_kayobe_environments`: list of environments the workflows should target. Only has effect when `github_environment_selector` is not `none`. +`github_kayobe_environments`: list of environments the workflows should target. Only has effect when `github_environment_selector` is not `input` or `single`. `github_runs_on`: control which runner can accept this workflow. See GitHub for more information on [runs-on](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on). @@ -50,11 +50,11 @@ The following variables can be used to make small adjustments to the composition `github_kayobe_base_image`: select the base image used when building the kayobe docker image. Default is `quay.io/centos/centos:stream8` supports OpenStack Wallaby, Xena and Yoga. Zed and higher would require `quay.io/rockylinux/rockylinux:9`. -`github_kayobe_arguments`: a dictionary of arguments that can be used to override the default arguments found within `vars/main.yml`. For example if you wanted to change the value of `KAYOBE_ENVIRONMENT` from its default of `production` you can simply add `KAYOBE_ENVIRONMENT` to this dictionary and it will take precedence over the defaults. +`github_kayobe_arguments`: a dictionary of arguments that can be used to override the default arguments found within `vars/main.yml`. For example if you wanted to change the value of `KAYOBE_AUTOMATION_PR_TITLE` from its default, you can do by simply adding `KAYOBE_AUTOMATION_PR_TITLE` to this dictionary and it will take precedence over the default. `github_*_hook:` see section [Template Hooks](#template-hooks) for information about this variables -`github_buildx_enable`: In some deployments the build kayobe docker image workflow has had difficulties successfully pushing the image to container registries such as Pulp if buildx has been used. It situations where failure to push images is been experienced a user might wish to disable buildx. Buildx is enabled by default. +`github_buildx_enable`: In some deployments the build kayobe docker image workflow has had difficulties successfully pushing the image to container registries such as Pulp if buildx has been used. It situations where failure to push images is been experienced a user might wish to disable buildx. Buildx is disabled by default. `github_buildx_inline_config`: provide configuration parameters to buildx. Useful for connecting to insecure docker registry. @@ -143,7 +143,7 @@ The following example playbook will generate a series of `reference` workflows w - name: Write Kayobe Automation Workflows for GitHub hosts: localhost roles: - - stackhpc.kayobe_automation_workflows.github + - stackhpc.kayobe_workflows.github ``` License From 26870a3e13a27fa28fc8e43c1f51696fe00ae1f6 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 25 Oct 2023 12:37:10 +0100 Subject: [PATCH 55/55] feat: update README.md Co-authored-by: Mark Goddard --- roles/github/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/README.md b/roles/github/README.md index 5f25cdf..90d9292 100644 --- a/roles/github/README.md +++ b/roles/github/README.md @@ -38,7 +38,7 @@ The following variables can be used to make small adjustments to the composition `github_environment_selector`: control the type of environment support the workflows should be generated with. Either `single` for fixed environment or `input` whereby the environment is controlled at `workflow_dispatch`. No environment is the default by setting `github_environment_selector` to no value or `Null`. -`github_kayobe_environments`: list of environments the workflows should target. Only has effect when `github_environment_selector` is not `input` or `single`. +`github_kayobe_environments`: list of environments the workflows should target. Only has effect when `github_environment_selector` is `input` or `single`. `github_runs_on`: control which runner can accept this workflow. See GitHub for more information on [runs-on](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on).