From 5b361ede35eaa7467aec8f5682643897aa5e46a4 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 21 Aug 2023 15:30:29 +0100 Subject: [PATCH 01/42] feat: add support for adjusting kayobe inputs Previously users of the role could not adjust the templates due `github_kayobe_*_input` using YAML anchors. This has now beeen replaced with a scalar block string. --- roles/github/defaults/main.yml | 101 ++++++++++++++++++++------- roles/github/templates/header.yml.j2 | 10 +-- 2 files changed, 78 insertions(+), 33 deletions(-) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index a68cb6e..fabf3b4 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -19,25 +19,26 @@ github_kayobe_hook: "" github_final_hook: "" -github_kayobe_inputs: &github_kayobe_inputs +github_kayobe_limit_input: | kayobeLimit: description: | The ansible limit to use when running kayobe playbooks. + +github_kayobe_tags_input: | kayobeTags: description: | The ansible tags to use when running kayobe playbooks. -github_kolla_inputs: &github_kolla_inputs +github_kolla_limit_input: | kollaLimit: description: | The ansible limit to use for kolla-ansible playbooks. + +github_kolla_tags_input: | kollaTags: description: | The ansible tags to use when running kolla-ansible playbooks. -github_combined_inputs: &github_combined_inputs - <<: [*github_kayobe_inputs, *github_kolla_inputs] - github_workflows: - "{{ github_build_kayobe_image }}" - "{{ github_run_kolla_config_diff }}" @@ -73,7 +74,9 @@ github_run_kolla_config_diff: github_run_infra_vm_host_configure: file_name: run-infra-vm-host-configure.yml trigger: - workflow_dispatch: *github_kayobe_inputs + workflow_dispatch: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" arguments: - KAYOBE_TAGS - KAYOBE_LIMIT @@ -82,7 +85,9 @@ github_run_infra_vm_host_configure: github_run_infra_vm_host_package_update: file_name: run-infra-vm-host-package-update.yml trigger: - workflow_dispatch: *github_kayobe_inputs + workflow_dispatch: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" arguments: - KAYOBE_TAGS - KAYOBE_LIMIT @@ -91,7 +96,9 @@ github_run_infra_vm_host_package_update: github_run_infra_vm_provision: file_name: run-infra-vm-provision.yml trigger: - workflow_dispatch: *github_kayobe_inputs + workflow_dispatch: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" arguments: - KAYOBE_TAGS - KAYOBE_LIMIT @@ -100,7 +107,9 @@ github_run_infra_vm_provision: github_run_infra_vm_service_deploy: file_name: run-infra-vm-service-deploy.yml trigger: - workflow_dispatch: *github_kayobe_inputs + workflow_dispatch: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" arguments: - KAYOBE_TAGS - KAYOBE_LIMIT @@ -109,7 +118,9 @@ github_run_infra_vm_service_deploy: github_run_network_connectivity_check: file_name: run-network-connectivity-check.yml trigger: - workflow_dispatch: *github_kayobe_inputs + workflow_dispatch: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" arguments: - KAYOBE_TAGS - KAYOBE_LIMIT @@ -118,7 +129,11 @@ github_run_network_connectivity_check: github_run_overcloud_container_image_pull: file_name: run-overcloud-container-image-pull.yml trigger: - workflow_dispatch: *github_combined_inputs + workflow_dispatch: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" + - "{{ github_kolla_tags_input }}" + - "{{ github_kolla_limit_input }}" arguments: - KOLLA_TAGS - KOLLA_LIMIT @@ -129,7 +144,11 @@ github_run_overcloud_container_image_pull: github_run_overcloud_database_backup: file_name: run-overcloud-database-backup.yml trigger: - workflow_dispatch: *github_combined_inputs + workflow_dispatch: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" + - "{{ github_kolla_tags_input }}" + - "{{ github_kolla_limit_input }}" arguments: - KOLLA_TAGS - KOLLA_LIMIT @@ -140,7 +159,11 @@ github_run_overcloud_database_backup: github_run_overcloud_host_configure: file_name: run-overcloud-host-configure.yml trigger: - workflow_dispatch: *github_combined_inputs + workflow_dispatch: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" + - "{{ github_kolla_tags_input }}" + - "{{ github_kolla_limit_input }}" arguments: - KOLLA_TAGS - KOLLA_LIMIT @@ -151,7 +174,9 @@ github_run_overcloud_host_configure: github_run_overcloud_host_package_update: file_name: run-overcloud-host-package-update.yml trigger: - workflow_dispatch: *github_kayobe_inputs + workflow_dispatch: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" arguments: - KAYOBE_TAGS - KAYOBE_LIMIT @@ -160,7 +185,9 @@ github_run_overcloud_host_package_update: github_run_overcloud_inventory_discover: file_name: run-overcloud-inventory-discover.yml trigger: - workflow_dispatch: *github_kayobe_inputs + workflow_dispatch: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" arguments: - KAYOBE_TAGS - KAYOBE_LIMIT @@ -176,7 +203,9 @@ github_run_overcloud_inventory_discover: github_run_overcloud_provision: file_name: run-overcloud-provision.yml trigger: - workflow_dispatch: *github_kayobe_inputs + workflow_dispatch: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" arguments: - KAYOBE_TAGS - KAYOBE_LIMIT @@ -185,7 +214,11 @@ github_run_overcloud_provision: github_run_overcloud_service_deploy: file_name: run-overcloud-service-deploy.yml trigger: - workflow_dispatch: *github_combined_inputs + workflow_dispatch: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" + - "{{ github_kolla_tags_input }}" + - "{{ github_kolla_limit_input }}" arguments: - KOLLA_TAGS - KOLLA_LIMIT @@ -203,7 +236,11 @@ github_run_overcloud_service_deploy: github_run_overcloud_service_upgrade: file_name: run-overcloud-service-upgrade.yml trigger: - workflow_dispatch: *github_combined_inputs + workflow_dispatch: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" + - "{{ github_kolla_tags_input }}" + - "{{ github_kolla_limit_input }}" arguments: - KOLLA_TAGS - KOLLA_LIMIT @@ -221,7 +258,11 @@ github_run_overcloud_service_upgrade: github_run_seed_host_configure: file_name: run-seed-host-configure.yml trigger: - workflow_dispatch: *github_combined_inputs + workflow_dispatch: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" + - "{{ github_kolla_tags_input }}" + - "{{ github_kolla_limit_input }}" arguments: - KOLLA_TAGS - KOLLA_LIMIT @@ -232,7 +273,9 @@ github_run_seed_host_configure: github_run_seed_host_package_update: file_name: run-seed-host-package-update.yml trigger: - workflow_dispatch: *github_kayobe_inputs + workflow_dispatch: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" arguments: - KAYOBE_TAGS - KAYOBE_LIMIT @@ -241,7 +284,9 @@ github_run_seed_host_package_update: github_run_seed_hypervisor_host_configure: file_name: run-seed-hypervisor-host-configure.yml trigger: - workflow_dispatch: *github_kayobe_inputs + workflow_dispatch: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" arguments: - KAYOBE_TAGS - KAYOBE_LIMIT @@ -250,7 +295,9 @@ github_run_seed_hypervisor_host_configure: github_run_seed_hypervisor_host_package_update: file_name: run-seed-hypervisor-host-package-update.yml trigger: - workflow_dispatch: *github_kayobe_inputs + workflow_dispatch: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" arguments: - KAYOBE_TAGS - KAYOBE_LIMIT @@ -259,7 +306,11 @@ github_run_seed_hypervisor_host_package_update: github_run_seed_service_deploy: file_name: run-seed-service-deploy.yml trigger: - workflow_dispatch: *github_combined_inputs + workflow_dispatch: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" + - "{{ github_kolla_tags_input }}" + - "{{ github_kolla_limit_input }}" arguments: - KOLLA_TAGS - KOLLA_LIMIT @@ -277,7 +328,9 @@ github_run_seed_service_deploy: github_run_seed_vm_provision: file_name: run-seed-vm-provision.yml trigger: - workflow_dispatch: *github_kayobe_inputs + workflow_dispatch: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" arguments: - KAYOBE_TAGS - KAYOBE_LIMIT diff --git a/roles/github/templates/header.yml.j2 b/roles/github/templates/header.yml.j2 index 8e81f84..a6a54d4 100644 --- a/roles/github/templates/header.yml.j2 +++ b/roles/github/templates/header.yml.j2 @@ -10,15 +10,7 @@ on: workflow_dispatch: <%- if workflow.trigger['workflow_dispatch'] is not none +%> inputs: - <%- for input_name in workflow.trigger['workflow_dispatch'].keys() +%> - %% input_name %%: - <%- for property in workflow.trigger['workflow_dispatch'][input_name] +%> - <%- if property == 'description' +%> - %% property %%: | - %% workflow.trigger['workflow_dispatch'][input_name][property] | trim %% - <%- endif +%> - <%- endfor +%> - <%- endfor +%> + %% workflow.trigger['workflow_dispatch'] | join('') | indent(6) | trim %% <%- endif +%> <%- endif +%> <%- endfor +%> From e559e46613078901c68cdd964e1c3ad07d62458a Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 21 Aug 2023 15:38:33 +0100 Subject: [PATCH 02/42] feat: build image on `workflow_dispatch` --- roles/github/templates/build-kayobe-docker-image.yml.j2 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index 058b025..a17d04a 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -1,9 +1,7 @@ name: %% format_file_name(workflow.file_name, is_title=true) %% on: - push: - tags: - - '*' + workflow_dispatch: env: REGISTRY: ghcr.io From ad5243e7af35001078bb4fd825c28e3229028f05 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 21 Aug 2023 15:45:56 +0100 Subject: [PATCH 03/42] feat: push latest and SHA tagged images --- .../templates/build-kayobe-docker-image.yml.j2 | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index a17d04a..a84acb5 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -4,8 +4,6 @@ on: workflow_dispatch: env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} KAYOBE_USER_UID: 1000 KAYOBE_USER_GID: 1000 @@ -32,14 +30,6 @@ jobs: username: %% github_registry_username %% password: %% github_registry_password %% - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v4 - with: - flavor: | - latest=${{ !contains(github.ref, 'alpha') }} - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 with: @@ -59,8 +49,9 @@ jobs: KAYOBE_USER_UID=${{ env.KAYOBE_USER_UID }} KAYOBE_USER_GID=${{ env.KAYOBE_USER_UID }} push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + tags: | + %% github_registry_url %%/kayobe:latest + %% github_registry_url %%/kayobe:${{ github.sha }} <% if github_final_hook | length >= 1 +%> %% github_final_hook | indent(width=6, first=false) -%% <% endif %> From 561c6ef473e97fbd68e747996a162367206b5be0 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 21 Aug 2023 15:49:13 +0100 Subject: [PATCH 04/42] feat: bump `image` used when building kayobe image --- roles/github/templates/build-kayobe-docker-image.yml.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index a84acb5..a626802 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -10,7 +10,8 @@ env: jobs: %% format_file_name(workflow.file_name) %%: runs-on: %% github_runs_on %% - container: docker:git + container: + image: docker:24.0-git permissions: contents: read packages: write From 851b23f63a9848ea4427c459701b42a36c25c81c Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 21 Aug 2023 16:02:18 +0100 Subject: [PATCH 05/42] feat: add `github_image_tag` variable --- roles/github/defaults/main.yml | 4 +++- roles/github/templates/build-kayobe-docker-image.yml.j2 | 4 ++-- roles/github/templates/generic.yml.j2 | 2 +- roles/github/templates/run-config-diff.yml.j2 | 2 +- roles/github/templates/run-tempest.yml.j2 | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index fabf3b4..b54d889 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -9,7 +9,9 @@ github_registry_username: ${{ github.actor }} github_registry_password: ${{ secrets.GITHUB_TOKEN }} -github_image_name: ${{ github.repository }}:latest +github_image_name: kayobe + +github_image_tag: latest github_kayobe_arguments: {} diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index a626802..f610ef8 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -51,8 +51,8 @@ jobs: KAYOBE_USER_GID=${{ env.KAYOBE_USER_UID }} push: true tags: | - %% github_registry_url %%/kayobe:latest - %% github_registry_url %%/kayobe:${{ github.sha }} + %% github_registry_url %%/%% github_image_name %%:latest + %% github_registry_url %%/%% github_image_name %%:${{ github.sha }} <% if github_final_hook | length >= 1 +%> %% github_final_hook | indent(width=6, first=false) -%% <% endif %> diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 7b9d627..107a56c 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -7,7 +7,7 @@ jobs: contents: read packages: read container: - image: %% github_registry_url %%/%% github_image_name %% + image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% credentials: username: %% github_registry_username %% password: %% github_registry_password %% diff --git a/roles/github/templates/run-config-diff.yml.j2 b/roles/github/templates/run-config-diff.yml.j2 index e8a2d7c..c7aac73 100644 --- a/roles/github/templates/run-config-diff.yml.j2 +++ b/roles/github/templates/run-config-diff.yml.j2 @@ -14,7 +14,7 @@ jobs: contents: read packages: read container: - image: %% github_registry_url %%/%% github_image_name %% + image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% credentials: username: %% github_registry_username %% password: %% github_registry_password %% diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index f5404ef..10546de 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -25,7 +25,7 @@ jobs: contents: read packages: read container: - image: %% github_registry_url %%/%% github_image_name %% + image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% credentials: username: %% github_registry_username %% password: %% github_registry_password %% From 678c7681bc7fac9e29b43be70c1eefc8f482e27f Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 21 Aug 2023 16:17:48 +0100 Subject: [PATCH 06/42] fix: add missing `github_` prefix on kayobe args --- roles/github/templates/generic.yml.j2 | 4 ++-- roles/github/templates/run-config-diff.yml.j2 | 8 ++++---- roles/github/templates/run-tempest.yml.j2 | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 107a56c..30e037b 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -31,9 +31,9 @@ jobs: /src/.automation/pipeline/%% workflow.file_name[4:-4] %%.sh <%- if workflow.arguments is defined +%> env: - KAYOBE_ENVIRONMENT: '%% kayobe_arguments.KAYOBE_ENVIRONMENT | default(default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' + KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' <%- for argument in workflow.arguments +%> - %% argument %%: '%% kayobe_arguments[argument] | default(default_kayobe_arguments[argument]) %%' + %% argument %%: '%% github_kayobe_arguments[argument] | default(default_kayobe_arguments[argument]) %%' <%- endfor +%> <%- endif +%> <% if github_final_hook | length >= 1 +%> diff --git a/roles/github/templates/run-config-diff.yml.j2 b/roles/github/templates/run-config-diff.yml.j2 index c7aac73..ac87332 100644 --- a/roles/github/templates/run-config-diff.yml.j2 +++ b/roles/github/templates/run-config-diff.yml.j2 @@ -41,10 +41,10 @@ jobs: run: | sudo -E -u stack bash -c '/src/.automation/pipeline/config-diff.sh ${{ github.event.pull_request.base.sha }}' env: - KAYOBE_ENVIRONMENT: '%% kayobe_arguments.KAYOBE_ENVIRONMENT | default(default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' - KAYOBE_VAULT_PASSWORD: '%% kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' - HOME: '%% kayobe_arguments.HOME | default(default_kayobe_arguments.HOME) %%' + KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' + KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' + HOME: '%% github_kayobe_arguments.HOME | default(default_kayobe_arguments.HOME) %%' - name: Show summary of changes run: | diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index 10546de..7af1357 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -49,10 +49,10 @@ jobs: source /src/.automation/functions && kayobe_install env: - KAYOBE_ENVIRONMENT: '%% kayobe_arguments.KAYOBE_ENVIRONMENT | default(default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' - KAYOBE_VAULT_PASSWORD: '%% kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' - HOME: '%% kayobe_arguments.HOME | default(default_kayobe_arguments.HOME) %%' + KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' + KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' + HOME: '%% github_kayobe_arguments.HOME | default(default_kayobe_arguments.HOME) %%' USER: stack - name: Symlink kolla into /src/etc From 0a16b2591ed12fe52133e0a50b93a3be5e41f344 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 21 Aug 2023 16:38:24 +0100 Subject: [PATCH 07/42] feat: add missing kayobe arguments Both `KAYOBE_AUTOMATION_SSH_PRIVATE_KEY` and `KAYOBE_VAULT_PASSWORD` were missing from the workflow definitions and thus leading to broken workflows. --- roles/github/defaults/main.yml | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index b54d889..1d9d723 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -80,6 +80,8 @@ github_run_infra_vm_host_configure: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: + - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - HOME @@ -91,6 +93,8 @@ github_run_infra_vm_host_package_update: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: + - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - HOME @@ -102,6 +106,8 @@ github_run_infra_vm_provision: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: + - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - HOME @@ -113,6 +119,8 @@ github_run_infra_vm_service_deploy: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: + - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - HOME @@ -124,6 +132,8 @@ github_run_network_connectivity_check: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: + - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - HOME @@ -137,6 +147,8 @@ github_run_overcloud_container_image_pull: - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" arguments: + - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + - KAYOBE_VAULT_PASSWORD - KOLLA_TAGS - KOLLA_LIMIT - KAYOBE_TAGS @@ -152,6 +164,8 @@ github_run_overcloud_database_backup: - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" arguments: + - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + - KAYOBE_VAULT_PASSWORD - KOLLA_TAGS - KOLLA_LIMIT - KAYOBE_TAGS @@ -167,6 +181,8 @@ github_run_overcloud_host_configure: - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" arguments: + - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + - KAYOBE_VAULT_PASSWORD - KOLLA_TAGS - KOLLA_LIMIT - KAYOBE_TAGS @@ -180,6 +196,8 @@ github_run_overcloud_host_package_update: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: + - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - HOME @@ -191,6 +209,8 @@ github_run_overcloud_inventory_discover: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: + - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - KAYOBE_AUTOMATION_PR_TARGET_BRANCH @@ -209,6 +229,8 @@ github_run_overcloud_provision: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: + - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - HOME @@ -222,6 +244,8 @@ github_run_overcloud_service_deploy: - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" arguments: + - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + - KAYOBE_VAULT_PASSWORD - KOLLA_TAGS - KOLLA_LIMIT - KAYOBE_TAGS @@ -244,6 +268,8 @@ github_run_overcloud_service_upgrade: - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" arguments: + - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + - KAYOBE_VAULT_PASSWORD - KOLLA_TAGS - KOLLA_LIMIT - KAYOBE_TAGS @@ -266,6 +292,8 @@ github_run_seed_host_configure: - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" arguments: + - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + - KAYOBE_VAULT_PASSWORD - KOLLA_TAGS - KOLLA_LIMIT - KAYOBE_TAGS @@ -279,6 +307,8 @@ github_run_seed_host_package_update: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: + - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - HOME @@ -290,6 +320,8 @@ github_run_seed_hypervisor_host_configure: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: + - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - HOME @@ -301,6 +333,8 @@ github_run_seed_hypervisor_host_package_update: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: + - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - HOME @@ -314,6 +348,8 @@ github_run_seed_service_deploy: - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" arguments: + - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + - KAYOBE_VAULT_PASSWORD - KOLLA_TAGS - KOLLA_LIMIT - KAYOBE_TAGS @@ -334,6 +370,8 @@ github_run_seed_vm_provision: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: + - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - HOME From 6e2a105ce8f62b4bd51025bb00fcf5e550fae433 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 23 Aug 2023 11:02:58 +0100 Subject: [PATCH 08/42] feat: add support for inline buildx config --- roles/github/README.md | 2 -- roles/github/defaults/main.yml | 2 ++ roles/github/templates/build-kayobe-docker-image.yml.j2 | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/roles/github/README.md b/roles/github/README.md index 988ee89..0177c46 100644 --- a/roles/github/README.md +++ b/roles/github/README.md @@ -80,8 +80,6 @@ github_build_kayobe_image: path_override: "{{ playbook_dir }}/templates/build-kayobe-docker-image.yml.j2" ``` - - Template Hooks -------------- diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 1d9d723..daf4f31 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -21,6 +21,8 @@ github_kayobe_hook: "" github_final_hook: "" +github_buildx_inline_config: "" + github_kayobe_limit_input: | kayobeLimit: description: | diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index f610ef8..4f12780 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -34,8 +34,12 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 with: - driver-opts: | - image=moby/buildkit:master + driver-opts: | + image=moby/buildkit:master +<%if github_buildx_inline_config | length >= 1 %> + config-inline: | + %% github_buildx_inline_config | indent(12) | trim %% +<% endif %> <% if github_kayobe_hook | length >= 1 %> %% github_kayobe_hook | indent(width=6, first=false) %% From 69a122a7c82701c8b0593eca99bec574aa3bca2e Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 24 Aug 2023 14:51:34 +0100 Subject: [PATCH 09/42] feat: remove redundant `arguments` --- roles/github/defaults/main.yml | 57 --------------------------- roles/github/templates/generic.yml.j2 | 3 ++ 2 files changed, 3 insertions(+), 57 deletions(-) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index daf4f31..4c36404 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -82,11 +82,8 @@ github_run_infra_vm_host_configure: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: - - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - - HOME github_run_infra_vm_host_package_update: file_name: run-infra-vm-host-package-update.yml @@ -95,11 +92,8 @@ github_run_infra_vm_host_package_update: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: - - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - - HOME github_run_infra_vm_provision: file_name: run-infra-vm-provision.yml @@ -108,11 +102,8 @@ github_run_infra_vm_provision: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: - - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - - HOME github_run_infra_vm_service_deploy: file_name: run-infra-vm-service-deploy.yml @@ -121,11 +112,8 @@ github_run_infra_vm_service_deploy: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: - - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - - HOME github_run_network_connectivity_check: file_name: run-network-connectivity-check.yml @@ -134,11 +122,8 @@ github_run_network_connectivity_check: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: - - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - - HOME github_run_overcloud_container_image_pull: file_name: run-overcloud-container-image-pull.yml @@ -149,13 +134,10 @@ github_run_overcloud_container_image_pull: - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" arguments: - - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - - KAYOBE_VAULT_PASSWORD - KOLLA_TAGS - KOLLA_LIMIT - KAYOBE_TAGS - KAYOBE_LIMIT - - HOME github_run_overcloud_database_backup: file_name: run-overcloud-database-backup.yml @@ -166,13 +148,10 @@ github_run_overcloud_database_backup: - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" arguments: - - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - - KAYOBE_VAULT_PASSWORD - KOLLA_TAGS - KOLLA_LIMIT - KAYOBE_TAGS - KAYOBE_LIMIT - - HOME github_run_overcloud_host_configure: file_name: run-overcloud-host-configure.yml @@ -183,13 +162,10 @@ github_run_overcloud_host_configure: - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" arguments: - - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - - KAYOBE_VAULT_PASSWORD - KOLLA_TAGS - KOLLA_LIMIT - KAYOBE_TAGS - KAYOBE_LIMIT - - HOME github_run_overcloud_host_package_update: file_name: run-overcloud-host-package-update.yml @@ -198,11 +174,8 @@ github_run_overcloud_host_package_update: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: - - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - - HOME github_run_overcloud_inventory_discover: file_name: run-overcloud-inventory-discover.yml @@ -211,8 +184,6 @@ github_run_overcloud_inventory_discover: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: - - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - KAYOBE_AUTOMATION_PR_TARGET_BRANCH @@ -222,7 +193,6 @@ github_run_overcloud_inventory_discover: - KAYOBE_AUTOMATION_PR_TYPE - KAYOBE_AUTOMATION_PR_TITLE - KAYOBE_AUTOMATION_PR_URL - - HOME github_run_overcloud_provision: file_name: run-overcloud-provision.yml @@ -231,11 +201,8 @@ github_run_overcloud_provision: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: - - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - - HOME github_run_overcloud_service_deploy: file_name: run-overcloud-service-deploy.yml @@ -246,8 +213,6 @@ github_run_overcloud_service_deploy: - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" arguments: - - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - - KAYOBE_VAULT_PASSWORD - KOLLA_TAGS - KOLLA_LIMIT - KAYOBE_TAGS @@ -259,7 +224,6 @@ github_run_overcloud_service_deploy: - KAYOBE_AUTOMATION_PR_TYPE - KAYOBE_AUTOMATION_PR_TITLE - KAYOBE_AUTOMATION_PR_URL - - HOME github_run_overcloud_service_upgrade: file_name: run-overcloud-service-upgrade.yml @@ -270,8 +234,6 @@ github_run_overcloud_service_upgrade: - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" arguments: - - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - - KAYOBE_VAULT_PASSWORD - KOLLA_TAGS - KOLLA_LIMIT - KAYOBE_TAGS @@ -283,7 +245,6 @@ github_run_overcloud_service_upgrade: - KAYOBE_AUTOMATION_PR_TYPE - KAYOBE_AUTOMATION_PR_TITLE - KAYOBE_AUTOMATION_PR_URL - - HOME github_run_seed_host_configure: file_name: run-seed-host-configure.yml @@ -294,13 +255,10 @@ github_run_seed_host_configure: - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" arguments: - - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - - KAYOBE_VAULT_PASSWORD - KOLLA_TAGS - KOLLA_LIMIT - KAYOBE_TAGS - KAYOBE_LIMIT - - HOME github_run_seed_host_package_update: file_name: run-seed-host-package-update.yml @@ -309,11 +267,8 @@ github_run_seed_host_package_update: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: - - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - - HOME github_run_seed_hypervisor_host_configure: file_name: run-seed-hypervisor-host-configure.yml @@ -322,11 +277,8 @@ github_run_seed_hypervisor_host_configure: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: - - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - - HOME github_run_seed_hypervisor_host_package_update: file_name: run-seed-hypervisor-host-package-update.yml @@ -335,11 +287,8 @@ github_run_seed_hypervisor_host_package_update: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: - - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - - HOME github_run_seed_service_deploy: file_name: run-seed-service-deploy.yml @@ -350,8 +299,6 @@ github_run_seed_service_deploy: - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" arguments: - - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - - KAYOBE_VAULT_PASSWORD - KOLLA_TAGS - KOLLA_LIMIT - KAYOBE_TAGS @@ -363,7 +310,6 @@ github_run_seed_service_deploy: - KAYOBE_AUTOMATION_PR_TYPE - KAYOBE_AUTOMATION_PR_TITLE - KAYOBE_AUTOMATION_PR_URL - - HOME github_run_seed_vm_provision: file_name: run-seed-vm-provision.yml @@ -372,11 +318,8 @@ github_run_seed_vm_provision: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" arguments: - - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - - KAYOBE_VAULT_PASSWORD - KAYOBE_TAGS - KAYOBE_LIMIT - - HOME github_run_tempest: file_name: run-tempest.yml diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 30e037b..2a0e260 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -32,9 +32,12 @@ jobs: <%- if workflow.arguments is defined +%> env: KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' + KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' <%- for argument in workflow.arguments +%> %% argument %%: '%% github_kayobe_arguments[argument] | default(default_kayobe_arguments[argument]) %%' <%- endfor +%> + HOME: '%% github_kayobe_arguments.HOME | default(default_kayobe_arguments.HOME) %%' <%- endif +%> <% if github_final_hook | length >= 1 +%> %% github_final_hook | indent(width=6, first=false) -%% From 8299d847b930ad858e33c018c4486ae2daa75ce1 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 24 Aug 2023 15:56:20 +0100 Subject: [PATCH 10/42] feat: reduce duplication `env` arguments --- roles/github/defaults/main.yml | 118 +++++--------------------- roles/github/templates/generic.yml.j2 | 2 +- roles/github/vars/main.yml | 17 ++++ 3 files changed, 37 insertions(+), 100 deletions(-) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 4c36404..7efebae 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -81,9 +81,7 @@ github_run_infra_vm_host_configure: workflow_dispatch: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" - arguments: - - KAYOBE_TAGS - - KAYOBE_LIMIT + arguments: "{{ github_kayobe_task_arguments }}" github_run_infra_vm_host_package_update: file_name: run-infra-vm-host-package-update.yml @@ -91,9 +89,7 @@ github_run_infra_vm_host_package_update: workflow_dispatch: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" - arguments: - - KAYOBE_TAGS - - KAYOBE_LIMIT + arguments: "{{ github_kayobe_task_arguments }}" github_run_infra_vm_provision: file_name: run-infra-vm-provision.yml @@ -101,9 +97,7 @@ github_run_infra_vm_provision: workflow_dispatch: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" - arguments: - - KAYOBE_TAGS - - KAYOBE_LIMIT + arguments: "{{ github_kayobe_task_arguments }}" github_run_infra_vm_service_deploy: file_name: run-infra-vm-service-deploy.yml @@ -111,9 +105,7 @@ github_run_infra_vm_service_deploy: workflow_dispatch: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" - arguments: - - KAYOBE_TAGS - - KAYOBE_LIMIT + arguments: "{{ github_kayobe_task_arguments }}" github_run_network_connectivity_check: file_name: run-network-connectivity-check.yml @@ -121,9 +113,7 @@ github_run_network_connectivity_check: workflow_dispatch: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" - arguments: - - KAYOBE_TAGS - - KAYOBE_LIMIT + arguments: "{{ github_kayobe_task_arguments }}" github_run_overcloud_container_image_pull: file_name: run-overcloud-container-image-pull.yml @@ -133,11 +123,7 @@ github_run_overcloud_container_image_pull: - "{{ github_kayobe_tags_input }}" - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" - arguments: - - KOLLA_TAGS - - KOLLA_LIMIT - - KAYOBE_TAGS - - KAYOBE_LIMIT + arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}"] github_run_overcloud_database_backup: file_name: run-overcloud-database-backup.yml @@ -147,11 +133,7 @@ github_run_overcloud_database_backup: - "{{ github_kayobe_tags_input }}" - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" - arguments: - - KOLLA_TAGS - - KOLLA_LIMIT - - KAYOBE_TAGS - - KAYOBE_LIMIT + arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}"] github_run_overcloud_host_configure: file_name: run-overcloud-host-configure.yml @@ -161,11 +143,7 @@ github_run_overcloud_host_configure: - "{{ github_kayobe_tags_input }}" - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" - arguments: - - KOLLA_TAGS - - KOLLA_LIMIT - - KAYOBE_TAGS - - KAYOBE_LIMIT + arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}"] github_run_overcloud_host_package_update: file_name: run-overcloud-host-package-update.yml @@ -173,9 +151,7 @@ github_run_overcloud_host_package_update: workflow_dispatch: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" - arguments: - - KAYOBE_TAGS - - KAYOBE_LIMIT + arguments: "{{ github_kayobe_task_arguments }}" github_run_overcloud_inventory_discover: file_name: run-overcloud-inventory-discover.yml @@ -183,16 +159,7 @@ github_run_overcloud_inventory_discover: workflow_dispatch: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" - arguments: - - KAYOBE_TAGS - - KAYOBE_LIMIT - - KAYOBE_AUTOMATION_PR_TARGET_BRANCH - - KAYOBE_AUTOMATION_PR_REMOTE - - KAYOBE_AUTOMATION_PR_GITHUB_USER - - KAYOBE_AUTOMATION_PR_AUTH_TOKEN - - KAYOBE_AUTOMATION_PR_TYPE - - KAYOBE_AUTOMATION_PR_TITLE - - KAYOBE_AUTOMATION_PR_URL + arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kayobe_pull_request_arguments }}"] github_run_overcloud_provision: file_name: run-overcloud-provision.yml @@ -200,9 +167,7 @@ github_run_overcloud_provision: workflow_dispatch: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" - arguments: - - KAYOBE_TAGS - - KAYOBE_LIMIT + arguments: "{{ github_kayobe_task_arguments }}" github_run_overcloud_service_deploy: file_name: run-overcloud-service-deploy.yml @@ -212,18 +177,7 @@ github_run_overcloud_service_deploy: - "{{ github_kayobe_tags_input }}" - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" - arguments: - - KOLLA_TAGS - - KOLLA_LIMIT - - KAYOBE_TAGS - - KAYOBE_LIMIT - - KAYOBE_AUTOMATION_PR_TARGET_BRANCH - - KAYOBE_AUTOMATION_PR_REMOTE - - KAYOBE_AUTOMATION_PR_GITHUB_USER - - KAYOBE_AUTOMATION_PR_AUTH_TOKEN - - KAYOBE_AUTOMATION_PR_TYPE - - KAYOBE_AUTOMATION_PR_TITLE - - KAYOBE_AUTOMATION_PR_URL + arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}", "{{ github_kayobe_pull_request_arguments }}"] github_run_overcloud_service_upgrade: file_name: run-overcloud-service-upgrade.yml @@ -233,18 +187,7 @@ github_run_overcloud_service_upgrade: - "{{ github_kayobe_tags_input }}" - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" - arguments: - - KOLLA_TAGS - - KOLLA_LIMIT - - KAYOBE_TAGS - - KAYOBE_LIMIT - - KAYOBE_AUTOMATION_PR_TARGET_BRANCH - - KAYOBE_AUTOMATION_PR_REMOTE - - KAYOBE_AUTOMATION_PR_GITHUB_USER - - KAYOBE_AUTOMATION_PR_AUTH_TOKEN - - KAYOBE_AUTOMATION_PR_TYPE - - KAYOBE_AUTOMATION_PR_TITLE - - KAYOBE_AUTOMATION_PR_URL + arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}", "{{ github_kayobe_pull_request_arguments }}"] github_run_seed_host_configure: file_name: run-seed-host-configure.yml @@ -254,11 +197,7 @@ github_run_seed_host_configure: - "{{ github_kayobe_tags_input }}" - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" - arguments: - - KOLLA_TAGS - - KOLLA_LIMIT - - KAYOBE_TAGS - - KAYOBE_LIMIT + arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}"] github_run_seed_host_package_update: file_name: run-seed-host-package-update.yml @@ -266,9 +205,7 @@ github_run_seed_host_package_update: workflow_dispatch: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" - arguments: - - KAYOBE_TAGS - - KAYOBE_LIMIT + arguments: "{{ github_kayobe_task_arguments }}" github_run_seed_hypervisor_host_configure: file_name: run-seed-hypervisor-host-configure.yml @@ -276,9 +213,7 @@ github_run_seed_hypervisor_host_configure: workflow_dispatch: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" - arguments: - - KAYOBE_TAGS - - KAYOBE_LIMIT + arguments: "{{ github_kayobe_task_arguments }}" github_run_seed_hypervisor_host_package_update: file_name: run-seed-hypervisor-host-package-update.yml @@ -286,9 +221,7 @@ github_run_seed_hypervisor_host_package_update: workflow_dispatch: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" - arguments: - - KAYOBE_TAGS - - KAYOBE_LIMIT + arguments: "{{ github_kayobe_task_arguments }}" github_run_seed_service_deploy: file_name: run-seed-service-deploy.yml @@ -298,18 +231,7 @@ github_run_seed_service_deploy: - "{{ github_kayobe_tags_input }}" - "{{ github_kolla_tags_input }}" - "{{ github_kolla_limit_input }}" - arguments: - - KOLLA_TAGS - - KOLLA_LIMIT - - KAYOBE_TAGS - - KAYOBE_LIMIT - - KAYOBE_AUTOMATION_PR_TARGET_BRANCH - - KAYOBE_AUTOMATION_PR_REMOTE - - KAYOBE_AUTOMATION_PR_GITHUB_USER - - KAYOBE_AUTOMATION_PR_AUTH_TOKEN - - KAYOBE_AUTOMATION_PR_TYPE - - KAYOBE_AUTOMATION_PR_TITLE - - KAYOBE_AUTOMATION_PR_URL + arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}", "{{ github_kayobe_pull_request_arguments }}"] github_run_seed_vm_provision: file_name: run-seed-vm-provision.yml @@ -317,9 +239,7 @@ github_run_seed_vm_provision: workflow_dispatch: - "{{ github_kayobe_limit_input }}" - "{{ github_kayobe_tags_input }}" - arguments: - - KAYOBE_TAGS - - KAYOBE_LIMIT + arguments: "{{ github_kayobe_task_arguments }}" github_run_tempest: file_name: run-tempest.yml diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 2a0e260..12056ae 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -34,7 +34,7 @@ jobs: KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' - <%- for argument in workflow.arguments +%> + <%- for argument in workflow.arguments | flatten +%> %% argument %%: '%% github_kayobe_arguments[argument] | default(default_kayobe_arguments[argument]) %%' <%- endfor +%> HOME: '%% github_kayobe_arguments.HOME | default(default_kayobe_arguments.HOME) %%' diff --git a/roles/github/vars/main.yml b/roles/github/vars/main.yml index cade44b..774d01c 100644 --- a/roles/github/vars/main.yml +++ b/roles/github/vars/main.yml @@ -20,3 +20,20 @@ default_kayobe_arguments: KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_USERNAME: ${{ github.actor }} KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} HOME: /stack + +github_kayobe_task_arguments: + - KAYOBE_TAGS + - KAYOBE_LIMIT + +github_kolla_task_arguments: + - KOLLA_TAGS + - KOLLA_LIMIT + +github_kayobe_pull_request_arguments: + - KAYOBE_AUTOMATION_PR_TARGET_BRANCH + - KAYOBE_AUTOMATION_PR_REMOTE + - KAYOBE_AUTOMATION_PR_GITHUB_USER + - KAYOBE_AUTOMATION_PR_AUTH_TOKEN + - KAYOBE_AUTOMATION_PR_TYPE + - KAYOBE_AUTOMATION_PR_TITLE + - KAYOBE_AUTOMATION_PR_URL \ No newline at end of file From 419fbd19c8ba32650981bf789bba938e98e9022e Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 24 Aug 2023 16:17:02 +0100 Subject: [PATCH 11/42] feat: reduce excess `workflow_dispatch` arguments --- roles/github/defaults/main.yml | 90 ++++++---------------------- roles/github/templates/header.yml.j2 | 2 +- roles/github/vars/main.yml | 8 +++ 3 files changed, 28 insertions(+), 72 deletions(-) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 7efebae..5b8e539 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -78,167 +78,115 @@ github_run_kolla_config_diff: github_run_infra_vm_host_configure: file_name: run-infra-vm-host-configure.yml trigger: - workflow_dispatch: - - "{{ github_kayobe_limit_input }}" - - "{{ github_kayobe_tags_input }}" + workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" github_run_infra_vm_host_package_update: file_name: run-infra-vm-host-package-update.yml trigger: - workflow_dispatch: - - "{{ github_kayobe_limit_input }}" - - "{{ github_kayobe_tags_input }}" + workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" github_run_infra_vm_provision: file_name: run-infra-vm-provision.yml trigger: - workflow_dispatch: - - "{{ github_kayobe_limit_input }}" - - "{{ github_kayobe_tags_input }}" + workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" github_run_infra_vm_service_deploy: file_name: run-infra-vm-service-deploy.yml trigger: - workflow_dispatch: - - "{{ github_kayobe_limit_input }}" - - "{{ github_kayobe_tags_input }}" + workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" github_run_network_connectivity_check: file_name: run-network-connectivity-check.yml trigger: - workflow_dispatch: - - "{{ github_kayobe_limit_input }}" - - "{{ github_kayobe_tags_input }}" + workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" github_run_overcloud_container_image_pull: file_name: run-overcloud-container-image-pull.yml trigger: - workflow_dispatch: - - "{{ github_kayobe_limit_input }}" - - "{{ github_kayobe_tags_input }}" - - "{{ github_kolla_tags_input }}" - - "{{ github_kolla_limit_input }}" + workflow_dispatch: ["{{ github_kolla_dispatch_inputs }}", "{{ github_kayobe_dispatch_inputs }}"] arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}"] github_run_overcloud_database_backup: file_name: run-overcloud-database-backup.yml trigger: - workflow_dispatch: - - "{{ github_kayobe_limit_input }}" - - "{{ github_kayobe_tags_input }}" - - "{{ github_kolla_tags_input }}" - - "{{ github_kolla_limit_input }}" + workflow_dispatch: ["{{ github_kolla_dispatch_inputs }}", "{{ github_kayobe_dispatch_inputs }}"] arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}"] github_run_overcloud_host_configure: file_name: run-overcloud-host-configure.yml trigger: - workflow_dispatch: - - "{{ github_kayobe_limit_input }}" - - "{{ github_kayobe_tags_input }}" - - "{{ github_kolla_tags_input }}" - - "{{ github_kolla_limit_input }}" + workflow_dispatch: ["{{ github_kolla_dispatch_inputs }}", "{{ github_kayobe_dispatch_inputs }}"] arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}"] github_run_overcloud_host_package_update: file_name: run-overcloud-host-package-update.yml trigger: - workflow_dispatch: - - "{{ github_kayobe_limit_input }}" - - "{{ github_kayobe_tags_input }}" + workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" github_run_overcloud_inventory_discover: file_name: run-overcloud-inventory-discover.yml trigger: - workflow_dispatch: - - "{{ github_kayobe_limit_input }}" - - "{{ github_kayobe_tags_input }}" + workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kayobe_pull_request_arguments }}"] github_run_overcloud_provision: file_name: run-overcloud-provision.yml trigger: - workflow_dispatch: - - "{{ github_kayobe_limit_input }}" - - "{{ github_kayobe_tags_input }}" + workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" github_run_overcloud_service_deploy: file_name: run-overcloud-service-deploy.yml trigger: - workflow_dispatch: - - "{{ github_kayobe_limit_input }}" - - "{{ github_kayobe_tags_input }}" - - "{{ github_kolla_tags_input }}" - - "{{ github_kolla_limit_input }}" + workflow_dispatch: ["{{ github_kolla_dispatch_inputs }}", "{{ github_kayobe_dispatch_inputs }}"] arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}", "{{ github_kayobe_pull_request_arguments }}"] github_run_overcloud_service_upgrade: file_name: run-overcloud-service-upgrade.yml trigger: - workflow_dispatch: - - "{{ github_kayobe_limit_input }}" - - "{{ github_kayobe_tags_input }}" - - "{{ github_kolla_tags_input }}" - - "{{ github_kolla_limit_input }}" + workflow_dispatch: ["{{ github_kolla_dispatch_inputs }}", "{{ github_kayobe_dispatch_inputs }}"] arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}", "{{ github_kayobe_pull_request_arguments }}"] github_run_seed_host_configure: file_name: run-seed-host-configure.yml trigger: - workflow_dispatch: - - "{{ github_kayobe_limit_input }}" - - "{{ github_kayobe_tags_input }}" - - "{{ github_kolla_tags_input }}" - - "{{ github_kolla_limit_input }}" + workflow_dispatch: ["{{ github_kolla_dispatch_inputs }}", "{{ github_kayobe_dispatch_inputs }}"] arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}"] github_run_seed_host_package_update: file_name: run-seed-host-package-update.yml trigger: - workflow_dispatch: - - "{{ github_kayobe_limit_input }}" - - "{{ github_kayobe_tags_input }}" + workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" github_run_seed_hypervisor_host_configure: file_name: run-seed-hypervisor-host-configure.yml trigger: - workflow_dispatch: - - "{{ github_kayobe_limit_input }}" - - "{{ github_kayobe_tags_input }}" + workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" github_run_seed_hypervisor_host_package_update: file_name: run-seed-hypervisor-host-package-update.yml trigger: - workflow_dispatch: - - "{{ github_kayobe_limit_input }}" - - "{{ github_kayobe_tags_input }}" + workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" github_run_seed_service_deploy: file_name: run-seed-service-deploy.yml trigger: - workflow_dispatch: - - "{{ github_kayobe_limit_input }}" - - "{{ github_kayobe_tags_input }}" - - "{{ github_kolla_tags_input }}" - - "{{ github_kolla_limit_input }}" + workflow_dispatch: ["{{ github_kolla_dispatch_inputs }}", "{{ github_kayobe_dispatch_inputs }}"] arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}", "{{ github_kayobe_pull_request_arguments }}"] github_run_seed_vm_provision: file_name: run-seed-vm-provision.yml trigger: - workflow_dispatch: - - "{{ github_kayobe_limit_input }}" - - "{{ github_kayobe_tags_input }}" + workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" github_run_tempest: diff --git a/roles/github/templates/header.yml.j2 b/roles/github/templates/header.yml.j2 index a6a54d4..966135d 100644 --- a/roles/github/templates/header.yml.j2 +++ b/roles/github/templates/header.yml.j2 @@ -10,7 +10,7 @@ on: workflow_dispatch: <%- if workflow.trigger['workflow_dispatch'] is not none +%> inputs: - %% workflow.trigger['workflow_dispatch'] | join('') | indent(6) | trim %% + %% workflow.trigger['workflow_dispatch'] | flatten | join('') | indent(6) | trim %% <%- endif +%> <%- endif +%> <%- endfor +%> diff --git a/roles/github/vars/main.yml b/roles/github/vars/main.yml index 774d01c..0579056 100644 --- a/roles/github/vars/main.yml +++ b/roles/github/vars/main.yml @@ -21,6 +21,14 @@ default_kayobe_arguments: KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} HOME: /stack +github_kayobe_dispatch_inputs: + - "{{ github_kayobe_limit_input }}" + - "{{ github_kayobe_tags_input }}" + +github_kolla_dispatch_inputs: + - "{{ github_kolla_tags_input }}" + - "{{ github_kolla_limit_input }}" + github_kayobe_task_arguments: - KAYOBE_TAGS - KAYOBE_LIMIT From ae573f863bb127ade405af50c6138c24d951f63c Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 24 Aug 2023 16:24:12 +0100 Subject: [PATCH 12/42] fix: set `RALLY_DOCKER_` using existing vars --- roles/github/templates/run-tempest.yml.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index 7af1357..5d645d2 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -69,9 +69,9 @@ jobs: KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' KAYOBE_AUTOMATION_TEMPEST_LOADLIST: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_TEMPEST_LOADLIST | default(default_kayobe_arguments.KAYOBE_AUTOMATION_TEMPEST_LOADLIST) %%' TEMPEST_PATTERN: '%% github_kayobe_arguments.TEMPEST_PATTERN | default(default_kayobe_arguments.TEMPEST_PATTERN) %%' - KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY | default(default_kayobe_arguments.KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY) %%' - KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_USERNAME: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_USERNAME | default(default_kayobe_arguments.KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_USERNAME) %%' - KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_PASSWORD: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_PASSWORD | default(default_kayobe_arguments.KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_PASSWORD) %%' + KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY: '%% github_registry_url %%' + KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_USERNAME: '%% github_registry_username %%' + KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_PASSWORD: '%% github_registry_password %%' HOME: '%% github_kayobe_arguments.HOME | default(default_kayobe_arguments.HOME) %%' - name: Print stdout From c40c184c3fb172a75099f8849d4cbce0f6f1c92a Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 24 Aug 2023 16:27:42 +0100 Subject: [PATCH 13/42] fix: add missing newline --- roles/github/vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/vars/main.yml b/roles/github/vars/main.yml index 0579056..8833a65 100644 --- a/roles/github/vars/main.yml +++ b/roles/github/vars/main.yml @@ -44,4 +44,4 @@ github_kayobe_pull_request_arguments: - KAYOBE_AUTOMATION_PR_AUTH_TOKEN - KAYOBE_AUTOMATION_PR_TYPE - KAYOBE_AUTOMATION_PR_TITLE - - KAYOBE_AUTOMATION_PR_URL \ No newline at end of file + - KAYOBE_AUTOMATION_PR_URL From 3a541ae762b084bd7f0d5366bb99a98933c057d0 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 24 Aug 2023 16:38:48 +0100 Subject: [PATCH 14/42] fix: set permissions to write for `packages` --- roles/github/templates/run-tempest.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index 5d645d2..299d054 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -23,7 +23,7 @@ jobs: runs-on: %% github_runs_on %% permissions: contents: read - packages: read + packages: write container: image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% credentials: From 73a17b19a1acc7d4e5789d412c9eadc3e03d6120 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 24 Aug 2023 16:45:37 +0100 Subject: [PATCH 15/42] feat: add support for defining `timeout-minutes` --- roles/github/defaults/main.yml | 2 ++ roles/github/templates/generic.yml.j2 | 1 + roles/github/templates/run-tempest.yml.j2 | 1 + 3 files changed, 4 insertions(+) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 5b8e539..cd4a2cb 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -23,6 +23,8 @@ github_final_hook: "" github_buildx_inline_config: "" +github_timeout: 360 + github_kayobe_limit_input: | kayobeLimit: description: | diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 12056ae..a3a083c 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -11,6 +11,7 @@ jobs: credentials: username: %% github_registry_username %% password: %% github_registry_password %% + timeout-minutes: %% github_timeout %% steps: <% if github_checkout_hook | length >= 1 %> %% github_checkout_hook | indent(width=6, first=false) %% diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index 299d054..22d0a92 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -29,6 +29,7 @@ jobs: credentials: username: %% github_registry_username %% password: %% github_registry_password %% + timeout-minutes: %% github_timeout %% steps: <% if github_checkout_hook | length >= 1 %> %% github_checkout_hook | indent(width=6, first=false) %% From 75581e87bea01170fbcd5c8b7fef0b640abd80f0 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 24 Aug 2023 17:40:15 +0100 Subject: [PATCH 16/42] feat: add workflow to test collection --- .github/workflows/test-collection.yml | 25 +++++++++++++++++++++++++ roles/github/tests/inventory | 1 - tests/roles | 1 + {roles/github/tests => tests}/test.yml | 3 +-- 4 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/test-collection.yml delete mode 100644 roles/github/tests/inventory create mode 120000 tests/roles rename {roles/github/tests => tests}/test.yml (50%) diff --git a/.github/workflows/test-collection.yml b/.github/workflows/test-collection.yml new file mode 100644 index 0000000..167f8f7 --- /dev/null +++ b/.github/workflows/test-collection.yml @@ -0,0 +1,25 @@ +--- +name: Test stackhpc.kayobe_workflows collection +'on': + pull_request: + +jobs: + test: + name: Test github role + runs-on: ubuntu-latest + steps: + - name: Check out the codebase. + uses: actions/checkout@v3 + + - name: Set up Python 3. + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Install Ansible + run: pip3 install ansible + + - name: Test the playbook. + run: ansible-playbook tests/test.yml + env: + ANSIBLE_FORCE_COLOR: '1' diff --git a/roles/github/tests/inventory b/roles/github/tests/inventory deleted file mode 100644 index 2fbb50c..0000000 --- a/roles/github/tests/inventory +++ /dev/null @@ -1 +0,0 @@ -localhost diff --git a/tests/roles b/tests/roles new file mode 120000 index 0000000..d8c4472 --- /dev/null +++ b/tests/roles @@ -0,0 +1 @@ +../roles \ No newline at end of file diff --git a/roles/github/tests/test.yml b/tests/test.yml similarity index 50% rename from roles/github/tests/test.yml rename to tests/test.yml index c8acd22..59fe65b 100644 --- a/roles/github/tests/test.yml +++ b/tests/test.yml @@ -1,6 +1,5 @@ --- -- name: Test github role +- name: Test `stackhpc.kayobe_automation` hosts: localhost - remote_user: root roles: - github From fc24b8724a7bce103447caaabf2bc73e294f7f8d Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 24 Aug 2023 22:26:29 +0100 Subject: [PATCH 17/42] fix: add `!unsafe` on vars to avoid eval Newer versions than `Ansible 7.7.0` attempt to evaluate strings such as `${{ github.actor }}` to avoid this behaviour `!unsafe` is used. --- roles/github/defaults/main.yml | 4 ++-- roles/github/vars/main.yml | 32 ++++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index cd4a2cb..04457d6 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -5,9 +5,9 @@ github_runs_on: self-hosted github_registry_url: ghcr.io -github_registry_username: ${{ github.actor }} +github_registry_username: !unsafe "${{ github.actor }}" -github_registry_password: ${{ secrets.GITHUB_TOKEN }} +github_registry_password: !unsafe "${{ secrets.GITHUB_TOKEN }}" github_image_name: kayobe diff --git a/roles/github/vars/main.yml b/roles/github/vars/main.yml index 8833a65..ee5da12 100644 --- a/roles/github/vars/main.yml +++ b/roles/github/vars/main.yml @@ -1,24 +1,24 @@ --- default_kayobe_arguments: KAYOBE_ENVIRONMENT: production - KAYOBE_VAULT_PASSWORD: "${{ secrets.KAYOBE_VAULT_PASSWORD }}" - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: "${{ secrets.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY }}" - KOLLA_LIMIT: "${{ github.event.inputs.kollaLimit }}" - KOLLA_TAGS: "${{ github.event.inputs.kollaTags }}" - KAYOBE_TAGS: "${{ github.event.inputs.kayobeTags }}" - KAYOBE_LIMIT: "${{ github.event.inputs.kayobeLimit }}" - KAYOBE_AUTOMATION_PR_TARGET_BRANCH: ${{ github.event.ref }} - KAYOBE_AUTOMATION_PR_REMOTE: https://${KAYOBE_AUTOMATION_PR_GITHUB_USER}:${KAYOBE_AUTOMATION_PR_AUTH_TOKEN}@github.com/${{ github.repository }} - KAYOBE_AUTOMATION_PR_GITHUB_USER: ${{ github.actor }} - KAYOBE_AUTOMATION_PR_AUTH_TOKEN: ${{ github.token }} + KAYOBE_VAULT_PASSWORD: !unsafe "${{ secrets.KAYOBE_VAULT_PASSWORD }}" + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: !unsafe "${{ secrets.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY }}" + KOLLA_LIMIT: !unsafe "${{ github.event.inputs.kollaLimit }}" + KOLLA_TAGS: !unsafe "${{ github.event.inputs.kollaTags }}" + KAYOBE_TAGS: !unsafe "${{ github.event.inputs.kayobeTags }}" + KAYOBE_LIMIT: !unsafe "${{ github.event.inputs.kayobeLimit }}" + KAYOBE_AUTOMATION_PR_TARGET_BRANCH: !unsafe ${{ github.event.ref }} + KAYOBE_AUTOMATION_PR_REMOTE: !unsafe https://${KAYOBE_AUTOMATION_PR_GITHUB_USER}:${KAYOBE_AUTOMATION_PR_AUTH_TOKEN}@github.com/${{ github.repository }} + KAYOBE_AUTOMATION_PR_GITHUB_USER: !unsafe ${{ github.actor }} + KAYOBE_AUTOMATION_PR_AUTH_TOKEN: !unsafe ${{ github.token }} KAYOBE_AUTOMATION_PR_TYPE: github - KAYOBE_AUTOMATION_PR_TITLE: "[kayobe-automation] ${{ github.workflow }} #${{ github.run_id }}" - KAYOBE_AUTOMATION_PR_URL: https://api.github.com/repos/${{ github.repository }}/pulls - KAYOBE_AUTOMATION_TEMPEST_LOADLIST: "${{ github.event.inputs.testSuite }}" - TEMPEST_PATTERN: "${{ github.event.inputs.tempestPattern }}" + KAYOBE_AUTOMATION_PR_TITLE: !unsafe "[kayobe-automation] ${{ github.workflow }} #${{ github.run_id }}" + KAYOBE_AUTOMATION_PR_URL: !unsafe https://api.github.com/repos/${{ github.repository }}/pulls + KAYOBE_AUTOMATION_TEMPEST_LOADLIST: !unsafe "${{ github.event.inputs.testSuite }}" + TEMPEST_PATTERN: !unsafe "${{ github.event.inputs.tempestPattern }}" KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY: ghcr.io - KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_USERNAME: ${{ github.actor }} - KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_USERNAME: !unsafe ${{ github.actor }} + KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_PASSWORD: !unsafe ${{ secrets.GITHUB_TOKEN }} HOME: /stack github_kayobe_dispatch_inputs: From 37415d6705fd51e7b1255b080b83d1af656fe681 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 24 Aug 2023 22:33:37 +0100 Subject: [PATCH 18/42] feat: add support for collecting test artifacts --- .github/workflows/test-collection.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test-collection.yml b/.github/workflows/test-collection.yml index 167f8f7..50da212 100644 --- a/.github/workflows/test-collection.yml +++ b/.github/workflows/test-collection.yml @@ -23,3 +23,9 @@ jobs: run: ansible-playbook tests/test.yml env: ANSIBLE_FORCE_COLOR: '1' + + - name: Upload workflows produced + uses: actions/upload-artifact@v3 + with: + name: github_kayobe_workflows + path: tests/.github/workflows From b0a4d90099580b9e920ac11df0e7ae7fc849f859 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 25 Aug 2023 14:48:19 +0100 Subject: [PATCH 19/42] feat: update documentation --- roles/github/README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/roles/github/README.md b/roles/github/README.md index 0177c46..0f85c6e 100644 --- a/roles/github/README.md +++ b/roles/github/README.md @@ -38,7 +38,9 @@ The following variables can be used to make small adjustments to the composition `github_runs_on`: control which runner can accept this workflow. See GitHub for more information on [runs-on](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on). -`github_image_url`: full URL of the kayobe container image complete with registry and tag. +`github_image_name`: name of the kayobe image defaults to `kayobe`. + +`github_image_tag`: tag used to select kayobe image defaults to `latest` `github_registry_username`: username used to authenticate with the docker registry. @@ -48,6 +50,17 @@ The following variables can be used to make small adjustments to the composition `github_*_hook:` see section [Template Hooks](#template-hooks) for information about this variables +`github_buildx_inline_config`: provide configuration parameters to buildx. Useful for connecting to insecure docker registry. + +```yaml +github_buildx_inline_config: | + [registry."10.20.30.40:80"] + http = true + insecure = true +``` + +`github_timeout`: control how a long a job may run before being cancelled. Timeout is defined in minutes and defaults to 360 minutes (6 hours) + If you wish to make more impactful changes such as which workflows are built and what they contain then see the list of dictionaries called `workflows` in `defaults/main.yml` `github_workflows:` is a list of dictionaries that contains each of the workflows described above. A given list element is made up of the following: From 689362240ff60bd4f367b5a4697e496034d080be Mon Sep 17 00:00:00 2001 From: technowhizz <7688823+technowhizz@users.noreply.github.com> Date: Fri, 1 Sep 2023 15:19:03 +0100 Subject: [PATCH 20/42] Use separate directories for building and running Uses a separate directory for building the docker image and running the other workflows. This is because the user the is used by the docker git image is root which results in permission issues when the workflows try to run. This change means that the directories do not overlap, avoiding any permission issues. Co-Authored-By: Will Szumski --- roles/github/templates/build-kayobe-docker-image.yml.j2 | 5 +++-- roles/github/templates/generic.yml.j2 | 3 ++- roles/github/templates/run-config-diff.yml.j2 | 3 ++- roles/github/templates/run-tempest.yml.j2 | 3 ++- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index 4f12780..0da1b4a 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -23,6 +23,7 @@ jobs: uses: actions/checkout@v3 with: submodules: true + path: docker-image-build - name: Log in to the Container registry uses: docker/login-action@v2 @@ -47,8 +48,8 @@ jobs: - name: Build and push Docker image uses: docker/build-push-action@v4 with: - file: ./.automation/docker/kayobe/Dockerfile - context: . + file: ./docker-image-build/.automation/docker/kayobe/Dockerfile + context: docker-image-build build-args: | KAYOBE_DOCKER_SSH_CONFIG_PATH=.automation/docker/kayobe/ssh_config KAYOBE_USER_UID=${{ env.KAYOBE_USER_UID }} diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index a3a083c..33f585f 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -20,12 +20,13 @@ jobs: uses: actions/checkout@v3 with: submodules: true + path: kayobe-config <% if github_kayobe_hook | length >= 1 %> %% github_kayobe_hook | indent(width=6, first=false) %% <% endif %> - name: Symlink source checkout to expected location - run: sudo ln -s $PWD /src + run: sudo ln -s $PWD/kayobe-config /src - name: %% format_file_name(workflow.file_name, is_subtitle=true) %% run: | diff --git a/roles/github/templates/run-config-diff.yml.j2 b/roles/github/templates/run-config-diff.yml.j2 index ac87332..b1f77f5 100644 --- a/roles/github/templates/run-config-diff.yml.j2 +++ b/roles/github/templates/run-config-diff.yml.j2 @@ -26,6 +26,7 @@ jobs: uses: actions/checkout@v3 with: submodules: true + path: kayobe-config fetch-depth: 0 ref: ${{ github.ref }} @@ -34,7 +35,7 @@ jobs: <% endif %> - name: Copy checkout to expected location run: | - sudo cp -rf $GITHUB_WORKSPACE/ /src + sudo cp -rf $GITHUB_WORKSPACE/kayobe-config /src sudo chown stack:stack -Rf /src - name: Run config diff diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index 22d0a92..b2e3fc7 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -36,11 +36,12 @@ jobs: <% endif %> - name: Checkout kayobe config uses: actions/checkout@v3 + path: kayobe-config with: submodules: true - name: Symlink source checkout to expected location - run: sudo ln -s $PWD /src + run: sudo ln -s $PWD/kayobe-config /src <% if github_kayobe_hook | length >= 1 %> %% github_kayobe_hook | indent(width=6, first=false) %% From 02312aa28682642af7223a8e2663df1d406e2283 Mon Sep 17 00:00:00 2001 From: technowhizz <7688823+technowhizz@users.noreply.github.com> Date: Tue, 5 Sep 2023 16:39:36 +0100 Subject: [PATCH 21/42] Pin buildkit and buildx versions Pins versions for buildkit and buildx to the current latest working version to avoid futures issues. Co-authored-by: Will Szumski --- roles/github/templates/build-kayobe-docker-image.yml.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index 0da1b4a..d15b575 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -36,11 +36,12 @@ jobs: uses: docker/setup-buildx-action@v2 with: driver-opts: | - image=moby/buildkit:master + image=moby/buildkit:v0.12.1 <%if github_buildx_inline_config | length >= 1 %> config-inline: | %% github_buildx_inline_config | indent(12) | trim %% <% endif %> + version: v0.11.2 <% if github_kayobe_hook | length >= 1 %> %% github_kayobe_hook | indent(width=6, first=false) %% From 4b8b2973db354586585489bc3eaeaaee425374b0 Mon Sep 17 00:00:00 2001 From: technowhizz <7688823+technowhizz@users.noreply.github.com> Date: Tue, 5 Sep 2023 16:57:13 +0100 Subject: [PATCH 22/42] Add flag to disable provenance Adds a flag to disable build attestations/provenance. This causes issues on some clouds and so is defaulted to `false`. Whether or not it is enabled can be chaged with the github_buildx_enable_provenance variable. Co-authored-by: Will Szumski --- roles/github/README.md | 4 +++- roles/github/defaults/main.yml | 2 ++ roles/github/templates/build-kayobe-docker-image.yml.j2 | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/roles/github/README.md b/roles/github/README.md index 0f85c6e..d6d3ef2 100644 --- a/roles/github/README.md +++ b/roles/github/README.md @@ -40,7 +40,7 @@ The following variables can be used to make small adjustments to the composition `github_image_name`: name of the kayobe image defaults to `kayobe`. -`github_image_tag`: tag used to select kayobe image defaults to `latest` +`github_image_tag`: tag used to select kayobe image defaults to `latest` `github_registry_username`: username used to authenticate with the docker registry. @@ -63,6 +63,8 @@ github_buildx_inline_config: | If you wish to make more impactful changes such as which workflows are built and what they contain then see the list of dictionaries called `workflows` in `defaults/main.yml` +`github_buildx_enable_provenance`: whether or not to enable build attestations/provenence. This causes issues on some clouds and so is defaulted to `false`. + `github_workflows:` is a list of dictionaries that contains each of the workflows described above. A given list element is made up of the following: - `name`: the name which the workflow shall refer to itself as within GitHub workflows user interface. diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 04457d6..97549a3 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -25,6 +25,8 @@ github_buildx_inline_config: "" github_timeout: 360 +github_buildx_enable_provenance: false + github_kayobe_limit_input: | kayobeLimit: description: | diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index 4f12780..da11114 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -57,6 +57,9 @@ jobs: tags: | %% github_registry_url %%/%% github_image_name %%:latest %% github_registry_url %%/%% github_image_name %%:${{ github.sha }} +<% if not github_buildx_enable_provenance %> + provenance: false +<% endif %> <% if github_final_hook | length >= 1 +%> %% github_final_hook | indent(width=6, first=false) -%% <% endif %> From 796a27592e732b4470a7b6025e299cec04470253 Mon Sep 17 00:00:00 2001 From: Dawud <7688823+technowhizz@users.noreply.github.com> Date: Wed, 6 Sep 2023 14:12:46 +0100 Subject: [PATCH 23/42] Update roles/github/README.md Co-authored-by: Jack Hodgkiss --- roles/github/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/README.md b/roles/github/README.md index d6d3ef2..fc14d20 100644 --- a/roles/github/README.md +++ b/roles/github/README.md @@ -63,7 +63,7 @@ github_buildx_inline_config: | If you wish to make more impactful changes such as which workflows are built and what they contain then see the list of dictionaries called `workflows` in `defaults/main.yml` -`github_buildx_enable_provenance`: whether or not to enable build attestations/provenence. This causes issues on some clouds and so is defaulted to `false`. +`github_buildx_enable_provenance`: whether or not to enable build attestations/provenence. This has been [noted](https://github.com/docker/build-push-action/releases/tag/v4.1.1) to cause issues with docker registries such as Pulp. Default to false. `github_workflows:` is a list of dictionaries that contains each of the workflows described above. A given list element is made up of the following: From bff6590d00859ddcaa8c209c70cac9181b58d879 Mon Sep 17 00:00:00 2001 From: Dawud <7688823+technowhizz@users.noreply.github.com> Date: Wed, 6 Sep 2023 14:12:52 +0100 Subject: [PATCH 24/42] Update roles/github/defaults/main.yml Co-authored-by: Jack Hodgkiss --- roles/github/defaults/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 97549a3..c676dfb 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -23,10 +23,9 @@ github_final_hook: "" github_buildx_inline_config: "" -github_timeout: 360 - github_buildx_enable_provenance: false +github_timeout: 360 github_kayobe_limit_input: | kayobeLimit: description: | From e6654e4c8d959da8db8abbee853f650b07fcc392 Mon Sep 17 00:00:00 2001 From: Dawud <7688823+technowhizz@users.noreply.github.com> Date: Wed, 6 Sep 2023 14:19:56 +0100 Subject: [PATCH 25/42] Update roles/github/defaults/main.yml Co-authored-by: Jack Hodgkiss --- roles/github/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index c676dfb..6fc99ba 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -26,6 +26,7 @@ github_buildx_inline_config: "" github_buildx_enable_provenance: false github_timeout: 360 + github_kayobe_limit_input: | kayobeLimit: description: | From f58b587a6ed3318d67b0a98b4ba409a78801b93d Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Sat, 16 Sep 2023 14:49:18 +0100 Subject: [PATCH 26/42] fix: move `path` under `with` --- roles/github/templates/run-tempest.yml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index b2e3fc7..438f701 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -36,9 +36,9 @@ jobs: <% endif %> - name: Checkout kayobe config uses: actions/checkout@v3 - path: kayobe-config with: - submodules: true + submodules: true + path: kayobe-config - name: Symlink source checkout to expected location run: sudo ln -s $PWD/kayobe-config /src From f522fdad9f4b9c121f7095483a70df6ae6470ae4 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Sat, 16 Sep 2023 14:55:29 +0100 Subject: [PATCH 27/42] feat: allow for `buildx` to be toggled on or off --- roles/github/README.md | 6 ++++-- roles/github/defaults/main.yml | 2 ++ roles/github/templates/build-kayobe-docker-image.yml.j2 | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/roles/github/README.md b/roles/github/README.md index fc14d20..1961f2e 100644 --- a/roles/github/README.md +++ b/roles/github/README.md @@ -50,6 +50,8 @@ The following variables can be used to make small adjustments to the composition `github_*_hook:` see section [Template Hooks](#template-hooks) for information about this variables +`github_buildx_enable`: In some deployments the build kayobe docker image workflow has had difficulties successfully pushing the image to container registries such as Pulp if buildx has been used. It situations where failure to push images is been experienced a user might wish to disable buildx. Buildx is enabled by default. + `github_buildx_inline_config`: provide configuration parameters to buildx. Useful for connecting to insecure docker registry. ```yaml @@ -59,12 +61,12 @@ github_buildx_inline_config: | insecure = true ``` +`github_buildx_enable_provenance`: whether or not to enable build attestations/provenence. This has been [noted](https://github.com/docker/build-push-action/releases/tag/v4.1.1) to cause issues with docker registries such as Pulp. Default to false. + `github_timeout`: control how a long a job may run before being cancelled. Timeout is defined in minutes and defaults to 360 minutes (6 hours) If you wish to make more impactful changes such as which workflows are built and what they contain then see the list of dictionaries called `workflows` in `defaults/main.yml` -`github_buildx_enable_provenance`: whether or not to enable build attestations/provenence. This has been [noted](https://github.com/docker/build-push-action/releases/tag/v4.1.1) to cause issues with docker registries such as Pulp. Default to false. - `github_workflows:` is a list of dictionaries that contains each of the workflows described above. A given list element is made up of the following: - `name`: the name which the workflow shall refer to itself as within GitHub workflows user interface. diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 6fc99ba..bbfd570 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -21,6 +21,8 @@ github_kayobe_hook: "" github_final_hook: "" +github_buildx_enabled: true + github_buildx_inline_config: "" github_buildx_enable_provenance: false diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index 04ae581..8a0cabd 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -32,17 +32,19 @@ jobs: username: %% github_registry_username %% password: %% github_registry_password %% +<% if github_buildx_enabled %> - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 with: driver-opts: | image=moby/buildkit:v0.12.1 -<%if github_buildx_inline_config | length >= 1 %> +<% if github_buildx_inline_config | length >= 1 %> config-inline: | %% github_buildx_inline_config | indent(12) | trim %% <% endif %> version: v0.11.2 +<% endif %> <% if github_kayobe_hook | length >= 1 %> %% github_kayobe_hook | indent(width=6, first=false) %% <% endif %> From 45a398203933b7cb4023f798d1a22e94ba6ece63 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Sat, 16 Sep 2023 15:07:08 +0100 Subject: [PATCH 28/42] feat: provide control over kayobe base image used --- roles/github/README.md | 2 ++ roles/github/defaults/main.yml | 2 ++ roles/github/templates/build-kayobe-docker-image.yml.j2 | 1 + 3 files changed, 5 insertions(+) diff --git a/roles/github/README.md b/roles/github/README.md index 1961f2e..325fa50 100644 --- a/roles/github/README.md +++ b/roles/github/README.md @@ -46,6 +46,8 @@ The following variables can be used to make small adjustments to the composition `github_registry_password`: password used to authenticate with the docker registry. +`github_kayobe_base_image`: select the base image used when building the kayobe docker image. Default is `quay.io/centos/centos:stream8` supports OpenStack Wallaby, Xena and Yoga. Zed and higher would require `quay.io/rockylinux/rockylinux:9`. + `github_kayobe_arguments`: a dictionary of arguments that can be used to override the default arguments found within `vars/main.yml`. For example if you wanted to change the value of `KAYOBE_ENVIRONMENT` from its default of `production` you can simply add `KAYOBE_ENVIRONMENT` to this dictionary and it will take precedence over the defaults. `github_*_hook:` see section [Template Hooks](#template-hooks) for information about this variables diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index bbfd570..06970f7 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -13,6 +13,8 @@ github_image_name: kayobe github_image_tag: latest +github_kayobe_base_image: "quay.io/centos/centos:stream8" + github_kayobe_arguments: {} github_checkout_hook: "" diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index 8a0cabd..19a8465 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -57,6 +57,7 @@ jobs: KAYOBE_DOCKER_SSH_CONFIG_PATH=.automation/docker/kayobe/ssh_config KAYOBE_USER_UID=${{ env.KAYOBE_USER_UID }} KAYOBE_USER_GID=${{ env.KAYOBE_USER_UID }} + BASE_IMAGE: %% github_kayobe_base_image %% push: true tags: | %% github_registry_url %%/%% github_image_name %%:latest From d600947fee1b5a6ea80c3dd514d0b59b58742795 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Sat, 16 Sep 2023 15:10:26 +0100 Subject: [PATCH 29/42] fix: `=` not `:` --- roles/github/templates/build-kayobe-docker-image.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index 19a8465..bcda0b2 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -57,7 +57,7 @@ jobs: KAYOBE_DOCKER_SSH_CONFIG_PATH=.automation/docker/kayobe/ssh_config KAYOBE_USER_UID=${{ env.KAYOBE_USER_UID }} KAYOBE_USER_GID=${{ env.KAYOBE_USER_UID }} - BASE_IMAGE: %% github_kayobe_base_image %% + BASE_IMAGE=%% github_kayobe_base_image %% push: true tags: | %% github_registry_url %%/%% github_image_name %%:latest From 8087abbb6b2a7f5aa3b0d68cf56eacbddad9b977 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Sat, 16 Sep 2023 15:11:39 +0100 Subject: [PATCH 30/42] feat: adjust permissions for workflows --- roles/github/templates/build-kayobe-docker-image.yml.j2 | 1 - roles/github/templates/generic.yml.j2 | 3 +-- roles/github/templates/run-config-diff.yml.j2 | 1 - roles/github/templates/run-tempest.yml.j2 | 1 - 4 files changed, 1 insertion(+), 5 deletions(-) diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index bcda0b2..f39724d 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -14,7 +14,6 @@ jobs: image: docker:24.0-git permissions: contents: read - packages: write steps: <% if github_checkout_hook | length >= 1 %> %% github_checkout_hook | indent(width=6, first=false) %% diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 33f585f..f3a96e9 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -4,8 +4,7 @@ jobs: %% format_file_name(workflow.file_name) %%: runs-on: %% github_runs_on %% permissions: - contents: read - packages: read + contents: write container: image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% credentials: diff --git a/roles/github/templates/run-config-diff.yml.j2 b/roles/github/templates/run-config-diff.yml.j2 index b1f77f5..ab1a1c3 100644 --- a/roles/github/templates/run-config-diff.yml.j2 +++ b/roles/github/templates/run-config-diff.yml.j2 @@ -12,7 +12,6 @@ jobs: runs-on: %% github_runs_on %% permissions: contents: read - packages: read container: image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% credentials: diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index 438f701..303a7df 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -23,7 +23,6 @@ jobs: runs-on: %% github_runs_on %% permissions: contents: read - packages: write container: image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% credentials: From 297695e75e598fbe720b795117c37f503e3179fb Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Sat, 16 Sep 2023 15:12:47 +0100 Subject: [PATCH 31/42] fix: resolve issue with tempest logging in with `Jinja` --- roles/github/templates/run-tempest.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index 303a7df..056713e 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -63,7 +63,7 @@ jobs: - name: Run ${{ github.event.inputs.testSuite }} tempest testsuite run: | - /src/.automation/pipeline/tempest.sh + /src/.automation/pipeline/tempest.sh -e ansible_user=stack env: KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' From d9619a231c019582baa5db90dd039b74a489858c Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Sat, 16 Sep 2023 15:30:02 +0100 Subject: [PATCH 32/42] feat: provide greater control over `packages` permissions --- roles/github/templates/build-kayobe-docker-image.yml.j2 | 1 + roles/github/templates/generic.yml.j2 | 1 + roles/github/templates/run-config-diff.yml.j2 | 1 + roles/github/templates/run-tempest.yml.j2 | 1 + 4 files changed, 4 insertions(+) diff --git a/roles/github/templates/build-kayobe-docker-image.yml.j2 b/roles/github/templates/build-kayobe-docker-image.yml.j2 index f39724d..9df472b 100644 --- a/roles/github/templates/build-kayobe-docker-image.yml.j2 +++ b/roles/github/templates/build-kayobe-docker-image.yml.j2 @@ -14,6 +14,7 @@ jobs: image: docker:24.0-git permissions: contents: read + packages: %% 'write' if github_registry_url == 'ghcr.io' else 'none' %% steps: <% if github_checkout_hook | length >= 1 %> %% github_checkout_hook | indent(width=6, first=false) %% diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index f3a96e9..2a259bb 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -5,6 +5,7 @@ jobs: runs-on: %% github_runs_on %% permissions: contents: write + packages: %% 'read' if github_registry_url == 'ghcr.io' else 'none' %% container: image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% credentials: diff --git a/roles/github/templates/run-config-diff.yml.j2 b/roles/github/templates/run-config-diff.yml.j2 index ab1a1c3..9771e16 100644 --- a/roles/github/templates/run-config-diff.yml.j2 +++ b/roles/github/templates/run-config-diff.yml.j2 @@ -12,6 +12,7 @@ jobs: runs-on: %% github_runs_on %% permissions: contents: read + packages: %% 'read' if github_registry_url == 'ghcr.io' else 'none' %% container: image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% credentials: diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index 056713e..a1ee75c 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -23,6 +23,7 @@ jobs: runs-on: %% github_runs_on %% permissions: contents: read + packages: %% 'read' if github_registry_url == 'ghcr.io' else 'none' %% container: image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% credentials: From 10451b8e89867dd2d0214933a157f8060c257c68 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Sat, 16 Sep 2023 19:03:09 +0100 Subject: [PATCH 33/42] feat: add basic `concurrency` --- roles/github/defaults/main.yml | 20 ++++++++++++++++++++ roles/github/templates/generic.yml.j2 | 3 +++ roles/github/templates/run-tempest.yml.j2 | 3 +++ 3 files changed, 26 insertions(+) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 06970f7..08407b4 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -88,114 +88,134 @@ github_run_infra_vm_host_configure: trigger: workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" + concurrency_group: infra github_run_infra_vm_host_package_update: file_name: run-infra-vm-host-package-update.yml trigger: workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" + concurrency_group: infra github_run_infra_vm_provision: file_name: run-infra-vm-provision.yml trigger: workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" + concurrency_group: infra github_run_infra_vm_service_deploy: file_name: run-infra-vm-service-deploy.yml trigger: workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" + concurrency_group: infra github_run_network_connectivity_check: file_name: run-network-connectivity-check.yml trigger: workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" + concurrency_group: network github_run_overcloud_container_image_pull: file_name: run-overcloud-container-image-pull.yml trigger: workflow_dispatch: ["{{ github_kolla_dispatch_inputs }}", "{{ github_kayobe_dispatch_inputs }}"] arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}"] + concurrency_group: overcloud github_run_overcloud_database_backup: file_name: run-overcloud-database-backup.yml trigger: workflow_dispatch: ["{{ github_kolla_dispatch_inputs }}", "{{ github_kayobe_dispatch_inputs }}"] arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}"] + concurrency_group: overcloud github_run_overcloud_host_configure: file_name: run-overcloud-host-configure.yml trigger: workflow_dispatch: ["{{ github_kolla_dispatch_inputs }}", "{{ github_kayobe_dispatch_inputs }}"] arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}"] + concurrency_group: overcloud github_run_overcloud_host_package_update: file_name: run-overcloud-host-package-update.yml trigger: workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" + concurrency_group: overcloud github_run_overcloud_inventory_discover: file_name: run-overcloud-inventory-discover.yml trigger: workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kayobe_pull_request_arguments }}"] + concurrency_group: overcloud github_run_overcloud_provision: file_name: run-overcloud-provision.yml trigger: workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" + concurrency_group: overcloud github_run_overcloud_service_deploy: file_name: run-overcloud-service-deploy.yml trigger: workflow_dispatch: ["{{ github_kolla_dispatch_inputs }}", "{{ github_kayobe_dispatch_inputs }}"] arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}", "{{ github_kayobe_pull_request_arguments }}"] + concurrency_group: overcloud github_run_overcloud_service_upgrade: file_name: run-overcloud-service-upgrade.yml trigger: workflow_dispatch: ["{{ github_kolla_dispatch_inputs }}", "{{ github_kayobe_dispatch_inputs }}"] arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}", "{{ github_kayobe_pull_request_arguments }}"] + concurrency_group: overcloud github_run_seed_host_configure: file_name: run-seed-host-configure.yml trigger: workflow_dispatch: ["{{ github_kolla_dispatch_inputs }}", "{{ github_kayobe_dispatch_inputs }}"] arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}"] + concurrency_group: seed github_run_seed_host_package_update: file_name: run-seed-host-package-update.yml trigger: workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" + concurrency_group: seed + github_run_seed_hypervisor_host_configure: file_name: run-seed-hypervisor-host-configure.yml trigger: workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" + concurrency_group: seed github_run_seed_hypervisor_host_package_update: file_name: run-seed-hypervisor-host-package-update.yml trigger: workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" + concurrency_group: seed github_run_seed_service_deploy: file_name: run-seed-service-deploy.yml trigger: workflow_dispatch: ["{{ github_kolla_dispatch_inputs }}", "{{ github_kayobe_dispatch_inputs }}"] arguments: ["{{ github_kayobe_task_arguments }}", "{{ github_kolla_task_arguments }}", "{{ github_kayobe_pull_request_arguments }}"] + concurrency_group: seed github_run_seed_vm_provision: file_name: run-seed-vm-provision.yml trigger: workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" arguments: "{{ github_kayobe_task_arguments }}" + concurrency_group: seed github_run_tempest: file_name: run-tempest.yml diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 2a259bb..06721f9 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -11,6 +11,9 @@ jobs: credentials: username: %% github_registry_username %% password: %% github_registry_password %% + concurrency: + group: %% workflow.concurrency_group %% + cancel-in-progress: false timeout-minutes: %% github_timeout %% steps: <% if github_checkout_hook | length >= 1 %> diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index a1ee75c..f1e3fa1 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -29,6 +29,9 @@ jobs: credentials: username: %% github_registry_username %% password: %% github_registry_password %% + concurrency: + group: %% workflow.concurrency_group %% + cancel-in-progress: false timeout-minutes: %% github_timeout %% steps: <% if github_checkout_hook | length >= 1 %> From ed93f143d5db6a3135471ee2937b98400efb9b4c Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Sat, 16 Sep 2023 19:05:51 +0100 Subject: [PATCH 34/42] fix: explicitly set `tempest` concurrency group --- roles/github/templates/run-tempest.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index f1e3fa1..f34b97f 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -30,7 +30,7 @@ jobs: username: %% github_registry_username %% password: %% github_registry_password %% concurrency: - group: %% workflow.concurrency_group %% + group: tempest cancel-in-progress: false timeout-minutes: %% github_timeout %% steps: From 976f00bdec1474081e3c9c04855af13f37f900ad Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Sat, 16 Sep 2023 19:27:01 +0100 Subject: [PATCH 35/42] feat: provide control over tempest test suites. --- roles/github/README.md | 2 ++ roles/github/defaults/main.yml | 5 +++++ roles/github/templates/run-tempest.yml.j2 | 3 +-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/roles/github/README.md b/roles/github/README.md index 325fa50..cc81f79 100644 --- a/roles/github/README.md +++ b/roles/github/README.md @@ -67,6 +67,8 @@ github_buildx_inline_config: | `github_timeout`: control how a long a job may run before being cancelled. Timeout is defined in minutes and defaults to 360 minutes (6 hours) +`github_tempest_test_suites`: provide a list of load lists to be made available within the drop-down list for running tempest. Defaults to `default` and `tempest-full`. + If you wish to make more impactful changes such as which workflows are built and what they contain then see the list of dictionaries called `workflows` in `defaults/main.yml` `github_workflows:` is a list of dictionaries that contains each of the workflows described above. A given list element is made up of the following: diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 08407b4..660be0c 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -31,6 +31,10 @@ github_buildx_enable_provenance: false github_timeout: 360 +github_tempest_test_suites: | + - default + - tempest-full + github_kayobe_limit_input: | kayobeLimit: description: | @@ -220,3 +224,4 @@ github_run_seed_vm_provision: github_run_tempest: file_name: run-tempest.yml use_bespoke: true + test_suites: "{{ github_tempest_test_suites }}" diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index f34b97f..9f5b352 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -12,8 +12,7 @@ on: default: 'default' type: choice options: - - default - - tempest-full + %% workflow.test_suites | flatten | join('') | indent(10) | trim %% tempestPattern: description: | Limit tests to this regex. Takes precedence over testSuite. From 51b5667ca72de9d139a0cd6d4f30f30b4ae466f8 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 18 Sep 2023 08:58:29 +0100 Subject: [PATCH 36/42] fix: remove unused variables --- roles/github/vars/main.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/roles/github/vars/main.yml b/roles/github/vars/main.yml index ee5da12..50b15e4 100644 --- a/roles/github/vars/main.yml +++ b/roles/github/vars/main.yml @@ -16,9 +16,6 @@ default_kayobe_arguments: KAYOBE_AUTOMATION_PR_URL: !unsafe https://api.github.com/repos/${{ github.repository }}/pulls KAYOBE_AUTOMATION_TEMPEST_LOADLIST: !unsafe "${{ github.event.inputs.testSuite }}" TEMPEST_PATTERN: !unsafe "${{ github.event.inputs.tempestPattern }}" - KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY: ghcr.io - KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_USERNAME: !unsafe ${{ github.actor }} - KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_PASSWORD: !unsafe ${{ secrets.GITHUB_TOKEN }} HOME: /stack github_kayobe_dispatch_inputs: From e19e57df7fe95558990ed529f29bfb9f191b4ae3 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 27 Sep 2023 13:00:09 +0100 Subject: [PATCH 37/42] fix: request a token with `pull-request` write support --- roles/github/templates/generic.yml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 06721f9..277ec20 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -6,6 +6,7 @@ jobs: permissions: contents: write packages: %% 'read' if github_registry_url == 'ghcr.io' else 'none' %% + pull-requests: write container: image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% credentials: From 3c960d48c59c02c0ac125bcd2365ce301efce0c1 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 27 Sep 2023 13:05:32 +0100 Subject: [PATCH 38/42] fix: add `github_` prefix to variable --- roles/github/templates/generic.yml.j2 | 10 +++++----- roles/github/templates/run-config-diff.yml.j2 | 8 ++++---- roles/github/templates/run-tempest.yml.j2 | 20 +++++++++---------- roles/github/vars/main.yml | 2 +- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 277ec20..c7b97a1 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -37,13 +37,13 @@ jobs: /src/.automation/pipeline/%% workflow.file_name[4:-4] %%.sh <%- if workflow.arguments is defined +%> env: - KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' - KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' + KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' + KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' <%- for argument in workflow.arguments | flatten +%> - %% argument %%: '%% github_kayobe_arguments[argument] | default(default_kayobe_arguments[argument]) %%' + %% argument %%: '%% github_kayobe_arguments[argument] | default(github_default_kayobe_arguments[argument]) %%' <%- endfor +%> - HOME: '%% github_kayobe_arguments.HOME | default(default_kayobe_arguments.HOME) %%' + HOME: '%% github_kayobe_arguments.HOME | default(github_default_kayobe_arguments.HOME) %%' <%- endif +%> <% if github_final_hook | length >= 1 +%> %% github_final_hook | indent(width=6, first=false) -%% diff --git a/roles/github/templates/run-config-diff.yml.j2 b/roles/github/templates/run-config-diff.yml.j2 index 9771e16..fefe9c4 100644 --- a/roles/github/templates/run-config-diff.yml.j2 +++ b/roles/github/templates/run-config-diff.yml.j2 @@ -42,10 +42,10 @@ jobs: run: | sudo -E -u stack bash -c '/src/.automation/pipeline/config-diff.sh ${{ github.event.pull_request.base.sha }}' env: - KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' - KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' - HOME: '%% github_kayobe_arguments.HOME | default(default_kayobe_arguments.HOME) %%' + KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' + KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' + HOME: '%% github_kayobe_arguments.HOME | default(github_default_kayobe_arguments.HOME) %%' - name: Show summary of changes run: | diff --git a/roles/github/templates/run-tempest.yml.j2 b/roles/github/templates/run-tempest.yml.j2 index 9f5b352..e5bbb1d 100644 --- a/roles/github/templates/run-tempest.yml.j2 +++ b/roles/github/templates/run-tempest.yml.j2 @@ -53,10 +53,10 @@ jobs: source /src/.automation/functions && kayobe_install env: - KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' - KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' - HOME: '%% github_kayobe_arguments.HOME | default(default_kayobe_arguments.HOME) %%' + KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' + KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' + HOME: '%% github_kayobe_arguments.HOME | default(github_default_kayobe_arguments.HOME) %%' USER: stack - name: Symlink kolla into /src/etc @@ -68,15 +68,15 @@ jobs: run: | /src/.automation/pipeline/tempest.sh -e ansible_user=stack env: - KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' - KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' - KAYOBE_AUTOMATION_TEMPEST_LOADLIST: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_TEMPEST_LOADLIST | default(default_kayobe_arguments.KAYOBE_AUTOMATION_TEMPEST_LOADLIST) %%' - TEMPEST_PATTERN: '%% github_kayobe_arguments.TEMPEST_PATTERN | default(default_kayobe_arguments.TEMPEST_PATTERN) %%' + KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT | default(github_default_kayobe_arguments.KAYOBE_ENVIRONMENT) %%' + KAYOBE_VAULT_PASSWORD: '%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%' + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%' + KAYOBE_AUTOMATION_TEMPEST_LOADLIST: '%% github_kayobe_arguments.KAYOBE_AUTOMATION_TEMPEST_LOADLIST | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_TEMPEST_LOADLIST) %%' + TEMPEST_PATTERN: '%% github_kayobe_arguments.TEMPEST_PATTERN | default(github_default_kayobe_arguments.TEMPEST_PATTERN) %%' KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY: '%% github_registry_url %%' KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_USERNAME: '%% github_registry_username %%' KAYOBE_AUTOMATION_RALLY_DOCKER_REGISTRY_PASSWORD: '%% github_registry_password %%' - HOME: '%% github_kayobe_arguments.HOME | default(default_kayobe_arguments.HOME) %%' + HOME: '%% github_kayobe_arguments.HOME | default(github_default_kayobe_arguments.HOME) %%' - name: Print stdout run: | diff --git a/roles/github/vars/main.yml b/roles/github/vars/main.yml index 50b15e4..d9f54b4 100644 --- a/roles/github/vars/main.yml +++ b/roles/github/vars/main.yml @@ -1,5 +1,5 @@ --- -default_kayobe_arguments: +github_default_kayobe_arguments: KAYOBE_ENVIRONMENT: production KAYOBE_VAULT_PASSWORD: !unsafe "${{ secrets.KAYOBE_VAULT_PASSWORD }}" KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: !unsafe "${{ secrets.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY }}" From 4bb6629656f953b73acb58a64908b45d3f29a8b3 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 27 Sep 2023 15:11:06 +0100 Subject: [PATCH 39/42] feat: only set `pull-requests` to `write` when needed --- roles/github/templates/generic.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index c7b97a1..0936b22 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -6,7 +6,7 @@ jobs: permissions: contents: write packages: %% 'read' if github_registry_url == 'ghcr.io' else 'none' %% - pull-requests: write + pull-requests: %% 'write' if KAYOBE_AUTOMATION_PR_TYPE in workflow.arguments else 'none' %% container: image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% credentials: From 777cf7b527381374c3d42cce9954cc7bbe5c1bc5 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 27 Sep 2023 15:32:56 +0100 Subject: [PATCH 40/42] fix: add missing quotes --- roles/github/templates/generic.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 0936b22..9e719c4 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -6,7 +6,7 @@ jobs: permissions: contents: write packages: %% 'read' if github_registry_url == 'ghcr.io' else 'none' %% - pull-requests: %% 'write' if KAYOBE_AUTOMATION_PR_TYPE in workflow.arguments else 'none' %% + pull-requests: %% 'write' if 'KAYOBE_AUTOMATION_PR_TYPE' in workflow.arguments else 'none' %% container: image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% credentials: From 748bd1b8fc6f3c2b07f08360e2c202cfd8f87389 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 27 Sep 2023 15:54:37 +0100 Subject: [PATCH 41/42] fix: flatten `workflow.argument` for `pull-requests` The `workflow.arguments` variable can be a list of lists and simply checking if the string `KAYOBE_AUTOMATION_PR_TYPE` is contained will not work without flattening the list --- roles/github/templates/generic.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 9e719c4..664c793 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -6,7 +6,7 @@ jobs: permissions: contents: write packages: %% 'read' if github_registry_url == 'ghcr.io' else 'none' %% - pull-requests: %% 'write' if 'KAYOBE_AUTOMATION_PR_TYPE' in workflow.arguments else 'none' %% + pull-requests: %% 'write' if 'KAYOBE_AUTOMATION_PR_TYPE' in workflow.arguments | flatten else 'none' %% container: image: %% github_registry_url %%/%% github_image_name %%:%% github_image_tag %% credentials: From f20ebd466e25ecadddaff2c3c417636a7ad69d7d Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 27 Sep 2023 16:42:12 +0100 Subject: [PATCH 42/42] feat: set `r/w` permissions for `contents` --- roles/github/templates/generic.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github/templates/generic.yml.j2 b/roles/github/templates/generic.yml.j2 index 664c793..af2391a 100644 --- a/roles/github/templates/generic.yml.j2 +++ b/roles/github/templates/generic.yml.j2 @@ -4,7 +4,7 @@ jobs: %% format_file_name(workflow.file_name) %%: runs-on: %% github_runs_on %% permissions: - contents: write + contents: %% 'write' if 'KAYOBE_AUTOMATION_PR_TYPE' in workflow.arguments | flatten else 'read' %% packages: %% 'read' if github_registry_url == 'ghcr.io' else 'none' %% pull-requests: %% 'write' if 'KAYOBE_AUTOMATION_PR_TYPE' in workflow.arguments | flatten else 'none' %% container: