diff --git a/.github/workflows/multinode-inputs.py b/.github/workflows/multinode-inputs.py new file mode 100644 index 000000000..b862cbf91 --- /dev/null +++ b/.github/workflows/multinode-inputs.py @@ -0,0 +1,80 @@ +# Generate inputs for the reusable multinode.yml workflow. +# The test scenario is randomly selected. +# The inputs are printed to stdout in GitHub step output key=value format. + +from dataclasses import dataclass +import random +import typing as t + + +@dataclass +class OSRelease: + distribution: str + release: str + ssh_username: str + + +@dataclass +class OpenStackRelease: + version: str + previous_version: str + os_releases: t.List[OSRelease] + + +@dataclass +class Scenario: + openstack_release: OpenStackRelease + os_release: OSRelease + neutron_plugin: str + upgrade: bool + + +ROCKY_9 = OSRelease("rocky", "9", "cloud-user") +UBUNTU_JAMMY = OSRelease("ubuntu", "jammy", "ubuntu") +# NOTE(upgrade): Add supported releases here. +OPENSTACK_RELEASES = [ + OpenStackRelease("2023.1", "zed", [ROCKY_9, UBUNTU_JAMMY]) +] +NEUTRON_PLUGINS = ["ovs", "ovn"] + + +def main() -> None: + scenario = random_scenario() + inputs = generate_inputs(scenario) + for name, value in inputs.items(): + write_output(name, value) + + +def random_scenario() -> Scenario: + openstack_release = random.choice(OPENSTACK_RELEASES) + os_release = random.choice(openstack_release.os_releases) + neutron_plugin = random.choice(NEUTRON_PLUGINS) + upgrade = random.random() > 0.6 + return Scenario(openstack_release, os_release, neutron_plugin, upgrade) + + +def generate_inputs(scenario: Scenario) -> t.Dict[str, str]: + branch = get_branch(scenario.openstack_release.version) + previous_branch = get_branch(scenario.openstack_release.previous_version) + inputs = { + "os_distribution": scenario.os_release.distribution, + "os_release": scenario.os_release.release, + "ssh_username": scenario.os_release.ssh_username, + "neutron_plugin": scenario.neutron_plugin, + "upgrade": str(scenario.upgrade).lower(), + "stackhpc_kayobe_config_version": branch, + "stackhpc_kayobe_config_previous_version": previous_branch, + } + return inputs + + +def get_branch(version: str) -> str: + return f"stackhpc/{version}" + + +def write_output(name: str, value: str) -> None: + print(f"{name}={value}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml index b8a1b8402..1752d6c05 100644 --- a/.github/workflows/overcloud-host-image-build.yml +++ b/.github/workflows/overcloud-host-image-build.yml @@ -191,7 +191,7 @@ jobs: source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && kayobe seed host command run \ - --command "sudo dnf config-manager --set-enabled crb && sudo dnf -y install epel-release && sudo dnf -y install zstd debootstrap kpartx cloud-init" --show-output + --command "sudo dnf config-manager --set-enabled crb && sudo dnf -y install epel-release && sudo dnf -y install cloud-init debootstrap git kpartx zstd" --show-output env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} diff --git a/.github/workflows/stackhpc-multinode-periodic.yml b/.github/workflows/stackhpc-multinode-periodic.yml new file mode 100644 index 000000000..341fbe55b --- /dev/null +++ b/.github/workflows/stackhpc-multinode-periodic.yml @@ -0,0 +1,50 @@ +--- +# This workflow provides a periodic deploy of a multi-node test cluster. +# The test scenario is randomly selected. + +name: Multinode periodic +'on': + schedule: + # Runs nightly at 2:42 AM. + - cron: "42 2 * * *" +jobs: + generate-inputs: + name: Generate inputs + runs-on: ubuntu-latest + outputs: + os_distribution: ${{ steps.generate-inputs.outputs.os_distribution }} + os_release: ${{ steps.generate-inputs.outputs.os_release }} + ssh_username: ${{ steps.generate-inputs.outputs.ssh_username }} + neutron_plugin: ${{ steps.generate-inputs.outputs.neutron_plugin }} + upgrade: ${{ steps.generate-inputs.outputs.upgrade }} + stackhpc_kayobe_config_version: ${{ steps.generate-inputs.outputs.stackhpc_kayobe_config_version }} + stackhpc_kayobe_config_previous_version: ${{ steps.generate-inputs.outputs.stackhpc_kayobe_config_previous_version }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Generate inputs for multinode workflow + id: generate-inputs + run: | + python3 .github/workflows/multinode-inputs.py >> $GITHUB_OUTPUT + + - name: Display generated inputs + run: | + echo '${{ toJSON(steps.generate-inputs.outputs) }}' + multinode: + name: Multinode periodic + needs: + - generate-inputs + uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.0.1 + with: + multinode_name: mn-prdc-${{ github.run_id }} + os_distribution: ${{ needs.generate-inputs.outputs.os_distribution }} + os_release: ${{ needs.generate-inputs.outputs.os_release }} + ssh_username: ${{ needs.generate-inputs.outputs.ssh_username }} + neutron_plugin: ${{ needs.generate-inputs.outputs.neutron_plugin }} + upgrade: ${{ needs.generate-inputs.outputs.upgrade == 'true' }} + stackhpc_kayobe_config_version: ${{ needs.generate-inputs.outputs.stackhpc_kayobe_config_version }} + stackhpc_kayobe_config_previous_version: ${{ needs.generate-inputs.outputs.stackhpc_kayobe_config_previous_version }} + enable_slack_alert: true + secrets: inherit + if: github.repository == 'stackhpc/stackhpc-kayobe-config' diff --git a/.github/workflows/stackhpc-multinode.yml b/.github/workflows/stackhpc-multinode.yml index 22f497a7d..2ca96a50c 100644 --- a/.github/workflows/stackhpc-multinode.yml +++ b/.github/workflows/stackhpc-multinode.yml @@ -52,7 +52,7 @@ name: Multinode jobs: multinode: name: Multinode - uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.0.0 + uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.0.1 with: multinode_name: ${{ inputs.multinode_name }} os_distribution: ${{ inputs.os_distribution }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..498ddee33 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,11 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/sirwart/ripsecrets + rev: v0.1.7 + hooks: + - id: ripsecrets diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst index 346e67989..988957541 100644 --- a/doc/source/contributor/index.rst +++ b/doc/source/contributor/index.rst @@ -11,3 +11,4 @@ This guide is for contributors of the StackHPC Kayobe configuration project. release-notes environments/index package-updates + pre-commit diff --git a/doc/source/contributor/pre-commit.rst b/doc/source/contributor/pre-commit.rst new file mode 100644 index 000000000..3afffc11b --- /dev/null +++ b/doc/source/contributor/pre-commit.rst @@ -0,0 +1,47 @@ +================ +Pre-commit Hooks +================ + +StackHPC Kayobe configuration carries support for +`pre-commit hooks `_ which simplify the use of git +hooks enabling the identification and repairing of broken or poor code +before committing. +These hooks are designed to make working within SKC easier and less error prone. + +Currently the following hooks are provided: + +- ``check-yaml``: perform basic yaml syntax linting +- ``end-of-file-fixer``: identify and automatically fix missing newline +- ``trailing-whitespace``: identify and automatically fix excessive white space +- ``ripsecrets``: identify and prevent secrets from being committed to the branch + +.. warning:: + The hook ``ripsecrets`` is capable of preventing the accidental leaking of secrets + such as those found within `secrets.yml` or `passwords.yml`. + However if the secret is contained within a file on it's own and lacks a certain level + of entropy then the secret will not be identified as such as and maybe leaked as a result. + +Installation of `pre-commit` hooks is handled via the `install-pre-commit-hooks` playbook +found within the Ansible directory. +Either run the playbook manually or add the playbook as a hook within Kayobe config such as +within `control-host-bootstrap/post.d`. +Once done you should find `pre-commit` is available within the `kayobe` virtualenv. + +To run the playbook using the following command + +- ``kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/install-pre-commit-hooks.yml`` + +Whereas to run the playbook when control host bootstrap runs ensure it registered as symlink using the following command + +- ``mkdir -p ${KAYOBE_CONFIG_PATH}/hooks/control-host-bootstrap/post.d`` +- ``ln -s ${KAYOBE_CONFIG_PATH}/ansible/install-pre-commit-hooks.yml ${KAYOBE_CONFIG_PATH}/hooks/control-host-bootstrap/post.d/install-pre-commit-hooks.yml`` + +All that remains is the installation of the hooks themselves which can be accomplished either by +running `pre-commit run` or using `git commit` when you have changes that need to be committed. +This will trigger a brief installation process of the hooks which may take a few minutes. +This a one time process and will not be required again unless new hooks are added or existing ones are updated. + +.. note:: + Currently if you run ``pre-commit run --all-files`` it will make a series of changes to + release notes that lack new lines as well configuration files that ``check-yaml`` does not + approve of. diff --git a/etc/kayobe/ansible/install-pre-commit-hooks.yml b/etc/kayobe/ansible/install-pre-commit-hooks.yml new file mode 100644 index 000000000..694e07bec --- /dev/null +++ b/etc/kayobe/ansible/install-pre-commit-hooks.yml @@ -0,0 +1,21 @@ +--- +- name: Install pre-commit hooks + hosts: localhost + gather_facts: false + vars: + pre_commit_version: 3.5.0 + tasks: + - name: Install pre-commit hooks + block: + - name: Install pre-commit hooks into kayobe virtual env + ansible.builtin.pip: + name: pre-commit + version: "{{ pre_commit_version }}" + virtualenv: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') | default(omit, true) }}" + register: pip_install + + - name: Register pre-commit hooks with git + ansible.builtin.command: + cmd: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') | default(lookup('ansible.builtin.env', 'HOME') ~ '/.local', true) }}/bin/pre-commit install" + args: + chdir: "{{ playbook_dir | dirname | dirname | dirname }}" diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml index 303ef600e..4a8b6626f 100644 --- a/etc/kayobe/environments/ci-multinode/kolla/globals.yml +++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml @@ -1,3 +1,4 @@ +# yamllint disable-file --- ############################################################################ # TLS diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index f697c90bd..530ed5396 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -147,14 +147,6 @@ kolla_sources: type: git location: https://github.com/stackhpc/networking-generic-switch.git reference: stackhpc/{{ openstack_release }} - octavia-api-plugin-ovn-octavia-provider: - type: git - location: https://github.com/stackhpc/ovn-octavia-provider.git - reference: stackhpc/{{ openstack_release }} - octavia-driver-agent-plugin-ovn-octavia-provider: - type: git - location: https://github.com/stackhpc/ovn-octavia-provider.git - reference: stackhpc/{{ openstack_release }} ############################################################################### # Kolla image build configuration. diff --git a/releasenotes/notes/add-pre-commit-hooks-07ce3b82bbe1d7a3.yaml b/releasenotes/notes/add-pre-commit-hooks-07ce3b82bbe1d7a3.yaml new file mode 100644 index 000000000..1f0ac091d --- /dev/null +++ b/releasenotes/notes/add-pre-commit-hooks-07ce3b82bbe1d7a3.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Add playbook to install pre-commit hooks and register them with git. + The hooks currently configured to be installed will check yaml syntax, + fix new line at end of file and remove excess whitespace. This is + currently opt-in which can be achieved by running `install-pre-commit-hooks` + playbook.