diff --git a/.github/path-filters.yml b/.github/path-filters.yml index b9815d9d6..02d0ab609 100644 --- a/.github/path-filters.yml +++ b/.github/path-filters.yml @@ -1,5 +1,5 @@ # This file is a list of path filters for the PR workflow in .github/workflows/stackhpc-pull-request.yml. -aio: +aio: &aio - '.automation' - '.automation.conf/config.sh' - '.automation.conf/tempest/load-lists/default' @@ -20,6 +20,11 @@ aio: - 'kayobe-env' - 'requirements.txt' - 'terraform/aio/**' -check-tags: +check-tags: &check-tags - '.github/workflows/stackhpc-check-tags.yml' - 'etc/kayobe/kolla-image-tags.yml' + - 'etc/kayobe/pulp.yml' + - 'tools/kolla-images.py' +build-kayobe-image: + - *aio + - *check-tags diff --git a/.github/workflows/stackhpc-check-tags.yml b/.github/workflows/stackhpc-check-tags.yml index 4016c00e9..db2383e21 100644 --- a/.github/workflows/stackhpc-check-tags.yml +++ b/.github/workflows/stackhpc-check-tags.yml @@ -46,6 +46,15 @@ jobs: run: | docker image pull $KAYOBE_IMAGE + - name: Check kolla-images.py image map and tag hierarchy + run: | + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + $KAYOBE_IMAGE \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh \ + '$KAYOBE_CONFIG_PATH/ansible/check-kolla-images-py.yml' + - name: Check container image tags run: | docker run -t --rm \ diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml index 34d644bd9..f5cfb90de 100644 --- a/.github/workflows/stackhpc-pull-request.yml +++ b/.github/workflows/stackhpc-pull-request.yml @@ -20,6 +20,7 @@ jobs: if: github.repository == 'stackhpc/stackhpc-kayobe-config' outputs: aio: ${{ steps.changes.outputs.aio }} + build-kayobe-image: ${{ steps.changes.outputs.build-kayobe-image }} check-tags: ${{ steps.changes.outputs.check-tags }} steps: - name: GitHub Checkout @@ -74,7 +75,7 @@ jobs: - check-changes uses: ./.github/workflows/stackhpc-build-kayobe-image.yml with: - if: ${{ needs.check-changes.outputs.aio == 'true' }} + if: ${{ needs.check-changes.outputs.build-kayobe-image == 'true' }} if: github.repository == 'stackhpc/stackhpc-kayobe-config' check-tags: diff --git a/etc/kayobe/ansible/check-kolla-images-py.yml b/etc/kayobe/ansible/check-kolla-images-py.yml new file mode 100644 index 000000000..ba934a222 --- /dev/null +++ b/etc/kayobe/ansible/check-kolla-images-py.yml @@ -0,0 +1,35 @@ +--- +- name: Check kolla-images.py image map and tag hierarchy + hosts: localhost + gather_facts: false + tasks: + - name: Create a temporary directory + ansible.builtin.tempfile: + state: directory + suffix: kolla-ansible + register: tempdir_result + + - name: Clone Kolla Ansible repository + ansible.builtin.git: + repo: "{{ stackhpc_kolla_ansible_source_url }}" + version: "{{ stackhpc_kolla_ansible_source_version }}" + dest: "{{ tempdir_result.path }}" + + - name: Check image mapping + ansible.builtin.command: + cmd: >- + {{ kayobe_config_path }}/../../tools/kolla-images.py + check-image-map + --kolla-ansible-path {{ tempdir_result.path }} + + - name: Check tag hierarchy + ansible.builtin.command: + cmd: >- + {{ kayobe_config_path }}/../../tools/kolla-images.py + check-hierarchy + --kolla-ansible-path {{ tempdir_result.path }} + + - name: Remove temporary directory + ansible.builtin.file: + path: "{{ tempdir_result.path }}" + state: absent diff --git a/etc/kayobe/ansible/check-tags.yml b/etc/kayobe/ansible/check-tags.yml index dc429a7cd..bdfb294da 100644 --- a/etc/kayobe/ansible/check-tags.yml +++ b/etc/kayobe/ansible/check-tags.yml @@ -4,6 +4,7 @@ - name: Check whether tags exist in Pulp container registry hosts: localhost + gather_facts: false tasks: - name: Query images and tags command: diff --git a/tools/kolla-images.py b/tools/kolla-images.py index 06e67cb1c..34ea9d6b8 100755 --- a/tools/kolla-images.py +++ b/tools/kolla-images.py @@ -39,15 +39,20 @@ # Maps a Kolla image to a list of containers that use the image. IMAGE_TO_CONTAINERS_EXCEPTIONS: Dict[str, List[str]] = { + "dnsmasq": [ + "ironic_dnsmasq", + ], "haproxy": [ "glance_tls_proxy", + "haproxy", "neutron_tls_proxy", ], "mariadb-server": [ "mariadb", "mariabackup", ], - "neutron-eswitchd": [ + "neutron-mlnx-agent": [ + "neutron_eswitchd", "neutron_mlnx_agent", ], "neutron-metadata-agent": [ @@ -58,6 +63,15 @@ "nova_super_conductor", "nova_conductor", ], + "openvswitch-db-server": [ + "openvswitch_db", + ], + "ovn-nb-db-server": [ + "ovn_nb_db", + ], + "ovn-sb-db-server": [ + "ovn_sb_db", + ], "prometheus-v2-server": [ "prometheus_server", ], @@ -96,6 +110,9 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--base-distros", default=",".join(SUPPORTED_BASE_DISTROS), choices=SUPPORTED_BASE_DISTROS) subparsers = parser.add_subparsers(dest="command", required=True) + subparser = subparsers.add_parser("check-image-map", help="Check image mapping against kolla-ansible") + subparser.add_argument("--kolla-ansible-path", required=True, help="Path to kolla-ansible repostory checked out to correct branch") + subparser = subparsers.add_parser("check-hierarchy", help="Check tag variable hierarchy against kolla-ansible") subparser.add_argument("--kolla-ansible-path", required=True, help="Path to kolla-ansible repostory checked out to correct branch") @@ -114,7 +131,7 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def get_abs_path(relative_path: str) -> str: +def get_abs_path(relative_path: str) -> pathlib.Path: """Return the absolute path of a file in SKC.""" script_path = pathlib.Path(inspect.getfile(inspect.currentframe())) return script_path.parent.parent / relative_path @@ -277,6 +294,45 @@ def check_tags(base_distros: List[str], kolla_image_tags: KollaImageTags, regist sys.exit(1) +def check_image_map(kolla_ansible_path: str): + """Check the image mapping against Kolla Ansible variables. + + The *_image variables in Kolla Ansible define the mapping between + containers and images. Ensure that the mapping defined in this script + matches the one in Kolla Ansible. + """ + supported_images = read_images("etc/kayobe/pulp.yml") + assert supported_images + # Build a map from container to image name. + cmd = """git grep -h '^[a-z0-9_]*_image:' ansible/roles/*/defaults/main.yml""" + image_map_str = subprocess.check_output(cmd, shell=True, cwd=os.path.realpath(kolla_ansible_path)) + image_map = yaml.safe_load(image_map_str) + image_var_re = re.compile(r"^([a-z0-9_]+)_image$") + image_map = { + image_var_re.match(image_var).group(1): image.split("/")[-1] + for image_var, image in image_map.items() + } + # Filter out unsupported images. + image_map = { + container: image + for container, image in image_map.items() + if image in supported_images + } + assert image_map + errors = [] + # Check that our mapping is correct. + for container, image in image_map.items(): + containers = get_containers(image) + if container not in containers: + errors.append((container, image)) + if errors: + print("Errors:") + for tag_var, image in errors: + print(f"Expected {tag_var} container to use {image} image") + if errors: + sys.exit(1) + + def check_hierarchy(kolla_ansible_path: str): """Check the tag variable hierarchy against Kolla Ansible variables.""" cmd = """git grep -h '^[a-z0-9_]*_tag:' ansible/roles/*/defaults/main.yml""" @@ -352,7 +408,9 @@ def main(): validate(kolla_image_tags) - if args.command == "check-hierarchy": + if args.command == "check-image-map": + check_image_map(args.kolla_ansible_path) + elif args.command == "check-hierarchy": check_hierarchy(args.kolla_ansible_path) elif args.command == "check-tags": check_tags(base_distros, kolla_image_tags, args.registry, args.namespace)