From 33b2a585db4b11183f1083725d147143bf8abf3b Mon Sep 17 00:00:00 2001 From: Yuxiang Zhu Date: Mon, 19 Sep 2022 16:17:44 +0800 Subject: [PATCH] ART-4807: Add OSBS 2 builder - Add a builder to build container images using OSBS 2. Setting `default_image_build_method: osbs2` in group config to enable it. - Remove some dead code. - Remove hard-coded CA path for requests for better portability. It is not required on buildvm anymore. - Bump base image of devcontainer to Fedora 36. --- .devcontainer/dev.Dockerfile | 13 +- .devcontainer/devcontainer.json | 7 +- .devcontainer/settings.yaml | 2 +- doozerlib/__init__.py | 4 +- doozerlib/cli/__main__.py | 3 - doozerlib/cli/cli_opts.py | 2 +- doozerlib/cli/rpms_build.py | 2 - doozerlib/constants.py | 1 + doozerlib/distgit.py | 60 ++++--- doozerlib/osbs2_builder.py | 168 ++++++++++++++++++ doozerlib/rpmcfg.py | 17 -- doozerlib/runtime.py | 3 + setup.py | 9 +- tests/test_assertion.py | 2 +- .../test_image_distgit/test_image_distgit.py | 22 ++- tests/test_osbs2.py | 105 +++++++++++ 16 files changed, 355 insertions(+), 65 deletions(-) create mode 100644 doozerlib/osbs2_builder.py create mode 100644 tests/test_osbs2.py diff --git a/.devcontainer/dev.Dockerfile b/.devcontainer/dev.Dockerfile index aad0e9ccc..b4d501524 100644 --- a/.devcontainer/dev.Dockerfile +++ b/.devcontainer/dev.Dockerfile @@ -1,4 +1,7 @@ -FROM fedora:35 +FROM registry.fedoraproject.org/fedora:36 +LABEL name="doozer-dev" \ + description="Doozer development container image" \ + maintainer="OpenShift Automated Release Tooling (ART) Team " # Trust the Red Hat IT Root CA and set up rcm-tools repo RUN curl -o /etc/pki/ca-trust/source/anchors/RH-IT-Root-CA.crt --fail -L \ @@ -9,12 +12,12 @@ RUN curl -o /etc/pki/ca-trust/source/anchors/RH-IT-Root-CA.crt --fail -L \ RUN dnf install -y \ # runtime dependencies - krb5-workstation git tig rsync koji skopeo podman docker tito \ - python3.8 python3-certifi python3-rpm python3-kobo-rpmlib \ - # provides en_US.UTF-8 locale required by tito + krb5-workstation git tig rsync koji skopeo podman docker \ + python3.8 python3-certifi \ + # provides en_US.UTF-8 locale glibc-langpack-en \ # development dependencies - gcc krb5-devel \ + gcc gcc-c++ krb5-devel \ python3-devel python3-pip python3-wheel \ # other tools for development and troubleshooting bash-completion vim tmux procps-ng psmisc wget net-tools iproute socat \ diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 20d11bd19..3ad7c0c21 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -15,7 +15,6 @@ // This will ignore your local shell user setting for Linux since shells like zsh are typically // not in base container images. You can also update this to an specific shell to ensure VS Code // uses the right one for terminals and tasks. For example, /bin/bash (or /bin/ash for Alpine). - "terminal.integrated.shell.linux": "/bin/bash", "python.pythonPath": "/usr/bin/python3", "python.linting.pylintEnabled": false, "python.linting.flake8Enabled": true, @@ -29,16 +28,14 @@ ], "python.testing.pytestEnabled": false, "python.testing.nosetestsEnabled": false, - "python.testing.unittestEnabled": true, - "python.jediEnabled": false, // false to use Microsoft Python Language Server. This requires .NET Core 2.1+ - "python.languageServer": "Pylance", + "python.testing.unittestEnabled": true }, // Uncomment the next line if you want to publish any ports. // "appPort": [], // Uncomment the next line to run commands after the container is created - for example installing git. - "postCreateCommand": "sudo chown -R dev: /workspaces/doozer-working-dir && pip3 install --user -r requirements-dev.txt -e .", + "postCreateCommand": "sudo chown -R dev: /workspaces/doozer-working-dir", // Add the IDs of extensions you want installed when the container is created in the array below. "extensions": [ diff --git a/.devcontainer/settings.yaml b/.devcontainer/settings.yaml index 89ea87ce8..cddba998a 100644 --- a/.devcontainer/settings.yaml +++ b/.devcontainer/settings.yaml @@ -7,5 +7,5 @@ data_path: https://github.com/openshift/ocp-build-data.git #Sub-group directory or branch to pull build data # group: openshift-4.0 -#Username for running rhpkg / brew / tito +#Username for running rhpkg / brew # user: dev diff --git a/doozerlib/__init__.py b/doozerlib/__init__.py index 4e5026a15..df1650864 100644 --- a/doozerlib/__init__.py +++ b/doozerlib/__init__.py @@ -1,7 +1,7 @@ import os import sys -if sys.version_info < (3, 6): - sys.exit('Sorry, Python < 3.6 is not supported.') +if sys.version_info < (3, 8): + sys.exit('Sorry, Python < 3.8 is not supported.') from setuptools_scm import get_version diff --git a/doozerlib/cli/__main__.py b/doozerlib/cli/__main__.py index 2c394708a..929abb5f5 100644 --- a/doozerlib/cli/__main__.py +++ b/doozerlib/cli/__main__.py @@ -2228,9 +2228,6 @@ def rebase_and_build(operator): def main(): try: - if 'REQUESTS_CA_BUNDLE' not in os.environ: - os.environ['REQUESTS_CA_BUNDLE'] = '/etc/pki/tls/certs/ca-bundle.crt' - cli(obj={}) except DoozerFatalError as ex: # Allow capturing actual tool errors and print them diff --git a/doozerlib/cli/cli_opts.py b/doozerlib/cli/cli_opts.py index 3456e5dfa..4e22e7b16 100644 --- a/doozerlib/cli/cli_opts.py +++ b/doozerlib/cli/cli_opts.py @@ -33,7 +33,7 @@ def global_opt_default_string(): }, 'user': { 'env': 'DOOZER_USER', - 'help': 'Username for running rhpkg / brew / tito' + 'help': 'Username for running rhpkg / brew' }, 'global_opts': { 'help': 'Global option overrides that can only be set from settings.yaml', diff --git a/doozerlib/cli/rpms_build.py b/doozerlib/cli/rpms_build.py index 524146eb9..cf830f88d 100644 --- a/doozerlib/cli/rpms_build.py +++ b/doozerlib/cli/rpms_build.py @@ -13,7 +13,6 @@ from doozerlib.runtime import Runtime -# This command reimplements `rpms:build` without tito. Rename it to `rpms:build` when getting to prod. @cli.command("rpms:rebase-and-build", help="Rebase and build rpms in the group or given by --rpms.") @click.option("--version", metavar='VERSION', default=None, callback=validate_semver_major_minor_patch, help="Version string to populate in specfile.", required=True) @@ -165,7 +164,6 @@ async def _rebase_rpm(runtime: Runtime, builder: RPMBuilder, rpm: RPMMetadata, v return record["status"] -# This command reimplements `rpms:build` without tito. Rename it to `rpms:build` when getting to prod. @cli.command("rpms:build", help="Build rpms in the group or given by --rpms.") @click.option('--scratch', default=False, is_flag=True, help='Perform a scratch build.') @click.option('--dry-run', default=False, is_flag=True, help='Do not build anything, but only print build operations.') diff --git a/doozerlib/constants.py b/doozerlib/constants.py index 5d0ba4068..8997974b9 100644 --- a/doozerlib/constants.py +++ b/doozerlib/constants.py @@ -8,6 +8,7 @@ GITHUB_TOKEN = "GITHUB_TOKEN" BREWWEB_URL = "https://brewweb.engineering.redhat.com/brew" +DISTGIT_GIT_URL = "git://pkgs.devel.redhat.com" # Environment variables that should be set for doozer interaction with db for storing and retrieving build records. # DB ENV VARS diff --git a/doozerlib/distgit.py b/doozerlib/distgit.py index d42ca0b46..2f4f644bf 100644 --- a/doozerlib/distgit.py +++ b/doozerlib/distgit.py @@ -10,6 +10,7 @@ import re import shutil import sys +import threading import time import traceback from datetime import date @@ -21,17 +22,19 @@ import requests import yaml from dockerfile_parse import DockerfileParser -from doozerlib.rpm_utils import parse_nvr from tenacity import (before_sleep_log, retry, retry_if_not_result, stop_after_attempt, wait_fixed) +import doozerlib from doozerlib import assertion, constants, exectools, logutil, state, util from doozerlib.assembly import AssemblyTypes from doozerlib.brew import get_build_objects, watch_task from doozerlib.dblib import Record from doozerlib.exceptions import DoozerFatalError from doozerlib.model import ListModel, Missing, Model +from doozerlib.osbs2_builder import OSBS2Builder from doozerlib.pushd import Dir +from doozerlib.rpm_utils import parse_nvr from doozerlib.source_modifications import SourceModifierFactory from doozerlib.util import convert_remote_git_to_https, yellow_print @@ -94,7 +97,7 @@ class DistGitRepo(object): def __init__(self, metadata, autoclone=True): self.metadata = metadata self.config: Model = metadata.config - self.runtime = metadata.runtime + self.runtime: "doozerlib.runtime.Runtime" = metadata.runtime self.name: str = self.metadata.name self.distgit_dir: str = None self.dg_path: pathlib.Path = None @@ -196,7 +199,7 @@ def clone(self, distgits_root_dir, distgit_branch): timeout = str(self.runtime.global_opts['rhpkg_clone_timeout']) rhpkg_clone_depth = int(self.runtime.global_opts.get('rhpkg_clone_depth', '0')) - if self.metadata.namespace == 'containers' and self.runtime.cache_dir: + if self.metadata.namespace == 'containers': # Containers don't generally require distgit lookaside. We can rely on normal # git clone & leverage git caches to greatly accelerate things if the user supplied it. gitargs = ['--branch', distgit_branch] @@ -421,6 +424,10 @@ def __init__(self, metadata, autoclone=True, self.logger: logging.Logger = metadata.logger self.source_modifier_factory = source_modifier_factory + self.org_image_name = None + self.org_version = None + self.org_release = None + def clone(self, distgits_root_dir, distgit_branch): super(ImageDistGitRepo, self).clone(distgits_root_dir, distgit_branch) self._read_master_data() @@ -432,10 +439,7 @@ def _get_diff(self): @property def image_build_method(self): - build_method = self.runtime.group_config.default_image_build_method - # If the build is multistage, override with 'imagebuilder' as required for multistage. - if 'builder' in self.config.get('from', {}): - build_method = 'imagebuilder' + build_method = self.runtime.group_config.default_image_build_method or "imagebuilder" # If our config specifies something, override with that. if self.config.image_build_method is not Missing: build_method = self.config.image_build_method @@ -567,7 +571,7 @@ def _generate_osbs_image_config(self, version: str) -> Dict: ] }) - if self.image_build_method is not Missing: + if self.image_build_method is not Missing and self.image_build_method != "osbs2": config_overrides['image_build_method'] = self.image_build_method if arches: @@ -998,6 +1002,7 @@ def build_container( if self.config.wait_for is not Missing: self._set_wait_for(self.config.wait_for, terminate_event) + push_version, push_release = ('', '') if self.runtime.local: self.build_status = self._build_container_local(target_image, profile["repo_type"], realtime) if not self.build_status: @@ -1029,20 +1034,31 @@ def wait(n): raise DoozerFatalError("Building images against multiple targets is not currently supported.") target = self.metadata.targets[0] - exectools.retry( - retries=retries, wait_f=wait, - task_f=lambda: self._build_container( - target_image, target, profile["signing_intent"], profile["repo_type"], profile["repo_list"], terminate_event, - scratch, record, dry_run=dry_run)) - - # Just in case someone else is building an image, go ahead and find what was just - # built so that push_image will have a fixed point of reference and not detect any - # subsequent builds. - push_version, push_release = ('', '') - if not dry_run and not scratch: - nvr = parse_nvr(record["nvrs"].split(",")[0]) - push_version = nvr["version"] - push_release = nvr["release"] + if self.image_build_method == "osbs2": # use OSBS 2 + osbs2 = OSBS2Builder(self.runtime, scratch=scratch, dry_run=dry_run) + task_id, task_url, nvr = osbs2.build(self.metadata, profile, retries=retries) + record["task_id"] = task_id + record["task_url"] = task_url + record["nvrs"] = nvr + if not dry_run: + self.update_build_db(True, task_id=task_id, scratch=scratch) + if not scratch: + nvr_dict = parse_nvr(nvr) + push_version = nvr_dict["version"] + push_release = nvr_dict["release"] + else: # use OSBS 1 + exectools.retry( + retries=retries, wait_f=wait, + task_f=lambda: self._build_container( + target_image, target, profile["signing_intent"], profile["repo_type"], profile["repo_list"], terminate_event, + scratch, record, dry_run=dry_run)) + # Just in case someone else is building an image, go ahead and find what was just + # built so that push_image will have a fixed point of reference and not detect any + # subsequent builds. + if not dry_run and not scratch: + nvr_dict = parse_nvr(record["nvrs"].split(",")[0]) + push_version = nvr_dict["version"] + push_release = nvr_dict["release"] record["message"] = "Success" record["status"] = 0 self.build_status = True diff --git a/doozerlib/osbs2_builder.py b/doozerlib/osbs2_builder.py new file mode 100644 index 000000000..df9746f81 --- /dev/null +++ b/doozerlib/osbs2_builder.py @@ -0,0 +1,168 @@ +import re +import threading +import traceback +from time import sleep +from typing import Dict +from urllib.parse import quote + +import koji + +from doozerlib import brew, distgit, exectools, image, runtime +from doozerlib.constants import BREWWEB_URL, DISTGIT_GIT_URL +from doozerlib.exceptions import DoozerFatalError + + +class OSBS2Builder: + """ Builds container images with OSBS 2 + """ + + def __init__( + self, runtime: "runtime.Runtime", *, scratch: bool = False, dry_run: bool = False + ) -> None: + """ Create a OSBS2Builder instance. + :param runtime: Doozer runtime + :param scratch: Whether to create a scratch build + :param dry_run: Don't build anything but just exercise the code + """ + self._runtime = runtime + self.scratch = scratch + self.dry_run = dry_run + + def build(self, image: "image.ImageMetadata", profile: Dict, retries: int = 3): + dg: "distgit.ImageDistGitRepo" = image.distgit_repo() + logger = dg.logger + nvr = f"{dg.name}-{dg.org_version}-{dg.org_release}" + if len(image.targets) > 1: + # Currently we don't really support building images against multiple targets, + # or we would overwrite the image tag when pushing to the registry. + # `targets` is defined as an array just because we want to keep consistency with RPM build. + raise DoozerFatalError("Building images against multiple targets is not currently supported.") + target = image.targets[0] + logger.warning("[BETA] OSBS 2: Building image %s...", nvr) + koji_api = self._runtime.build_retrying_koji_client() + task_id = 0 + task_url = None + + for attempt in range(retries): + logger.info("Build attempt %s/%s", attempt + 1, retries) + error = None + try: + # Submit build task + task_id, task_url = self._start_build(dg, target, profile, koji_api) + logger.info("Waiting for build task %s to complete...", task_id) + if self.dry_run: + logger.warning("[DRY RUN] Build task %s would have completed", task_id) + error = None + else: + error = brew.watch_task(koji_api, logger.info, task_id, terminate_event=threading.Event()) + + # Gather brew-logs + cmd = ["brew", "download-logs", "--recurse", "-d", dg._logs_dir(), task_id] + if self.dry_run: + logger.warning("[DRY RUN] Would have downloaded Brew logs with %s", cmd) + else: + logs_rc, _, logs_err = exectools.cmd_gather(cmd) + if logs_rc != 0: + logger.warning("Error downloading build logs from brew for task %s: %s", task_id, logs_err) + + # Get build ID + build_id = 0 + build_info = None + if error: # Build failed + # Looking for something like the following to conclude the image has already been built: + # BuildError: Build for openshift-enterprise-base-v3.7.0-0.117.0.0 already exists, id 588961 + # Note it is possible that a Brew task fails with a build record left (https://issues.redhat.com/browse/ART-1723). + # Didn't find a variable in the context to get the Brew NVR or ID. Extracting the build ID from the error message. + # Hope the error message format will not change. + match = re.search(r"already exists, id (\d+)", error) + if match: + build_id = int(match[1]) + builds = brew.get_build_objects([build_id], koji_api) + if builds and builds[0] and builds[0].get('state') == 1: # State 1 means complete. + build_info = builds[0] + build_url = f"{BREWWEB_URL}/buildinfo?buildID={build_info['id']}" + logger.info("Image %s already built against this dist-git commit (or version-release tag): %s", nvr, build_url) + error = None # Treat as success + else: # Build succeeded + if self.dry_run: + build_id = 0 + build_info = {"id": build_id, "nvr": nvr} + else: + # Unlike rpm build, koji_api.listBuilds(taskID=...) doesn't support image build. For now, let's use a different approach. + taskResult = koji_api.getTaskResult(task_id) + build_id = int(taskResult["koji_builds"][0]) + build_info = koji_api.getBuild(build_id) + build_url = f"{BREWWEB_URL}/buildinfo?buildID={build_info['id']}" + except Exception as err: + error = f"Error building image {nvr}: {str(err)}: {traceback.format_exc()}" + + if error: + # An error occurred. We don't have a viable build. + message = f"Build failed: {error}" + logger.warning( + "Error building rpm %s [attempt #%s] in Brew: %s", + image.name, + attempt + 1, + message, + ) + if attempt < retries - 1: + # Brew does not handle an immediate retry correctly, wait before trying another build + logger.info("Will retry in 5 minutes") + sleep(5 * 60) + continue + + if self._runtime.hotfix: + # Tag the image so they don't get garbage collected. + logger.info(f'Tagging {image.get_component_name()} build {build_info["nvr"]} with {image.hotfix_brew_tag()} to prevent garbage collection') + if self.dry_run: + logger.warning("[DRY RUN] Build %s would have been tagged into %s", nvr, image.hotfix_brew_tag()) + else: + if not koji_api.logged_in: + koji_api.gssapi_login() + koji_api.tagBuild(image.hotfix_brew_tag(), build_info["nvr"]) + logger.warning("Build %s has been tagged into %s", nvr, image.hotfix_brew_tag()) + + logger.info("Successfully built image %s; task: %s; build record: %s", nvr, task_url, build_url) + return task_id, task_url, nvr + + def _start_build(self, dg: "distgit.ImageDistGitRepo", target: str, profile: Dict, koji_api: koji.ClientSession): + logger = dg.logger + src = self._construct_build_source_url(dg) + signing_intent = profile["signing_intent"] + repo_type = profile["repo_type"] + repo_list = profile["repo_list"] + if repo_type and not repo_list: # If --repo was not specified on the command line + repo_file = f".oit/{repo_type}.repo" + if not self.dry_run: + existence, repo_url = dg.cgit_file_available(repo_file) + else: + logger.warning("[DRY RUN] Would have checked if cgit repo file is present.") + existence, repo_url = True, f"https://cgit.example.com/{repo_file}" + if not existence: + raise FileNotFoundError(f"Repo file {repo_file} is not available on cgit; cgit cache may not be reflecting distgit in a timely manner.") + repo_list = [repo_url] + + opts = { + 'scratch': self.scratch, + 'signing_intent': signing_intent, + 'yum_repourls': repo_list, + 'git_branch': dg.branch, + } + + task_id = 0 + logger.info("Starting OSBS 2 build with source %s and target %s...", src, target) + if self.dry_run: + logger.warning("[DRY RUN] Would have started container build") + else: + if not koji_api.logged_in: + koji_api.gssapi_login() + task_id: int = koji_api.buildContainer(src, target, opts=opts, channel="container-binary") + + task_url = f"{BREWWEB_URL}/taskinfo?taskID={task_id}" + logger.info("OSBS 2 Task ID: %s, url: %s", task_id, task_url) + return task_id, task_url + + def _construct_build_source_url(self, dg: "distgit.ImageDistGitRepo"): + if not dg.sha: + raise ValueError(f"Image {dg.name} Distgit commit sha is unknown") + return f"{DISTGIT_GIT_URL}/containers/{quote(dg.name)}#{quote(dg.sha)}" diff --git a/doozerlib/rpmcfg.py b/doozerlib/rpmcfg.py index 84cbe11d9..4dff894cc 100644 --- a/doozerlib/rpmcfg.py +++ b/doozerlib/rpmcfg.py @@ -15,23 +15,6 @@ from .model import Missing from .pushd import Dir -RELEASERS_CONF = """ -[{target}] -releaser = tito.release.DistGitReleaser -branches = {branch} -build_targets = {branch}:{brew_target} -srpm_disttag = .el7aos -builder.test = 1 -remote_git_name = {name} -""" - -# note; appended to, does not replace existing props -TITO_PROPS = """ - -[{target}] -remote_git_name = {name} -""" - class RPMMetadata(Metadata): diff --git a/doozerlib/runtime.py b/doozerlib/runtime.py index f6547596a..83753057e 100644 --- a/doozerlib/runtime.py +++ b/doozerlib/runtime.py @@ -478,9 +478,11 @@ def initialize(self, mode='images', clone_distgits=True, self.assembly_type = assembly_type(self.get_releases_config(), self.assembly) if not self.brew_event: + self.logger.info("Basis brew event is not set. Using the latest event....") with self.shared_koji_client_session() as koji_session: # If brew event is not set as part of the assembly and not specified on the command line, # lock in an event so that there are no race conditions. + self.logger.info("Getting the latest event....") event_info = koji_session.getLastEvent() self.brew_event = event_info['id'] @@ -741,6 +743,7 @@ def shared_koji_client_session(self): if self._koji_client_session is None: self._koji_client_session = self.build_retrying_koji_client() if not self.disable_gssapi: + self.logger.info("Authenticating to Brew...") self._koji_client_session.gssapi_login() yield self._koji_client_session diff --git a/setup.py b/setup.py index 23039c87d..8c98b6e79 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- from setuptools import setup, find_packages import sys -if sys.version_info < (3, 6): - sys.exit('Sorry, Python < 3.6 is not supported.') +if sys.version_info < (3, 8): + sys.exit('Sorry, Python < 3.8 is not supported.') with open('./requirements.txt') as f: INSTALL_REQUIRES = f.read().splitlines() @@ -28,14 +28,13 @@ install_requires=INSTALL_REQUIRES, test_suite='tests', dependency_links=[], - python_requires='>=3.6', + python_requires='>=3.8', classifiers=[ "Development Status :: 5 - Production/Stable", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Environment :: Console", "Operating System :: POSIX", "License :: OSI Approved :: Apache Software License", diff --git a/tests/test_assertion.py b/tests/test_assertion.py index 8c5dc67da..8e9072a21 100644 --- a/tests/test_assertion.py +++ b/tests/test_assertion.py @@ -37,7 +37,7 @@ def test_isfile(self): """ Verify both positive and negative results for file test """ - file_exists = "/etc/motd" + file_exists = "/etc/hosts" file_missing = "/tmp/doesnotexist" not_file = "/usr" diff --git a/tests/test_distgit/test_image_distgit/test_image_distgit.py b/tests/test_distgit/test_image_distgit/test_image_distgit.py index 06a38d615..0a55a70da 100644 --- a/tests/test_distgit/test_image_distgit/test_image_distgit.py +++ b/tests/test_distgit/test_image_distgit/test_image_distgit.py @@ -83,7 +83,7 @@ def test_image_build_method_default(self): def test_image_build_method_imagebuilder(self): get = lambda key, default: dict({"builder": "..."}) if key == "from" else default - metadata = flexmock(runtime=self.mock_runtime(group_config=flexmock(default_image_build_method="default-method")), + metadata = flexmock(runtime=self.mock_runtime(group_config=flexmock(default_image_build_method=distgit.Missing)), config=flexmock(distgit=flexmock(branch=distgit.Missing), image_build_method=distgit.Missing, get=get), @@ -93,6 +93,26 @@ def test_image_build_method_imagebuilder(self): repo = distgit.ImageDistGitRepo(metadata, autoclone=False) self.assertEqual("imagebuilder", repo.image_build_method) + metadata = flexmock(runtime=self.mock_runtime(group_config=flexmock(default_image_build_method="osbs2")), + config=flexmock(distgit=flexmock(branch=distgit.Missing), + image_build_method=distgit.Missing, + get=get), + name="_irrelevant_", + logger="_irrelevant_") + + repo = distgit.ImageDistGitRepo(metadata, autoclone=False) + self.assertEqual("osbs2", repo.image_build_method) + + metadata = flexmock(runtime=self.mock_runtime(group_config=flexmock(default_image_build_method="osbs2")), + config=flexmock(distgit=flexmock(branch=distgit.Missing), + image_build_method="imagebuilder", + get=get), + name="_irrelevant_", + logger="_irrelevant_") + + repo = distgit.ImageDistGitRepo(metadata, autoclone=False) + self.assertEqual("imagebuilder", repo.image_build_method) + def test_image_build_method_from_config(self): metadata = flexmock(runtime=self.mock_runtime(group_config=flexmock(default_image_build_method="default-method")), config=flexmock(distgit=flexmock(branch=distgit.Missing), diff --git a/tests/test_osbs2.py b/tests/test_osbs2.py new file mode 100644 index 000000000..4d149f7f6 --- /dev/null +++ b/tests/test_osbs2.py @@ -0,0 +1,105 @@ +import unittest +from unittest.mock import ANY, MagicMock, patch + +from doozerlib import constants +from doozerlib.distgit import ImageDistGitRepo +from doozerlib.gitdata import DataObj +from doozerlib.image import ImageMetadata +from doozerlib.osbs2_builder import OSBS2Builder + + +class TestOSBS2Builder(unittest.TestCase): + + def _make_image_meta(self, runtime): + data_obj = DataObj("foo", "/path/to/ocp-build-data/images/foo.yml", { + "name": "foo", + "content": { + "source": { + "git": {"url": "git@github.com:openshift-priv/foo.git", "branch": {"target": "release-4.8"}}, + } + }, + "targets": ["rhaos-4.12-rhel-8-containers-candidate"], + }) + meta = ImageMetadata(runtime, data_obj, clone_source=False, prevent_cloning=True) + meta.branch = MagicMock(return_value="rhaos-4.12-rhel-8") + return meta + + def test_construct_build_source_url(self): + runtime = MagicMock() + osbs2 = OSBS2Builder(runtime) + meta = self._make_image_meta(runtime) + dg = ImageDistGitRepo(meta, autoclone=False) + + with self.assertRaises(ValueError): + dg.sha = None + actual = osbs2._construct_build_source_url(dg) + + dg.sha = "deadbeef" + actual = osbs2._construct_build_source_url(dg) + self.assertEqual(actual, f"{constants.DISTGIT_GIT_URL}/containers/foo#deadbeef") + + def test_start_build(self): + runtime = MagicMock() + osbs2 = OSBS2Builder(runtime) + meta = self._make_image_meta(runtime) + dg = ImageDistGitRepo(meta, autoclone=False) + dg.sha = "deadbeef" + dg.branch = "rhaos-4.12-rhel-8" + dg.cgit_file_available = MagicMock(return_value=(True, "http://cgit.example.com/foo.repo")) + profile = { + "signing_intent": "release", + "repo_type": "signed", + "repo_list": [], + } + koji_api = MagicMock(logged_in=False) + koji_api.buildContainer.return_value = 12345 + + task_id, task_url = osbs2._start_build(dg, "rhaos-4.12-rhel-8-containers-candidate", profile, koji_api) + dg.cgit_file_available.assert_called_once_with(".oit/signed.repo") + koji_api.gssapi_login.assert_called_once_with() + koji_api.buildContainer.assert_called_once_with( + f"{constants.DISTGIT_GIT_URL}/containers/foo#deadbeef", + "rhaos-4.12-rhel-8-containers-candidate", + opts={ + 'scratch': False, + 'signing_intent': "release", + 'yum_repourls': ["http://cgit.example.com/foo.repo"], + 'git_branch': "rhaos-4.12-rhel-8", + }, + channel="container-binary") + self.assertEqual(task_id, 12345) + self.assertEqual(task_url, f"{constants.BREWWEB_URL}/taskinfo?taskID=12345") + + @patch("doozerlib.exectools.cmd_gather", return_value=(0, "", "")) + @patch("doozerlib.brew.watch_task", return_value=None) + @patch("doozerlib.osbs2_builder.OSBS2Builder._start_build", return_value=(12345, f"{constants.BREWWEB_URL}/taskinfo?taskID=12345")) + def test_build(self, _start_build: MagicMock, watch_task: MagicMock, cmd_gather: MagicMock): + koji_api = MagicMock(logged_in=False) + koji_api.getTaskResult = MagicMock(return_value={"koji_builds": [42]}) + koji_api.getBuild = MagicMock(return_value={"id": 42, "nvr": "foo-v4.12.0-12345.p0.assembly.test"}) + runtime = MagicMock() + runtime.build_retrying_koji_client = MagicMock(return_value=koji_api) + osbs2 = OSBS2Builder(runtime) + meta = self._make_image_meta(runtime) + dg = ImageDistGitRepo(meta, autoclone=False) + meta.distgit_repo = MagicMock(return_value=dg) + dg.sha = "deadbeef" + dg.branch = "rhaos-4.12-rhel-8" + dg.org_version = "v4.12.0" + dg.org_release = "12345.p0.assembly.test" + profile = { + "signing_intent": "release", + "repo_type": "signed", + "repo_list": [], + } + + task_id, task_url, nvr = osbs2.build(meta, profile, retries=1) + self.assertEqual((task_id, task_url, nvr), (12345, f"{constants.BREWWEB_URL}/taskinfo?taskID=12345", "foo-v4.12.0-12345.p0.assembly.test")) + koji_api.gssapi_login.assert_called_once_with() + koji_api.getTaskResult.assert_called_once_with(12345) + koji_api.getBuild.assert_called_once_with(42) + koji_api.tagBuild.assert_called_once_with('rhaos-4.12-rhel-8-hotfix', "foo-v4.12.0-12345.p0.assembly.test") + runtime.build_retrying_koji_client.assert_called_once_with() + _start_build.assert_called_once_with(dg, 'rhaos-4.12-rhel-8-containers-candidate', {'signing_intent': 'release', 'repo_type': 'signed', 'repo_list': []}, koji_api) + watch_task.assert_called_once_with(koji_api, ANY, 12345, terminate_event=ANY) + cmd_gather.assert_called_once_with(['brew', 'download-logs', '--recurse', '-d', ANY, 12345])