ART-4807: Add OSBS 2 builder

- Add a builder to build container images using OSBS 2. Setting `default_image_build_method: osbs2` in group config to enable it.
- Remove some dead code.
- Remove hard-coded CA path for requests for better portability. It is not required on buildvm
  anymore.
- Bump base image of devcontainer to Fedora 36.

.devcontainer/dev.Dockerfile

@@ -1,4 +1,7 @@
-FROM fedora:35
+FROM registry.fedoraproject.org/fedora:36
+LABEL name="doozer-dev" \
+  description="Doozer development container image" \
+  maintainer="OpenShift Automated Release Tooling (ART) Team <aos-team-art@redhat.com>"
 
 # Trust the Red Hat IT Root CA and set up rcm-tools repo
 RUN curl -o /etc/pki/ca-trust/source/anchors/RH-IT-Root-CA.crt --fail -L \
@@ -9,12 +12,12 @@ RUN curl -o /etc/pki/ca-trust/source/anchors/RH-IT-Root-CA.crt --fail -L \
 
 RUN dnf install -y \
     # runtime dependencies
-    krb5-workstation git tig rsync koji skopeo podman docker tito \
-    python3.8 python3-certifi python3-rpm python3-kobo-rpmlib \
-    # provides en_US.UTF-8 locale required by tito
+    krb5-workstation git tig rsync koji skopeo podman docker \
+    python3.8 python3-certifi \
+    # provides en_US.UTF-8 locale
     glibc-langpack-en \
     # development dependencies
-    gcc krb5-devel \
+    gcc gcc-c++ krb5-devel \
     python3-devel python3-pip python3-wheel \
     # other tools for development and troubleshooting
     bash-completion vim tmux procps-ng psmisc wget net-tools iproute socat \

.devcontainer/devcontainer.json

@@ -15,7 +15,6 @@
 		// This will ignore your local shell user setting for Linux since shells like zsh are typically
 		// not in base container images. You can also update this to an specific shell to ensure VS Code
 		// uses the right one for terminals and tasks. For example, /bin/bash (or /bin/ash for Alpine).
-		"terminal.integrated.shell.linux": "/bin/bash",
 		"python.pythonPath": "/usr/bin/python3",
 		"python.linting.pylintEnabled": false,
 		"python.linting.flake8Enabled": true,
@@ -29,16 +28,14 @@
 		],
 		"python.testing.pytestEnabled": false,
 		"python.testing.nosetestsEnabled": false,
-		"python.testing.unittestEnabled": true,
-		"python.jediEnabled": false,  // false to use Microsoft Python Language Server. This requires .NET Core 2.1+
-		"python.languageServer": "Pylance",
+		"python.testing.unittestEnabled": true
 	},
 
 	// Uncomment the next line if you want to publish any ports.
 	// "appPort": [],
 
 	// Uncomment the next line to run commands after the container is created - for example installing git.
-	"postCreateCommand": "sudo chown -R dev: /workspaces/doozer-working-dir && pip3 install --user -r requirements-dev.txt -e .",
+	"postCreateCommand": "sudo chown -R dev: /workspaces/doozer-working-dir",
 
 	// Add the IDs of extensions you want installed when the container is created in the array below.
 	"extensions": [

.devcontainer/settings.yaml

@@ -7,5 +7,5 @@ data_path: https://github.com/openshift/ocp-build-data.git
 #Sub-group directory or branch to pull build data
 # group: openshift-4.0
 
-#Username for running rhpkg / brew / tito
+#Username for running rhpkg / brew
 # user: dev

doozerlib/__init__.py

@@ -1,7 +1,7 @@
 import os
 import sys
-if sys.version_info < (3, 6):
-    sys.exit('Sorry, Python < 3.6 is not supported.')
+if sys.version_info < (3, 8):
+    sys.exit('Sorry, Python < 3.8 is not supported.')
 
 from setuptools_scm import get_version
 

doozerlib/cli/__main__.py

@@ -2228,9 +2228,6 @@ def rebase_and_build(operator):
 
 def main():
     try:
-        if 'REQUESTS_CA_BUNDLE' not in os.environ:
-            os.environ['REQUESTS_CA_BUNDLE'] = '/etc/pki/tls/certs/ca-bundle.crt'
-
         cli(obj={})
     except DoozerFatalError as ex:
         # Allow capturing actual tool errors and print them

doozerlib/cli/cli_opts.py

@@ -33,7 +33,7 @@ def global_opt_default_string():
     },
     'user': {
         'env': 'DOOZER_USER',
-        'help': 'Username for running rhpkg / brew / tito'
+        'help': 'Username for running rhpkg / brew'
     },
     'global_opts': {
         'help': 'Global option overrides that can only be set from settings.yaml',

doozerlib/cli/rpms_build.py

@@ -13,7 +13,6 @@
 from doozerlib.runtime import Runtime
 
 
-# This command reimplements `rpms:build` without tito. Rename it to `rpms:build` when getting to prod.
 @cli.command("rpms:rebase-and-build", help="Rebase and build rpms in the group or given by --rpms.")
 @click.option("--version", metavar='VERSION', default=None, callback=validate_semver_major_minor_patch,
               help="Version string to populate in specfile.", required=True)
@@ -165,7 +164,6 @@ async def _rebase_rpm(runtime: Runtime, builder: RPMBuilder, rpm: RPMMetadata, v
     return record["status"]
 
 
-# This command reimplements `rpms:build` without tito. Rename it to `rpms:build` when getting to prod.
 @cli.command("rpms:build", help="Build rpms in the group or given by --rpms.")
 @click.option('--scratch', default=False, is_flag=True, help='Perform a scratch build.')
 @click.option('--dry-run', default=False, is_flag=True, help='Do not build anything, but only print build operations.')

doozerlib/constants.py

@@ -8,6 +8,7 @@
 
 GITHUB_TOKEN = "GITHUB_TOKEN"
 BREWWEB_URL = "https://brewweb.engineering.redhat.com/brew"
+DISTGIT_GIT_URL = "git://pkgs.devel.redhat.com"
 
 # Environment variables that should be set for doozer interaction with db for storing and retrieving build records.
 # DB ENV VARS

doozerlib/distgit.py

@@ -10,6 +10,7 @@
 import re
 import shutil
 import sys
+import threading
 import time
 import traceback
 from datetime import date
@@ -21,17 +22,19 @@
 import requests
 import yaml
 from dockerfile_parse import DockerfileParser
-from doozerlib.rpm_utils import parse_nvr
 from tenacity import (before_sleep_log, retry, retry_if_not_result,
                       stop_after_attempt, wait_fixed)
 
+import doozerlib
 from doozerlib import assertion, constants, exectools, logutil, state, util
 from doozerlib.assembly import AssemblyTypes
 from doozerlib.brew import get_build_objects, watch_task
 from doozerlib.dblib import Record
 from doozerlib.exceptions import DoozerFatalError
 from doozerlib.model import ListModel, Missing, Model
+from doozerlib.osbs2_builder import OSBS2Builder
 from doozerlib.pushd import Dir
+from doozerlib.rpm_utils import parse_nvr
 from doozerlib.source_modifications import SourceModifierFactory
 from doozerlib.util import convert_remote_git_to_https, yellow_print
 
@@ -94,7 +97,7 @@ class DistGitRepo(object):
     def __init__(self, metadata, autoclone=True):
         self.metadata = metadata
         self.config: Model = metadata.config
-        self.runtime = metadata.runtime
+        self.runtime: "doozerlib.runtime.Runtime" = metadata.runtime
         self.name: str = self.metadata.name
         self.distgit_dir: str = None
         self.dg_path: pathlib.Path = None
@@ -196,7 +199,7 @@ def clone(self, distgits_root_dir, distgit_branch):
                     timeout = str(self.runtime.global_opts['rhpkg_clone_timeout'])
                     rhpkg_clone_depth = int(self.runtime.global_opts.get('rhpkg_clone_depth', '0'))
 
-                    if self.metadata.namespace == 'containers' and self.runtime.cache_dir:
+                    if self.metadata.namespace == 'containers':
                         # Containers don't generally require distgit lookaside. We can rely on normal
                         # git clone & leverage git caches to greatly accelerate things if the user supplied it.
                         gitargs = ['--branch', distgit_branch]
@@ -421,6 +424,10 @@ def __init__(self, metadata, autoclone=True,
         self.logger: logging.Logger = metadata.logger
         self.source_modifier_factory = source_modifier_factory
 
+        self.org_image_name = None
+        self.org_version = None
+        self.org_release = None
+
     def clone(self, distgits_root_dir, distgit_branch):
         super(ImageDistGitRepo, self).clone(distgits_root_dir, distgit_branch)
         self._read_master_data()
@@ -432,10 +439,7 @@ def _get_diff(self):
 
     @property
     def image_build_method(self):
-        build_method = self.runtime.group_config.default_image_build_method
-        # If the build is multistage, override with 'imagebuilder' as required for multistage.
-        if 'builder' in self.config.get('from', {}):
-            build_method = 'imagebuilder'
+        build_method = self.runtime.group_config.default_image_build_method or "imagebuilder"
         # If our config specifies something, override with that.
         if self.config.image_build_method is not Missing:
             build_method = self.config.image_build_method
@@ -567,7 +571,7 @@ def _generate_osbs_image_config(self, version: str) -> Dict:
                 ]
             })
 
-        if self.image_build_method is not Missing:
+        if self.image_build_method is not Missing and self.image_build_method != "osbs2":
             config_overrides['image_build_method'] = self.image_build_method
 
         if arches:
@@ -998,6 +1002,7 @@ def build_container(
             if self.config.wait_for is not Missing:
                 self._set_wait_for(self.config.wait_for, terminate_event)
 
+            push_version, push_release = ('', '')
             if self.runtime.local:
                 self.build_status = self._build_container_local(target_image, profile["repo_type"], realtime)
                 if not self.build_status:
@@ -1029,20 +1034,31 @@ def wait(n):
                     raise DoozerFatalError("Building images against multiple targets is not currently supported.")
                 target = self.metadata.targets[0]
 
-                exectools.retry(
-                    retries=retries, wait_f=wait,
-                    task_f=lambda: self._build_container(
-                        target_image, target, profile["signing_intent"], profile["repo_type"], profile["repo_list"], terminate_event,
-                        scratch, record, dry_run=dry_run))
-
-            # Just in case someone else is building an image, go ahead and find what was just
-            # built so that push_image will have a fixed point of reference and not detect any
-            # subsequent builds.
-            push_version, push_release = ('', '')
-            if not dry_run and not scratch:
-                nvr = parse_nvr(record["nvrs"].split(",")[0])
-                push_version = nvr["version"]
-                push_release = nvr["release"]
+                if self.image_build_method == "osbs2":  # use OSBS 2
+                    osbs2 = OSBS2Builder(self.runtime, scratch=scratch, dry_run=dry_run)
+                    task_id, task_url, nvr = osbs2.build(self.metadata, profile, retries=retries)
+                    record["task_id"] = task_id
+                    record["task_url"] = task_url
+                    record["nvrs"] = nvr
+                    if not dry_run:
+                        self.update_build_db(True, task_id=task_id, scratch=scratch)
+                        if not scratch:
+                            nvr_dict = parse_nvr(nvr)
+                            push_version = nvr_dict["version"]
+                            push_release = nvr_dict["release"]
+                else:  # use OSBS 1
+                    exectools.retry(
+                        retries=retries, wait_f=wait,
+                        task_f=lambda: self._build_container(
+                            target_image, target, profile["signing_intent"], profile["repo_type"], profile["repo_list"], terminate_event,
+                            scratch, record, dry_run=dry_run))
+                    # Just in case someone else is building an image, go ahead and find what was just
+                    # built so that push_image will have a fixed point of reference and not detect any
+                    # subsequent builds.
+                    if not dry_run and not scratch:
+                        nvr_dict = parse_nvr(record["nvrs"].split(",")[0])
+                        push_version = nvr_dict["version"]
+                        push_release = nvr_dict["release"]
             record["message"] = "Success"
             record["status"] = 0
             self.build_status = True