local_ignitionprovider.go

package controllers

import (
	"context"
	"crypto/x509"
	"crypto/x509/pkix"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/blang/semver"
	"github.com/docker/distribution/manifest/manifestlist"
	hyperv1 "github.com/openshift/hypershift/api/v1beta1"
	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/common"
	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider"
	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests"
	"github.com/openshift/hypershift/support/api"
	"github.com/openshift/hypershift/support/certs"
	"github.com/openshift/hypershift/support/releaseinfo"
	"github.com/openshift/hypershift/support/releaseinfo/registryclient"
	"github.com/openshift/hypershift/support/util"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/yaml"
)

// LocalIgnitionProvider is an IgnitionProvider that executes MCO binaries
// directly to build ignition payload contents out of a given release image and
// a config string containing 0..N MachineConfig YAML definitions.
//
// To do this, MCO binaries and other static input files are extracted from a
// release image into WorkDir. These contents are cleaned up after each
// execution and are not currently cached between executions for a given release
// image because the effort of managing the cache is not yet justified by any
// performance measurements.
//
// Currently, all GetPayload executions are performed serially, enforced by a
// mutex. Enabling concurrent executions requires more work because of the of
// MCS, which is an HTTP server process, implying work to allocate
// non-conflicting ports. This effort is not yet justified by any performance
// measurements.
type LocalIgnitionProvider struct {
	Client          client.Client
	ReleaseProvider releaseinfo.Provider
	CloudProvider   hyperv1.PlatformType
	Namespace       string

	// WorkDir is the base working directory for contents extracted from a
	// release payload. Usually this would map to a volume mount.
	WorkDir string

	// PreserveOutput indicates whether the temporary working directory created
	// under WorkDir should be preserved. If false, the temporary directory is
	// deleted after use.
	PreserveOutput bool

	// FeatureGateManifest is the path to a rendered feature gate manifest.
	// This must be copied into the MCC directory as it is required
	// to render the ignition payload.
	FeatureGateManifest string

	ImageFileCache *imageFileCache

	lock sync.Mutex
}

var _ IgnitionProvider = (*LocalIgnitionProvider)(nil)

func (p *LocalIgnitionProvider) GetPayload(ctx context.Context, releaseImage string, customConfig string, pullSecretHash string) ([]byte, error) {
	p.lock.Lock()
	defer p.lock.Unlock()

	log := ctrl.Log.WithName("get-payload")

	// Fetch the pull secret contents
	pullSecret, err := func() ([]byte, error) {
		secret := &corev1.Secret{}
		if err := p.Client.Get(ctx, client.ObjectKey{Namespace: p.Namespace, Name: pullSecretName}, secret); err != nil {
			return nil, fmt.Errorf("failed to get pull secret: %w", err)
		}
		data, exists := secret.Data[corev1.DockerConfigJsonKey]
		if !exists {
			return nil, fmt.Errorf("pull secret missing %q key", corev1.DockerConfigJsonKey)
		}
		return data, nil
	}()
	if err != nil {
		return nil, fmt.Errorf("failed to get pull secret: %w", err)
	}

	// Verify the pullSecret hash matches the passed-in parameter pullSecretHash to ensure the correct pull secret gets loaded into the payload
	if pullSecretHash != "" && util.HashStruct(pullSecret) != pullSecretHash {
		return nil, fmt.Errorf("pull secret does not match hash")
	}

	// Fetch the bootstrap kubeconfig contents
	bootstrapKubeConfig, err := func() ([]byte, error) {
		secret := &corev1.Secret{}
		if err := p.Client.Get(ctx, client.ObjectKey{Namespace: p.Namespace, Name: "bootstrap-kubeconfig"}, secret); err != nil {
			return nil, fmt.Errorf("failed to get bootstrap kubeconfig secret: %w", err)
		}
		data, exists := secret.Data["kubeconfig"]
		if !exists {
			return nil, fmt.Errorf("bootstrap kubeconfig secret missing kubeconfig key")
		}
		return data, nil
	}()
	if err != nil {
		return nil, fmt.Errorf("failed to get bootstrap kubeconfig: %w", err)
	}

	// Fetch the MCS config
	mcsConfig := &corev1.ConfigMap{}
	if err := p.Client.Get(ctx, client.ObjectKey{Namespace: p.Namespace, Name: "machine-config-server"}, mcsConfig); err != nil {
		return nil, fmt.Errorf("failed to get machine-config-server configmap: %w", err)
	}

	// Look up the release image metadata
	imageProvider, err := func() (*imageprovider.ReleaseImageProvider, error) {
		img, err := p.ReleaseProvider.Lookup(ctx, releaseImage, pullSecret)
		if err != nil {
			return nil, fmt.Errorf("failed to look up release image metadata: %w", err)
		}
		return imageprovider.New(img), nil
	}()
	if err != nil {
		return nil, fmt.Errorf("failed to get component images: %v", err)
	}

	mcoImage, hasMcoImage := imageProvider.ImageExist("machine-config-operator")
	if !hasMcoImage {
		return nil, fmt.Errorf("release image does not contain machine-config-operator (images: %v)", imageProvider.ComponentImages())
	}
	log.Info("discovered mco image", "image", mcoImage)

	// Set up the base working directory
	workDir, err := os.MkdirTemp(p.WorkDir, "get-payload")
	if err != nil {
		return nil, fmt.Errorf("failed to create working directory: %w", err)
	}
	if !p.PreserveOutput {
		defer func() {
			if err := os.RemoveAll(workDir); err != nil {
				log.Error(err, "failed to delete working directory", "dir", workDir)
			}
		}()
	}
	log.Info("created working directory", "dir", workDir)

	// Prepare all the working subdirectories
	binDir := filepath.Join(workDir, "bin")
	mcoBaseDir := filepath.Join(workDir, "mco")
	mccBaseDir := filepath.Join(workDir, "mcc")
	mcsBaseDir := filepath.Join(workDir, "mcs")
	configDir := filepath.Join(workDir, "config")
	for _, dir := range []string{binDir, mcoBaseDir, mccBaseDir, mcsBaseDir, configDir} {
		if err := os.Mkdir(dir, 0755); err != nil {
			return nil, fmt.Errorf("failed to make directory %s: %w", dir, err)
		}
	}

	// Write out the custom config to the MCC directory
	if err := os.WriteFile(filepath.Join(mccBaseDir, "custom.yaml"), []byte(customConfig), 0644); err != nil {
		return nil, fmt.Errorf("failed to write mcc config: %w", err)
	}
	// Write out the bootstrap kubeconfig to the MCS directory
	if err := os.WriteFile(filepath.Join(mcsBaseDir, "kubeconfig"), bootstrapKubeConfig, 0644); err != nil {
		return nil, fmt.Errorf("failed to write bootstrap kubeconfig: %w", err)
	}
	// Extract MCS config files into the config directory
	for name, contents := range mcsConfig.Data {
		if err := os.WriteFile(filepath.Join(configDir, name), []byte(contents), 0644); err != nil {
			return nil, fmt.Errorf("failed to write MCS config file %q: %w", name, err)
		}
	}
	// Extract ImageReferences from release image to config directory
	err = func() error {
		start := time.Now()
		if err := registryclient.ExtractImageFilesToDir(ctx, releaseImage, pullSecret, "release-manifests/image-references", configDir); err != nil {
			return fmt.Errorf("failed to extract image-references: %w", err)
		}
		log.Info("extracted image-references", "time", time.Since(start).Round(time.Second).String())
		return nil
	}()

	// For Azure, extract the cloud provider config file as MCO input
	if p.CloudProvider == hyperv1.AzurePlatform {
		cloudConfigMap := &corev1.ConfigMap{}
		if err := p.Client.Get(ctx, client.ObjectKey{Namespace: p.Namespace, Name: manifests.AzureProviderConfig("").Name}, cloudConfigMap); err != nil {
			return nil, fmt.Errorf("failed to get cloud provider configmap: %w", err)
		}
		cloudConfYaml, err := yaml.Marshal(cloudConfigMap)
		if err != nil {
			return nil, fmt.Errorf("failed to marshal cloud config: %w", err)
		}
		if err := os.WriteFile(filepath.Join(mcoBaseDir, "cloud.conf.configmap.yaml"), cloudConfYaml, 0644); err != nil {
			return nil, fmt.Errorf("failed to write bootstrap kubeconfig: %w", err)
		}
	}

	isMultiArchManifestList, err := registryclient.IsMultiArchManifestList(ctx, mcoImage, pullSecret)
	if err != nil {
		return nil, fmt.Errorf("failed to determine if image is manifest listed: %w", err)
	}

	if isMultiArchManifestList {
		os := runtime.GOOS
		arch := runtime.GOARCH
		log.Info("mco image is a manifest listed image; extracting manifest for os/arch: " + os + "/" + arch)

		// Verify MF Image has the right os/arch image
		mcoImage, err = findImageRefByArch(ctx, mcoImage, pullSecret, os, arch)
		if err != nil {
			return nil, fmt.Errorf("failed to extract appropriate os/arch manifest from %s: %w", mcoImage, err)
		}
	}

	// Extract template files from the MCO image to the MCC input directory
	err = func() error {
		start := time.Now()
		if err := registryclient.ExtractImageFilesToDir(ctx, mcoImage, pullSecret, "etc/mcc/templates/*", mccBaseDir); err != nil {
			return fmt.Errorf("failed to extract mcc templates: %w", err)
		}
		log.Info("extracted templates", "time", time.Since(start).Round(time.Second).String())
		return nil
	}()
	if err != nil {
		return nil, fmt.Errorf("failed to extract templates from image: %w", err)
	}

	// Write out the feature gate manifest to the MCC dir.
	// Use the feature gate from the hosted control plane which should reflect the feature gate of the cluster.
	if err := func() error {
		featureGateBytes, err := os.ReadFile(p.FeatureGateManifest)
		if err != nil {
			return fmt.Errorf("failed to read feature gate: %w", err)
		}

		if err := os.WriteFile(filepath.Join(mccBaseDir, "99_feature-gate.yaml"), featureGateBytes, 0644); err != nil {
			return fmt.Errorf("failed to write feature gate: %w", err)
		}

		return nil
	}(); err != nil {
		return nil, fmt.Errorf("failed to extract feature gate: %w", err)
	}

	// Extract binaries from the MCO image into the bin directory
	err = func() error {
		start := time.Now()
		binaries := []string{"machine-config-operator", "machine-config-controller", "machine-config-server"}
		for _, name := range binaries {
			file, err := os.Create(filepath.Join(binDir, name))
			if err != nil {
				return fmt.Errorf("failed to create file: %w", err)
			}
			if err := file.Chmod(0777); err != nil {
				return fmt.Errorf("failed to chmod file: %w", err)
			}
			if err := p.ImageFileCache.extractImageFile(ctx, mcoImage, pullSecret, filepath.Join("usr/bin/", name), file); err != nil {
				return fmt.Errorf("failed to extract image file: %w", err)
			}
			if err := file.Close(); err != nil {
				return fmt.Errorf("failed to close file: %w", err)
			}
		}
		log.Info("downloaded binaries", "time", time.Since(start).Round(time.Second).String())
		return nil
	}()
	if err != nil {
		return nil, fmt.Errorf("failed to download binaries: %w", err)
	}

	// First, run the MCO using templates and image refs as input. This generates
	// output for the MCC.
	err = func() error {
		destDir := filepath.Join(workDir, "mco")
		if err := os.MkdirAll(destDir, 0755); err != nil {
			return fmt.Errorf("failed to make dir: %w", err)
		}

		// Write out pull secret to config directory
		// NOTE: This overwrites the one in the machine-config-server configmap to ensure it's the one that matches the hash used in the token secret.
		pullSecretObject := common.PullSecret("default")
		pullSecretObject.Data = map[string][]byte{
			corev1.DockerConfigJsonKey: pullSecret,
		}
		pullSecretObject.Type = corev1.SecretTypeDockerConfigJson
		serializedPullSecret, err := util.SerializeResource(pullSecretObject, api.Scheme)
		if err != nil {
			return fmt.Errorf("failed to serialize pull-secret.yaml: %w", err)
		}
		if err = os.WriteFile(fmt.Sprintf("%s/pull-secret.yaml", configDir), []byte(serializedPullSecret), 0644); err != nil {
			return fmt.Errorf("failed to write pull secret to config dir: %w", err)
		}

		payloadVersion, err := semver.Parse(imageProvider.Version())
		if err != nil {
			return fmt.Errorf("failed to parse payload version: %w", err)
		}

		// args contains the base args that have not changed over time.
		args := []string{
			"bootstrap",
			fmt.Sprintf("--root-ca=%s/root-ca.crt", configDir),
			fmt.Sprintf("--infra-config-file=%s/cluster-infrastructure-02-config.yaml", configDir),
			fmt.Sprintf("--network-config-file=%s/cluster-network-02-config.yaml", configDir),
			fmt.Sprintf("--proxy-config-file=%s/cluster-proxy-01-config.yaml", configDir),
			fmt.Sprintf("--config-file=%s/install-config.yaml", configDir),
			fmt.Sprintf("--dns-config-file=%s/cluster-dns-02-config.yaml", configDir),
			fmt.Sprintf("--pull-secret=%s/pull-secret.yaml", configDir),
			fmt.Sprintf("--dest-dir=%s", destDir),
			fmt.Sprintf("--additional-trust-bundle-config-file=%s/user-ca-bundle-config.yaml", configDir),
		}

		// Depending on the version, we need different args.
		switch y := payloadVersion.Minor; {
		case y >= 14:
			args = append(args,
				fmt.Sprintf("--payload-version=%s", imageProvider.Version()),
			)
			// We need to include 4.13 plus args here too.
			fallthrough
		case y >= 13:
			args = append(args,
				fmt.Sprintf("--image-references=%s", path.Join(configDir, "release-manifests", "image-references")),
				fmt.Sprintf("--kube-ca=%s/signer-ca.crt", configDir),
			)
		case y <= 12:
			// when the CPO is at N and the NodePool.spec.release at N-1
			// we fail to render ignition payload because https://github.com/openshift/machine-config-operator/pull/3286
			// broke backward compatibility.
			args = append(args,
				fmt.Sprintf("--machine-config-operator-image=%s", imageProvider.GetImage("machine-config-operator")),
				fmt.Sprintf("--machine-config-oscontent-image=%s", imageProvider.GetImage("machine-os-content")),
				fmt.Sprintf("--infra-image=%s", imageProvider.GetImage("pod")),
				fmt.Sprintf("--keepalived-image=%s", imageProvider.GetImage("keepalived-ipfailover")),
				fmt.Sprintf("--coredns-image=%s", imageProvider.GetImage("codedns")),
				fmt.Sprintf("--haproxy-image=%s", imageProvider.GetImage("haproxy")),
				fmt.Sprintf("--baremetal-runtimecfg-image=%s", imageProvider.GetImage("baremetal-runtimecfg")),
				fmt.Sprintf("--kube-ca=%s/root-ca.crt", configDir),
			)
		}

		if image, exists := imageProvider.ImageExist("mdns-publisher"); exists {
			args = append(args, fmt.Sprintf("--mdns-publisher-image=%s", image))
		}
		if mcsConfig.Data["user-ca-bundle-config.yaml"] != "" {
			args = append(args, fmt.Sprintf("--additional-trust-bundle-config-file=%s/user-ca-bundle-config.yaml", configDir))
		}
		if p.CloudProvider == hyperv1.AzurePlatform {
			args = append(args, fmt.Sprintf("--cloud-config-file=%s/cloud.conf.configmap.yaml", mcoBaseDir))
		}

		start := time.Now()
		cmd := exec.CommandContext(ctx, filepath.Join(binDir, "machine-config-operator"), args...)
		out, err := cmd.CombinedOutput()
		log.Info("machine-config-operator process completed", "time", time.Since(start).Round(time.Second).String(), "output", string(out))
		if err != nil {
			return fmt.Errorf("machine-config-operator process failed: %w", err)
		}

		// set missing images condition on the HCP
		if err := p.reconcileValidReleaseInfoCondition(ctx, imageProvider); err != nil {
			log.Error(err, "failed to reconcile IgnitionValidReleaseInfo condition")
		}

		// Copy output to the MCC base directory
		bootstrapManifestsDir := filepath.Join(destDir, "bootstrap", "manifests")
		manifests, err := os.ReadDir(bootstrapManifestsDir)
		if err != nil {
			return fmt.Errorf("failed to read dir: %w", err)
		}
		for _, fd := range manifests {
			src := path.Join(bootstrapManifestsDir, fd.Name())
			dst := path.Join(mccBaseDir, fd.Name())
			if fd.IsDir() {
				continue
			}
			if err := copyFile(src, dst); err != nil {
				return fmt.Errorf("failed to copy %s to %s: %w", src, dst, err)
			}
		}

		// Copy machineconfigpool config data to the MCC input directory. This is
		// important to override the pools with the ones generated by the CPO.
		err = func() error {
			matches, err := filepath.Glob(filepath.Join(configDir, "*.machineconfigpool.yaml"))
			if err != nil {
				return fmt.Errorf("failed to list dir %s: %w", configDir, err)
			}
			for _, src := range matches {
				dst := filepath.Join(mccBaseDir, filepath.Base(src))
				if err := copyFile(src, dst); err != nil {
					return fmt.Errorf("failed to copy %s to %s: %w", src, dst, err)
				}
			}
			return nil
		}()
		if err != nil {
			return fmt.Errorf("failed to copy mcs config to mcc directory: %w", err)
		}
		return nil
	}()
	if err != nil {
		return nil, fmt.Errorf("failed to execute machine-config-operator: %w", err)
	}

	// Next, run the MCC using templates and MCO output as input, producing output
	// for the MCS.
	err = func() error {
		start := time.Now()
		cmd := exec.CommandContext(ctx, filepath.Join(binDir, "machine-config-controller"), "bootstrap",
			fmt.Sprintf("--manifest-dir=%s", mccBaseDir),
			fmt.Sprintf("--templates=%s", filepath.Join(mccBaseDir, "etc", "mcc", "templates")),
			fmt.Sprintf("--pull-secret=%s/machineconfigcontroller-pull-secret", mccBaseDir),
			fmt.Sprintf("--dest-dir=%s", mcsBaseDir),
		)
		out, err := cmd.CombinedOutput()
		log.Info("machine-config-controller process completed", "time", time.Since(start).Round(time.Second).String(), "output", string(out))
		if err != nil {
			return fmt.Errorf("machine-config-controller process failed: %w", err)
		}
		return nil
	}()
	if err != nil {
		return nil, fmt.Errorf("failed to execute machine-config-controller: %w", err)
	}

	// Finally, run the MCS to generate a payload.
	payload, err := func() ([]byte, error) {
		start := time.Now()

		// Generate certificates. The MCS is hard-coded to expose a TLS listener
		// and requires both a certificate and a key.
		// TODO: This could be generated once up-front and cached for all processes
		err = func() error {
			cfg := &certs.CertCfg{
				Subject:   pkix.Name{CommonName: "machine-config-server", OrganizationalUnit: []string{"openshift"}},
				KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
				Validity:  certs.ValidityOneDay,
				IsCA:      true,
			}
			key, crt, err := certs.GenerateSelfSignedCertificate(cfg)
			if err != nil {
				return fmt.Errorf("failed to generate cert: %w", err)
			}
			if err := os.WriteFile(filepath.Join(mcsBaseDir, "tls.crt"), certs.CertToPem(crt), 0644); err != nil {
				return fmt.Errorf("failed to write mcs cert: %w", err)
			}
			if err := os.WriteFile(filepath.Join(mcsBaseDir, "tls.key"), certs.PrivateKeyToPem(key), 0644); err != nil {
				return fmt.Errorf("failed to write mcs cert: %w", err)
			}
			return nil
		}()
		if err != nil {
			return nil, fmt.Errorf("failed to generate certificates: %w", err)
		}

		// Spin up the MCS process and ensure it's signaled to terminate when
		// the function returns
		mcsCtx, cancel := context.WithCancel(ctx)
		defer cancel()
		cmd := exec.CommandContext(mcsCtx, filepath.Join(binDir, "machine-config-server"), "bootstrap",
			fmt.Sprintf("--server-basedir=%s", mcsBaseDir),
			fmt.Sprintf("--bootstrap-kubeconfig=%s/kubeconfig", mcsBaseDir),
			fmt.Sprintf("--cert=%s/tls.crt", mcsBaseDir),
			fmt.Sprintf("--key=%s/tls.key", mcsBaseDir),
			"--secure-port=22623",
			"--insecure-port=22624",
		)
		go func() {
			out, err := cmd.CombinedOutput()
			log.Info("machine-config-server process exited", "output", string(out), "error", err)
		}()

		// Try connecting to the server until we get a response or the context is
		// closed
		httpclient := &http.Client{
			Timeout: 5 * time.Second,
		}
		var payload []byte
		err = wait.PollUntilWithContext(ctx, 1*time.Second, func(ctx context.Context) (bool, error) {
			req, err := http.NewRequestWithContext(ctx, "GET", "http://localhost:22624/config/master", nil)
			if err != nil {
				return false, fmt.Errorf("error building http request: %w", err)
			}
			// We pass expected Headers to return the right config version.
			// https://www.iana.org/assignments/media-types/application/vnd.coreos.ignition+json
			// https://github.com/coreos/ignition/blob/0cbe33fee45d012515479a88f0fe94ef58d5102b/internal/resource/url.go#L61-L64
			// https://github.com/openshift/machine-config-operator/blob/9c6c2bfd7ed498bfbc296d530d1839bd6a177b0b/pkg/server/api.go#L269
			req.Header.Add("Accept", "application/vnd.coreos.ignition+json;version=3.2.0, */*;q=0.1")
			res, err := httpclient.Do(req)
			if err != nil {
				log.Error(err, "mcs request failed")
				return false, nil
			}
			if res.StatusCode != http.StatusOK {
				log.Error(err, "mcs returned unexpected response code", "code", res.StatusCode)
				return false, nil
			}

			defer func() {
				if err := res.Body.Close(); err != nil {
					log.Error(err, "failed to close mcs response body")
				}
			}()
			p, err := io.ReadAll(res.Body)
			if err != nil {
				log.Error(err, "failed to read mcs response body")
				return false, nil
			}
			payload = p
			log.Info("got mcs payload", "time", time.Since(start).Round(time.Second).String())
			return true, nil
		})
		return payload, err
	}()
	if err != nil {
		return nil, fmt.Errorf("failed to get payload from mcs: %w", err)
	}

	return payload, nil
}

func (r *LocalIgnitionProvider) reconcileValidReleaseInfoCondition(ctx context.Context, releaseImageProvider *imageprovider.ReleaseImageProvider) error {
	hcpList := &hyperv1.HostedControlPlaneList{}
	if err := r.Client.List(ctx, hcpList, client.InNamespace(r.Namespace)); err != nil {
		return err
	}
	if len(hcpList.Items) == 0 {
		return fmt.Errorf("failed to find HostedControlPlane in namespace %s", r.Namespace)
	}

	hostedControlPlane := hcpList.Items[0]

	if len(releaseImageProvider.GetMissingImages()) == 0 {
		meta.SetStatusCondition(&hostedControlPlane.Status.Conditions, metav1.Condition{
			Type:               string(hyperv1.IgnitionServerValidReleaseInfo),
			Status:             metav1.ConditionTrue,
			Reason:             hyperv1.AsExpectedReason,
			Message:            hyperv1.AllIsWellMessage,
			ObservedGeneration: hostedControlPlane.Generation,
		})
	} else {
		meta.SetStatusCondition(&hostedControlPlane.Status.Conditions, metav1.Condition{
			Type:               string(hyperv1.IgnitionServerValidReleaseInfo),
			Status:             metav1.ConditionFalse,
			Reason:             hyperv1.MissingReleaseImagesReason,
			Message:            strings.Join(releaseImageProvider.GetMissingImages(), ", "),
			ObservedGeneration: hostedControlPlane.Generation,
		})
	}

	return r.Client.Status().Update(ctx, &hostedControlPlane)
}

// copyFile copies a file named src to dst, preserving attributes.
func copyFile(src, dst string) error {
	srcfd, err := os.Open(src)
	if err != nil {
		return err
	}
	defer srcfd.Close()

	dstfd, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer dstfd.Close()

	if _, err = io.Copy(dstfd, srcfd); err != nil {
		return err
	}
	srcinfo, err := os.Stat(src)
	if err != nil {
		return err
	}
	return os.Chmod(dst, srcinfo.Mode())
}

// findImageRefByArch finds the appropriate image reference in a multi-arch manifest image based on the current platform's OS and processor architecture
func findImageRefByArch(ctx context.Context, imageRef string, pullSecret []byte, osToFind string, archToFind string) (manifestImageRef string, err error) {
	manifestList, err := registryclient.GetManifest(ctx, imageRef, pullSecret)
	if err != nil {
		return "", fmt.Errorf("failed to retrieve manifest from image ref, %s: %w", imageRef, err)
	}

	_, payload, err := manifestList.Payload()
	if err != nil {
		return "", fmt.Errorf("failed to get manifest payload: %w", err)
	}

	deserializedManifestList := new(manifestlist.DeserializedManifestList)
	if err = deserializedManifestList.UnmarshalJSON(payload); err != nil {
		return "", fmt.Errorf("failed to get unmarshalled manifest list: %w", err)
	}

	matchingManifestForArch, err := findMatchingManifest(imageRef, deserializedManifestList, osToFind, archToFind)
	if err != nil {
		return "", fmt.Errorf("failed to retrieve matching manifest for os/arch, %s/%s: %w", osToFind, archToFind, err)
	}

	return matchingManifestForArch, nil
}

// findMatchingManifest looks to find a manifest matching the current platform's OS and processor architecture from a deserialized manifest list from an image's payload
func findMatchingManifest(imageRef string, deserializedManifestList *manifestlist.DeserializedManifestList, osToFind string, archToFind string) (string, error) {
	var foundManifestDesc *manifestlist.ManifestDescriptor
	for _, manifestDesc := range deserializedManifestList.ManifestList.Manifests {
		if osToFind == manifestDesc.Platform.OS && archToFind == manifestDesc.Platform.Architecture {
			foundManifestDesc = &manifestDesc
			break
		}
	}

	if foundManifestDesc == nil {
		return "", fmt.Errorf("not found")
	}

	// Multi-arch image references look like either:
	//	quay.io/openshift-release-dev/ocp-release@sha256:1a101ef5215da468cea8bd2eb47114e85b2b64a6b230d5882f845701f55d057f
	//	quay.io/openshift-release-dev/ocp-release:4.11.0-0.nightly-multi-2022-07-12-131716
	if strings.Contains(imageRef, "@sha") {
		splitSHA := strings.Split(imageRef, "@")
		if len(splitSHA) != 2 {
			return "", fmt.Errorf("failed to parse imageRef %s", imageRef)
		}

		matchingManifestForArch := splitSHA[0] + "@" + string(foundManifestDesc.Descriptor.Digest)
		log.Info("Found matching manifest of os/arch: " + matchingManifestForArch)
		return matchingManifestForArch, nil
	}

	if strings.Contains(imageRef, "ocp-release:") {
		splitSHA := strings.Split(imageRef, ":")
		if len(splitSHA) != 2 {
			return "", fmt.Errorf("failed to parse imageRef %s", imageRef)
		}

		matchingManifestForArch := splitSHA[0] + "@" + string(foundManifestDesc.Descriptor.Digest)
		log.Info("Found matching manifest of os/arch: " + matchingManifestForArch)
		return matchingManifestForArch, nil
	}

	return "", fmt.Errorf("imageRef is an unknown format to parse, imageRef: %s", imageRef)
}