chore: refactor pull operation

src/cmd/uds.go

@@ -255,8 +255,7 @@ func configureZarf() {
 		TempDirectory:  config.CommonOptions.TempDirectory,
 		OCIConcurrency: config.CommonOptions.OCIConcurrency,
 		Confirm:        config.CommonOptions.Confirm,
-		// todo: decouple Zarf cache?
-		CachePath: config.CommonOptions.CachePath,
+		CachePath:      config.CommonOptions.CachePath, // use uds-cache instead of zarf-cache
 	}
 }
 

src/config/config.go

@@ -47,6 +47,9 @@ const (
 	// UDSCache is the directory containing cached bundle layers
 	UDSCache = ".uds-cache"
 
+	// UDSCacheLayers is the directory in the cache containing cached bundle layers
+	UDSCacheLayers = "layers"
+
 	// TasksYAML is the default name of the uds run cmd file
 	TasksYAML = "tasks.yaml"
 

src/pkg/bundle/provider.go

@@ -34,7 +34,7 @@ type Provider interface {
 	// LoadBundle loads a bundle into the temporary directory and returns a map of the bundle's files
 	//
 	// (currently only the remote provider utilizes the concurrency parameter)
-	LoadBundle(concurrency int) (types.PathMap, error)
+	LoadBundle(options types.BundlePullOptions, concurrency int) (*types.UDSBundle, types.PathMap, error)
 
 	// CreateBundleSBOM creates a bundle-level SBOM from the underlying Zarf packages, if the Zarf package contains an SBOM
 	CreateBundleSBOM(extractSBOM bool) error
@@ -43,7 +43,7 @@ type Provider interface {
 	PublishBundle(bundle types.UDSBundle, remote *oci.OrasRemote) error
 
 	// getBundleManifest gets the bundle's root manifest
-	getBundleManifest() error
+	getBundleManifest() (*oci.ZarfOCIManifest, error)
 
 	// ZarfPackageNameMap returns a map of the zarf package name specified in the uds-bundle.yaml to the actual zarf package name
 	ZarfPackageNameMap() (map[string]string, error)
@@ -52,20 +52,34 @@ type Provider interface {
 // NewBundleProvider returns a new bundler Provider based on the source type
 func NewBundleProvider(ctx context.Context, source, destination string) (Provider, error) {
 	if helpers.IsOCIURL(source) {
-		provider := ociProvider{ctx: ctx, src: source, dst: destination}
+		op := ociProvider{ctx: ctx, src: source, dst: destination}
 		platform := ocispec.Platform{
 			Architecture: config.GetArch(),
 			OS:           oci.MultiOS,
 		}
+		// get remote client
 		remote, err := oci.NewOrasRemote(source, platform)
 		if err != nil {
 			return nil, err
 		}
-		provider.OrasRemote = remote
-		return &provider, nil
+		op.OrasRemote = remote
+
+		// get root manifest
+		root, err := op.FetchRoot()
+		if err != nil {
+			return nil, err
+		}
+		op.rootManifest = root
+
+		return &op, nil
 	}
 	if !utils.IsValidTarballPath(source) {
 		return nil, fmt.Errorf("invalid tarball path: %s", source)
 	}
-	return &tarballBundleProvider{ctx: ctx, src: source, dst: destination}, nil
+	tp := tarballBundleProvider{ctx: ctx, src: source, dst: destination}
+	err := tp.loadBundleManifest()
+	if err != nil {
+		return nil, err
+	}
+	return &tp, nil
 }

src/pkg/bundle/pull.go

@@ -21,10 +21,11 @@ import (
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
-// Pull pulls a bundle and saves it locally + caches it
+// Pull pulls a bundle and saves it locally
 func (b *Bundle) Pull() error {
+	// use uds-cache/packages as the dst dir for the pull to get auto caching
+	// we use an ORAS ocistore to make that dir look like an OCI artifact
 	cacheDir := filepath.Join(zarfConfig.GetAbsCachePath(), "packages")
-	// create the cache directory if it doesn't exist
 	if err := utils.CreateDirectory(cacheDir, 0o755); err != nil {
 		return err
 	}
@@ -41,28 +42,12 @@ func (b *Bundle) Pull() error {
 		return err
 	}
 
-	// pull the bundle's metadata + sig
-	loadedMetadata, err := provider.LoadBundleMetadata()
-	if err != nil {
-		return err
-	}
-	if err := utils.ReadYaml(loadedMetadata[config.BundleYAML], &b.bundle); err != nil {
-		return err
-	}
-
-	// validate the sig (if present)
-	if err := ValidateBundleSignature(loadedMetadata[config.BundleYAML], loadedMetadata[config.BundleYAMLSignature], b.cfg.PullOpts.PublicKeyPath); err != nil {
-		return err
-	}
-
 	// pull the bundle's uds-bundle.yaml and it's Zarf pkgs
-	// todo: refactor this fn, think about pulling the rootDesc first and getting the hashes from there
-	// today, we are getting the Zarf image manifest hashes from the uds-bundle.yaml
-	// in that logic we end up pulling the root manifest twice, once in LoadBundle and the other below in remote.ResolveRoot()
-	loaded, err := provider.LoadBundle(zarfConfig.CommonOptions.OCIConcurrency)
+	bundle, loaded, err := provider.LoadBundle(b.cfg.PullOpts, zarfConfig.CommonOptions.OCIConcurrency)
 	if err != nil {
 		return err
 	}
+	b.bundle = *bundle
 
 	// create a remote client just to resolve the root descriptor
 	platform := ocispec.Platform{

src/pkg/bundle/remote.go

@@ -43,19 +43,14 @@ type ociProvider struct {
 	src string
 	dst string
 	*oci.OrasRemote
-	manifest *oci.ZarfOCIManifest
+	rootManifest *oci.ZarfOCIManifest
 }
 
-func (op *ociProvider) getBundleManifest() error {
-	if op.manifest != nil {
-		return nil
+func (op *ociProvider) getBundleManifest() (*oci.ZarfOCIManifest, error) {
+	if op.rootManifest != nil {
+		return op.rootManifest, nil
 	}
-	root, err := op.FetchRoot()
-	if err != nil {
-		return err
-	}
-	op.manifest = root
-	return nil
+	return nil, fmt.Errorf("bundle root manifest not loaded")
 }
 
 // LoadBundleMetadata loads a remote bundle's metadata
@@ -79,10 +74,6 @@ func (op *ociProvider) LoadBundleMetadata() (types.PathMap, error) {
 		}
 		loaded[rel] = absSha
 	}
-	err = op.getBundleManifest()
-	if err != nil {
-		return nil, err
-	}
 	return loaded, nil
 }
 
@@ -149,55 +140,60 @@ func (op *ociProvider) CreateBundleSBOM(extractSBOM bool) error {
 	return nil
 }
 
-// LoadBundle loads a bundle's uds-bundle.yaml and Zarf packages from a remote source
-func (op *ociProvider) LoadBundle(_ int) (types.PathMap, error) {
-	var layersToPull []ocispec.Descriptor
-	estimatedBytes := int64(0)
-
-	if err := op.getBundleManifest(); err != nil {
-		return nil, err
-	}
-
-	loaded, err := op.LoadBundleMetadata() // todo: remove? this seems redundant, can we pass the "loaded" var in
+// LoadBundle loads a bundle from a remote source
+func (op *ociProvider) LoadBundle(opts types.BundlePullOptions, _ int) (*types.UDSBundle, types.PathMap, error) {
+	var bundle types.UDSBundle
+	// pull the bundle's metadata + sig
+	loaded, err := op.LoadBundleMetadata()
 	if err != nil {
-		return nil, err
+		return nil, nil, err
+	}
+	if err := zarfUtils.ReadYaml(loaded[config.BundleYAML], &bundle); err != nil {
+		return nil, nil, err
 	}
 
-	b, err := os.ReadFile(loaded[config.BundleYAML])
-	if err != nil {
-		return nil, err
+	// validate the sig (if present) before pulling the whole bundle
+	if err := ValidateBundleSignature(loaded[config.BundleYAML], loaded[config.BundleYAMLSignature], opts.PublicKeyPath); err != nil {
+		return nil, nil, err
 	}
 
-	var bundle types.UDSBundle
-	if err := goyaml.Unmarshal(b, &bundle); err != nil {
-		return nil, err
+	var layersToPull []ocispec.Descriptor
+	estimatedBytes := int64(0)
+
+	// get the bundle's root manifest
+	rootManifest, err := op.getBundleManifest()
+	if err != nil {
+		return nil, nil, err
 	}
 
 	for _, pkg := range bundle.Packages {
+
 		// grab sha of zarf image manifest and pull it down
 		sha := strings.Split(pkg.Ref, "@sha256:")[1] // this is where we use the SHA appended to the Zarf pkg inside the bundle
-		manifestDesc := op.manifest.Locate(sha)
+		manifestDesc := rootManifest.Locate(sha)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		manifestBytes, err := op.FetchLayer(manifestDesc)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
+
 		// unmarshal the zarf image manifest and add it to the layers to pull
 		var manifest oci.ZarfOCIManifest
 		if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		layersToPull = append(layersToPull, manifestDesc)
 		progressBar := message.NewProgressBar(int64(len(manifest.Layers)), fmt.Sprintf("Verifying layers in Zarf package: %s", pkg.Name))
+
 		// go through the layers in the zarf image manifest and check if they exist in the remote
 		for _, layer := range manifest.Layers {
 			ok, err := op.Repo().Blobs().Exists(op.ctx, layer)
 			progressBar.Add(1)
 			estimatedBytes += layer.Size
 			if err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 			// if the layer exists in the remote, add it to the layers to pull
 			if ok {
@@ -209,13 +205,13 @@ func (op *ociProvider) LoadBundle(_ int) (types.PathMap, error) {
 
 	store, err := ocistore.NewWithContext(op.ctx, op.dst)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	// grab the bundle root manifest and add it to the layers to pull
 	rootDesc, err := op.ResolveRoot()
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	layersToPull = append(layersToPull, rootDesc)
 
@@ -232,7 +228,7 @@ func (op *ociProvider) LoadBundle(_ int) (types.PathMap, error) {
 	_, err = oras.Copy(op.ctx, op.Repo(), op.Repo().Reference.String(), store, op.Repo().Reference.String(), copyOpts)
 	if err != nil {
 		doneSaving <- 1
-		return nil, err
+		return nil, nil, err
 	}
 
 	doneSaving <- 1
@@ -243,7 +239,7 @@ func (op *ociProvider) LoadBundle(_ int) (types.PathMap, error) {
 		loaded[sha] = filepath.Join(op.dst, config.BlobsDir, sha)
 	}
 
-	return loaded, nil
+	return &bundle, loaded, nil
 }
 
 func (op *ociProvider) PublishBundle(_ types.UDSBundle, _ *oci.OrasRemote) error {
@@ -337,7 +333,8 @@ func CheckOCISourcePath(source string) (string, error) {
 
 // ZarfPackageNameMap returns the uds bundle zarf package name to actual zarf package name mappings from the oci provider
 func (op *ociProvider) ZarfPackageNameMap() (map[string]string, error) {
-	if err := op.getBundleManifest(); err != nil {
+	rootManifest, err := op.getBundleManifest()
+	if err != nil {
 		return nil, err
 	}
 
@@ -359,7 +356,7 @@ func (op *ociProvider) ZarfPackageNameMap() (map[string]string, error) {
 	nameMap := make(map[string]string)
 	for _, pkg := range bundle.Packages {
 		sha := strings.Split(pkg.Ref, "@sha256:")[1] // this is where we use the SHA appended to the Zarf pkg inside the bundle
-		manifestDesc := op.manifest.Locate(sha)
+		manifestDesc := rootManifest.Locate(sha)
 		nameMap[manifestDesc.Annotations[config.UDSPackageNameAnnotation]] = manifestDesc.Annotations[config.ZarfPackageNameAnnotation]
 	}
 	return nameMap, nil