Fix panic when pulling OCI-packaged helm chart

go.mod

@@ -31,7 +31,7 @@ require (
 	github.com/sylabs/squashfs v0.6.1
 	github.com/wagoodman/go-partybus v0.0.0-20200526224238-eb215533f07d
 	github.com/wagoodman/go-progress v0.0.0-20230925121702-07e42b3cdba0
-	golang.org/x/crypto v0.17.0
+	golang.org/x/crypto v0.21.0
 )
 
 require (
@@ -97,11 +97,11 @@ require (
 	go.opentelemetry.io/otel v1.19.0 // indirect
 	go.opentelemetry.io/otel/trace v1.19.0 // indirect
 	golang.org/x/mod v0.11.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/oauth2 v0.10.0 // indirect
+	golang.org/x/net v0.22.0 // indirect
+	golang.org/x/oauth2 v0.18.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
-	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/term v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.10.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

go.sum

@@ -282,8 +282,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
-golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -303,11 +303,11 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
+golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
-golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
+golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
+golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -333,11 +333,11 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

pkg/image/layer.go

@@ -80,24 +80,16 @@ func (l *Layer) uncompressedTarCache(uncompressedLayersCacheDir string) (string,
 
 // Read parses information from the underlying layer tar into this struct. This includes layer metadata, the layer
 // file tree, and the layer squash tree.
-func (l *Layer) Read(catalog *FileCatalog, imgMetadata Metadata, idx int, uncompressedLayersCacheDir string) error {
-	var err error
-	tree := filetree.New()
-	l.Tree = tree
-	l.fileCatalog = catalog
-	l.Metadata, err = newLayerMetadata(imgMetadata, l.layer, idx)
+func (l *Layer) Read(catalog *FileCatalog, _ Metadata, idx int, uncompressedLayersCacheDir string) error {
+	mediaType, err := l.layer.MediaType()
 	if err != nil {
 		return err
 	}
+	tree := filetree.New()
+	l.Tree = tree
+	l.fileCatalog = catalog
 
-	log.Debugf("layer metadata: index=%+v digest=%+v mediaType=%+v",
-		l.Metadata.Index,
-		l.Metadata.Digest,
-		l.Metadata.MediaType)
-
-	monitor := trackReadProgress(l.Metadata)
-
-	switch l.Metadata.MediaType {
+	switch mediaType {
 	case types.OCILayer,
 		types.OCIUncompressedLayer,
 		types.OCIRestrictedLayer,
@@ -107,44 +99,82 @@ func (l *Layer) Read(catalog *FileCatalog, imgMetadata Metadata, idx int, uncomp
 		types.DockerForeignLayer,
 		types.DockerUncompressedLayer:
 
-		tarFilePath, err := l.uncompressedTarCache(uncompressedLayersCacheDir)
+		err := l.readStandardImageLayer(idx, uncompressedLayersCacheDir, tree)
 		if err != nil {
 			return err
 		}
-
-		l.indexedContent, err = file.NewTarIndex(
-			tarFilePath,
-			layerTarIndexer(tree, l.fileCatalog, &l.Metadata.Size, l, monitor),
-		)
-		if err != nil {
-			return fmt.Errorf("failed to read layer=%q tar : %w", l.Metadata.Digest, err)
-		}
-
 	case SingularitySquashFSLayer:
-		r, err := l.layer.Uncompressed()
-		if err != nil {
-			return fmt.Errorf("failed to read layer=%q: %w", l.Metadata.Digest, err)
-		}
-		// defer r.Close() // TODO: if we close this here, we can't read file contents after we return.
-
-		// Walk the more efficient walk if we're blessed with an io.ReaderAt.
-		if ra, ok := r.(io.ReaderAt); ok {
-			err = file.WalkSquashFS(ra, squashfsVisitor(tree, l.fileCatalog, &l.Metadata.Size, l, monitor))
-		} else {
-			err = file.WalkSquashFSFromReader(r, squashfsVisitor(tree, l.fileCatalog, &l.Metadata.Size, l, monitor))
-		}
+		err := l.readSingularityImage(idx, tree)
 		if err != nil {
-			return fmt.Errorf("failed to walk layer=%q: %w", l.Metadata.Digest, err)
+			return err
 		}
-
 	default:
-		return fmt.Errorf("unknown layer media type: %+v", l.Metadata.MediaType)
+		return fmt.Errorf("unknown layer media type: %+v", mediaType)
 	}
 
 	l.SearchContext = filetree.NewSearchContext(l.Tree, l.fileCatalog.Index)
 
+	return nil
+}
+
+func (l *Layer) readStandardImageLayer(idx int, uncompressedLayersCacheDir string, tree *filetree.FileTree) error {
+	var err error
+	l.Metadata, err = newLayerMetadata(l.layer, idx)
+	monitor := trackReadProgress(l.Metadata)
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("layer metadata: index=%+v digest=%+v mediaType=%+v",
+		l.Metadata.Index,
+		l.Metadata.Digest,
+		l.Metadata.MediaType)
+	tarFilePath, err := l.uncompressedTarCache(uncompressedLayersCacheDir)
+	if err != nil {
+		return err
+	}
+
+	l.indexedContent, err = file.NewTarIndex(
+		tarFilePath,
+		layerTarIndexer(tree, l.fileCatalog, &l.Metadata.Size, l, monitor),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to read layer=%q tar : %w", l.Metadata.Digest, err)
+	}
+
 	monitor.SetCompleted()
+	return nil
+}
+
+func (l *Layer) readSingularityImage(idx int, tree *filetree.FileTree) error {
+	var err error
+	l.Metadata, err = newLayerMetadata(l.layer, idx)
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("layer metadata: index=%+v digest=%+v mediaType=%+v",
+		l.Metadata.Index,
+		l.Metadata.Digest,
+		l.Metadata.MediaType)
+	monitor := trackReadProgress(l.Metadata)
+	r, err := l.layer.Uncompressed()
+	if err != nil {
+		return fmt.Errorf("failed to read layer=%q: %w", l.Metadata.Digest, err)
+	}
+	// defer r.Close() // TODO: if we close this here, we can't read file contents after we return.
+
+	// Walk the more efficient walk if we're blessed with an io.ReaderAt.
+	if ra, ok := r.(io.ReaderAt); ok {
+		err = file.WalkSquashFS(ra, squashfsVisitor(tree, l.fileCatalog, &l.Metadata.Size, l, monitor))
+	} else {
+		err = file.WalkSquashFSFromReader(r, squashfsVisitor(tree, l.fileCatalog, &l.Metadata.Size, l, monitor))
+	}
+	if err != nil {
+		return fmt.Errorf("failed to walk layer=%q: %w", l.Metadata.Digest, err)
+	}
 
+	monitor.SetCompleted()
 	return nil
 }
 

pkg/image/layer_metadata.go

@@ -16,17 +16,19 @@ type LayerMetadata struct {
 }
 
 // newLayerMetadata aggregates pertinent layer metadata information.
-func newLayerMetadata(imgMetadata Metadata, layer v1.Layer, idx int) (LayerMetadata, error) {
+func newLayerMetadata(layer v1.Layer, idx int) (LayerMetadata, error) {
 	mediaType, err := layer.MediaType()
 	if err != nil {
 		return LayerMetadata{}, err
 	}
+	diffID, err := layer.DiffID()
+	if err != nil {
+		return LayerMetadata{}, err
+	}
 
-	// digest = diff-id = a digest of the uncompressed layer content
-	diffIDHash := imgMetadata.Config.RootFS.DiffIDs[idx]
 	return LayerMetadata{
 		Index:     uint(idx),
-		Digest:    diffIDHash.String(),
+		Digest:    diffID.String(),
 		MediaType: mediaType,
 	}, nil
 }

pkg/image/layer_test.go

@@ -0,0 +1,99 @@
+package image
+
+import (
+	"errors"
+	"io"
+	"strings"
+	"testing"
+
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	v1Types "github.com/google/go-containerregistry/pkg/v1/types"
+	"github.com/stretchr/testify/require"
+)
+
+type mockLayer struct {
+	mediaType v1Types.MediaType
+	err       error
+}
+
+func (m mockLayer) Digest() (v1.Hash, error) {
+	return v1.Hash{
+		Algorithm: "sha256",
+		Hex:       "aaaaaaaaaa1234",
+	}, nil
+}
+
+func (m mockLayer) DiffID() (v1.Hash, error) {
+	return v1.Hash{
+		Algorithm: "sha256",
+		Hex:       "aaaaaaaaaa1234",
+	}, nil
+}
+
+func (m mockLayer) Compressed() (io.ReadCloser, error) {
+	panic("implement me")
+}
+
+func (m mockLayer) Uncompressed() (io.ReadCloser, error) {
+	return io.NopCloser(strings.NewReader("")), nil
+}
+
+func (m mockLayer) Size() (int64, error) {
+	return 0, nil
+}
+
+func (m mockLayer) MediaType() (v1Types.MediaType, error) {
+	return m.mediaType, m.err
+}
+
+var _ v1.Layer = &mockLayer{}
+
+func fakeLayer(mediaType v1Types.MediaType, err error) v1.Layer {
+	return mockLayer{
+		mediaType: mediaType,
+		err:       err,
+	}
+}
+
+func TestRead(t *testing.T) {
+	tests := []struct {
+		name            string
+		mediaType       v1Types.MediaType
+		mediaTypeErr    error
+		wantErrContents string
+	}{
+		{
+			name:            "unsupported media type",
+			mediaType:       "garbage",
+			mediaTypeErr:    nil,
+			wantErrContents: "unknown layer media type: garbage",
+		},
+		{
+			name:            "unsupported media type: helm chart",
+			mediaType:       "application/vnd.cncf.helm.chart.content.v1.tar+gzip",
+			wantErrContents: "application/vnd.cncf.helm.chart.content.v1.tar+gzip",
+		},
+		{
+			name:            "err on media type returned",
+			mediaTypeErr:    errors.New("no media type for you"),
+			wantErrContents: "no media type for you",
+		},
+		{
+			name:      "no error",
+			mediaType: v1Types.DockerLayer,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			layer := Layer{layer: fakeLayer(tt.mediaType, tt.mediaTypeErr)}
+			catalog := NewFileCatalog()
+			err := layer.Read(catalog, Metadata{}, 0, t.TempDir())
+			if tt.wantErrContents != "" {
+				require.ErrorContains(t, err, tt.wantErrContents)
+				return
+			}
+			require.NoError(t, err)
+		})
+	}
+}