Merge 0818d04 into 60f2f72

adapters/mocksbom.go

@@ -50,7 +50,7 @@ func (m MockSBOMAdapter) CreateSBOM(ctx context.Context, imageID string, _ domai
 		},
 	}
 	if m.timeout {
-		sbom.Status = domain.SBOMStatusTimedOut
+		sbom.Status = domain.SBOMStatusIncomplete
 	}
 	return sbom, nil
 }

adapters/mocksbom_test.go

@@ -23,7 +23,7 @@ func TestMockSBOMAdapter_CreateSBOM_Error(t *testing.T) {
 func TestMockSBOMAdapter_CreateSBOM_Timeout(t *testing.T) {
 	m := NewMockSBOMAdapter(false, true)
 	sbom, _ := m.CreateSBOM(context.TODO(), "image", domain.RegistryOptions{})
-	assert.Assert(t, sbom.Status == domain.SBOMStatusTimedOut)
+	assert.Assert(t, sbom.Status == domain.SBOMStatusIncomplete)
 }
 
 func TestMockSBOMAdapter_Version(t *testing.T) {

adapters/v1/syft.go

@@ -2,8 +2,14 @@ package v1
 
 import (
 	"context"
+	"crypto/tls"
+	"fmt"
+	"net/http"
+	"runtime"
 	"time"
 
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/filetree"
 	"github.com/anchore/stereoscope/pkg/image"
 	"github.com/anchore/syft/syft"
 	"github.com/anchore/syft/syft/artifact"
@@ -13,6 +19,10 @@ import (
 	"github.com/anchore/syft/syft/sbom"
 	"github.com/anchore/syft/syft/source"
 	"github.com/eapache/go-resiliency/deadline"
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/name"
+	containerregistryV1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/kubescape/go-logger"
 	"github.com/kubescape/go-logger/helpers"
 	"github.com/kubescape/k8s-interface/instanceidhandler/v1"
@@ -24,15 +34,18 @@ import (
 
 // SyftAdapter implements SBOMCreator from ports using Syft's API
 type SyftAdapter struct {
-	scanTimeout time.Duration
+	maxImageSize int64
+	scanTimeout  time.Duration
 }
 
 var _ ports.SBOMCreator = (*SyftAdapter)(nil)
+var ErrImageTooLarge = fmt.Errorf("image size exceeds maximum allowed size")
 
 // NewSyftAdapter initializes the SyftAdapter struct
-func NewSyftAdapter(scanTimeout time.Duration) *SyftAdapter {
+func NewSyftAdapter(scanTimeout time.Duration, maxImageSize int64) *SyftAdapter {
 	return &SyftAdapter{
-		scanTimeout: scanTimeout,
+		maxImageSize: maxImageSize,
+		scanTimeout:  scanTimeout,
 	}
 }
 
@@ -52,7 +65,10 @@ func (s *SyftAdapter) CreateSBOM(ctx context.Context, imageID string, options do
 		Labels: tools.LabelsFromImageID(imageID),
 	}
 	// translate business models into Syft models
-	sourceInput, err := source.ParseInput(imageID, "")
+	if options.Platform == "" {
+		options.Platform = runtime.GOARCH
+	}
+	sourceInput, err := source.ParseInput(imageID, options.Platform)
 	if err != nil {
 		return domainSBOM, err
 	}
@@ -65,20 +81,31 @@ func (s *SyftAdapter) CreateSBOM(ctx context.Context, imageID string, options do
 			Token:     v.Token,
 		}
 	}
-	registryOptions := &image.RegistryOptions{
+	registryOptions := image.RegistryOptions{
 		InsecureSkipTLSVerify: options.InsecureSkipTLSVerify,
 		InsecureUseHTTP:       options.InsecureUseHTTP,
 		Credentials:           credentials,
 		Platform:              options.Platform,
 	}
+	// prepare temporary directory for image download
+	t := file.NewTempDirGenerator("stereoscope")
+	defer func(t *file.TempDirGenerator) {
+		err := t.Cleanup()
+		if err != nil {
+			logger.L().Ctx(ctx).Warning("failed to cleanup temp dir", helpers.String("imageID", imageID), helpers.Error(err))
+		}
+	}(t)
 	// download image
-	// TODO check ephemeral storage usage and see if we can kill the goroutine
 	logger.L().Debug("downloading image", helpers.String("imageID", imageID))
-	src, cleanup, err := source.NewFromRegistry(*sourceInput, registryOptions, []string{})
-	if cleanup != nil {
-		defer cleanup()
-	}
-	if err != nil {
+	src, err := newFromRegistry(ctx, t, sourceInput, registryOptions, s.maxImageSize)
+	switch err {
+	case ErrImageTooLarge:
+		logger.L().Ctx(ctx).Warning("Image exceeds size limit", helpers.Int("maxImageSize", int(s.maxImageSize)), helpers.String("imageID", imageID))
+		domainSBOM.Status = domain.SBOMStatusIncomplete
+		return domainSBOM, nil
+	case nil:
+		// continue
+	default:
 		return domainSBOM, err
 	}
 	// extract packages
@@ -94,13 +121,14 @@ func (s *SyftAdapter) CreateSBOM(ctx context.Context, imageID string, options do
 			Search:      cataloger.DefaultSearchConfig(),
 			Parallelism: 4, // TODO assess this value
 		}
-		pkgCatalog, relationships, actualDistro, err = syft.CatalogPackages(src, catalogOptions)
+		pkgCatalog, relationships, actualDistro, err = syft.CatalogPackages(&src, catalogOptions)
 		return err
 	})
 	switch err {
 	case deadline.ErrTimedOut:
 		logger.L().Ctx(ctx).Warning("Syft timed out", helpers.String("imageID", imageID))
-		domainSBOM.Status = domain.SBOMStatusTimedOut
+		domainSBOM.Status = domain.SBOMStatusIncomplete
+		return domainSBOM, nil
 	case nil:
 		// continue
 	default:
@@ -124,6 +152,201 @@ func (s *SyftAdapter) CreateSBOM(ctx context.Context, imageID string, options do
 	return domainSBOM, err
 }
 
+func newFromRegistry(ctx context.Context, t *file.TempDirGenerator, sourceInput *source.Input, registryOptions image.RegistryOptions, maxImageSize int64) (source.Source, error) {
+	imageTempDir, err := t.NewDirectory("oci-registry-image")
+	if err != nil {
+		return source.Source{}, err
+	}
+	// download image
+	ref, err := name.ParseReference(sourceInput.UserInput, prepareReferenceOptions(registryOptions)...)
+	if err != nil {
+		return source.Source{}, fmt.Errorf("unable to parse registry reference=%q: %+v", sourceInput.UserInput, err)
+	}
+	platform, err := image.NewPlatform(registryOptions.Platform)
+	if err != nil {
+		return source.Source{}, fmt.Errorf("unable to create platform reference=%q: %+v", sourceInput.UserInput, err)
+	}
+	descriptor, err := remote.Get(ref, prepareRemoteOptions(ctx, ref, registryOptions, platform)...)
+	if err != nil {
+		return source.Source{}, fmt.Errorf("failed to get image descriptor from registry: %+v", err)
+	}
+
+	imgRemote, err := descriptor.Image()
+	if err != nil {
+		return source.Source{}, fmt.Errorf("failed to get image from registry: %+v", err)
+	}
+
+	// craft a repo digest from the registry reference and the known digest
+	// note: the descriptor is fetched from the registry, and the descriptor digest is the same as the repo digest
+	repoDigest := fmt.Sprintf("%s/%s@%s", ref.Context().RegistryStr(), ref.Context().RepositoryStr(), descriptor.Digest.String())
+
+	metadata := []image.AdditionalMetadata{
+		image.WithRepoDigests(repoDigest),
+	}
+
+	// make a best effort to get the manifest, should not block getting an image though if it fails
+	if manifestBytes, err := imgRemote.RawManifest(); err == nil {
+		metadata = append(metadata, image.WithManifest(manifestBytes))
+	}
+
+	if platform != nil {
+		metadata = append(metadata,
+			image.WithArchitecture(platform.Architecture, platform.Variant),
+			image.WithOS(platform.OS),
+		)
+	}
+
+	img := image.New(imgRemote, t, imageTempDir, metadata...)
+
+	err = read(img, imgRemote, imageTempDir, maxImageSize)
+	if err != nil {
+		return source.Source{}, fmt.Errorf("could not read image: %+v", err)
+	}
+
+	src, err := source.NewFromImageWithName(img, sourceInput.Location, sourceInput.Name)
+	if err != nil {
+		return source.Source{}, fmt.Errorf("could not populate source with image: %w", err)
+	}
+	return src, nil
+}
+
+func prepareReferenceOptions(registryOptions image.RegistryOptions) []name.Option {
+	var options []name.Option
+	if registryOptions.InsecureUseHTTP {
+		options = append(options, name.Insecure)
+	}
+	return options
+}
+
+func prepareRemoteOptions(ctx context.Context, ref name.Reference, registryOptions image.RegistryOptions, p *image.Platform) (options []remote.Option) {
+	options = append(options, remote.WithContext(ctx))
+
+	if registryOptions.InsecureSkipTLSVerify {
+		t := &http.Transport{
+			//nolint: gosec
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		}
+		options = append(options, remote.WithTransport(t))
+	}
+
+	if p != nil {
+		options = append(options, remote.WithPlatform(containerregistryV1.Platform{
+			Architecture: p.Architecture,
+			OS:           p.OS,
+			Variant:      p.Variant,
+		}))
+	}
+
+	// note: the authn.Authenticator and authn.Keychain options are mutually exclusive, only one may be provided.
+	// If no explicit authenticator can be found, then fallback to the keychain.
+	authenticator := registryOptions.Authenticator(ref.Context().RegistryStr())
+	if authenticator != nil {
+		options = append(options, remote.WithAuth(authenticator))
+	} else {
+		// use the Keychain specified from a docker config file.
+		logger.L().Debug("no registry credentials configured, using the default keychain")
+		options = append(options, remote.WithAuthFromKeychain(authn.DefaultKeychain))
+	}
+
+	return options
+}
+
+func read(i *image.Image, imgRemote containerregistryV1.Image, imageTempDir string, maxImageSize int64) error {
+	var layers = make([]*image.Layer, 0)
+	var err error
+	i.Metadata, err = readImageMetadata(imgRemote)
+	if err != nil {
+		return err
+	}
+
+	v1Layers, err := imgRemote.Layers()
+	if err != nil {
+		return err
+	}
+
+	fileCatalog := image.NewFileCatalog()
+
+	for idx, v1Layer := range v1Layers {
+		layer := image.NewLayer(v1Layer)
+		err := layer.Read(fileCatalog, i.Metadata, idx, imageTempDir)
+		if err != nil {
+			return err
+		}
+		i.Metadata.Size += layer.Metadata.Size
+		// unfortunately we cannot check the size before we gunzip the layer
+		if i.Metadata.Size > maxImageSize {
+			return ErrImageTooLarge
+		}
+		layers = append(layers, layer)
+	}
+
+	i.Layers = layers
+
+	// in order to resolve symlinks all squashed trees must be available
+	err = squash(i, fileCatalog)
+
+	i.FileCatalog = fileCatalog
+	i.SquashedSearchContext = filetree.NewSearchContext(i.SquashedTree(), i.FileCatalog)
+
+	return err
+}
+
+func readImageMetadata(img containerregistryV1.Image) (image.Metadata, error) {
+	id, err := img.ConfigName()
+	if err != nil {
+		return image.Metadata{}, err
+	}
+
+	config, err := img.ConfigFile()
+	if err != nil {
+		return image.Metadata{}, err
+	}
+
+	mediaType, err := img.MediaType()
+	if err != nil {
+		return image.Metadata{}, err
+	}
+
+	rawConfig, err := img.RawConfigFile()
+	if err != nil {
+		return image.Metadata{}, err
+	}
+
+	return image.Metadata{
+		ID:        id.String(),
+		Config:    *config,
+		MediaType: mediaType,
+		RawConfig: rawConfig,
+	}, nil
+}
+
+func squash(i *image.Image, catalog *image.FileCatalog) error {
+	var lastSquashTree filetree.ReadWriter
+
+	for idx, layer := range i.Layers {
+		if idx == 0 {
+			lastSquashTree = layer.Tree.(filetree.ReadWriter)
+			layer.SquashedTree = layer.Tree
+			layer.SquashedSearchContext = filetree.NewSearchContext(layer.SquashedTree, catalog.Index)
+			continue
+		}
+
+		var unionTree = filetree.NewUnionFileTree()
+		unionTree.PushTree(lastSquashTree)
+		unionTree.PushTree(layer.Tree.(filetree.ReadWriter))
+
+		squashedTree, err := unionTree.Squash()
+		if err != nil {
+			return fmt.Errorf("failed to squash tree %d: %w", idx, err)
+		}
+
+		layer.SquashedTree = squashedTree
+		layer.SquashedSearchContext = filetree.NewSearchContext(layer.SquashedTree, catalog.Index)
+		lastSquashTree = squashedTree
+	}
+	return nil
+}
+
 // Version returns Syft's version which is used to tag SBOMs
 func (s *SyftAdapter) Version() string {
 	return tools.PackageVersion("github.com/anchore/syft")