go deeper and add size limit checks

Signed-off-by: Matthias Bertschy <matthias.bertschy@gmail.com>

adapters/mocksbom.go

@@ -50,7 +50,7 @@ func (m MockSBOMAdapter) CreateSBOM(ctx context.Context, imageID string, _ domai
 		},
 	}
 	if m.timeout {
-		sbom.Status = domain.SBOMStatusTimedOut
+		sbom.Status = domain.SBOMStatusIncomplete
 	}
 	return sbom, nil
 }

adapters/mocksbom_test.go

@@ -23,7 +23,7 @@ func TestMockSBOMAdapter_CreateSBOM_Error(t *testing.T) {
 func TestMockSBOMAdapter_CreateSBOM_Timeout(t *testing.T) {
 	m := NewMockSBOMAdapter(false, true)
 	sbom, _ := m.CreateSBOM(context.TODO(), "image", domain.RegistryOptions{})
-	assert.Assert(t, sbom.Status == domain.SBOMStatusTimedOut)
+	assert.Assert(t, sbom.Status == domain.SBOMStatusIncomplete)
 }
 
 func TestMockSBOMAdapter_Version(t *testing.T) {

adapters/v1/syft.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/filetree"
 	"github.com/anchore/stereoscope/pkg/image"
 	"github.com/anchore/syft/syft"
 	"github.com/anchore/syft/syft/artifact"
@@ -33,15 +34,18 @@ import (
 
 // SyftAdapter implements SBOMCreator from ports using Syft's API
 type SyftAdapter struct {
-	scanTimeout time.Duration
+	maxImageSize int64
+	scanTimeout  time.Duration
 }
 
 var _ ports.SBOMCreator = (*SyftAdapter)(nil)
+var ErrImageTooLarge = fmt.Errorf("image size exceeds maximum allowed size")
 
 // NewSyftAdapter initializes the SyftAdapter struct
-func NewSyftAdapter(scanTimeout time.Duration) *SyftAdapter {
+func NewSyftAdapter(scanTimeout time.Duration, maxImageSize int64) *SyftAdapter {
 	return &SyftAdapter{
-		scanTimeout: scanTimeout,
+		maxImageSize: maxImageSize,
+		scanTimeout:  scanTimeout,
 	}
 }
 
@@ -92,10 +96,16 @@ func (s *SyftAdapter) CreateSBOM(ctx context.Context, imageID string, options do
 		}
 	}(t)
 	// download image
-	// TODO check ephemeral storage usage and see if we can kill the goroutine
 	logger.L().Debug("downloading image", helpers.String("imageID", imageID))
-	src, err := newFromRegistry(ctx, t, sourceInput, registryOptions)
-	if err != nil {
+	src, err := newFromRegistry(t, sourceInput, registryOptions, s.maxImageSize)
+	switch err {
+	case ErrImageTooLarge:
+		logger.L().Ctx(ctx).Warning("Image exceeds size limit", helpers.Int("maxImageSize", int(s.maxImageSize)), helpers.String("imageID", imageID))
+		domainSBOM.Status = domain.SBOMStatusIncomplete
+		return domainSBOM, nil
+	case nil:
+		// continue
+	default:
 		return domainSBOM, err
 	}
 	// extract packages
@@ -117,7 +127,7 @@ func (s *SyftAdapter) CreateSBOM(ctx context.Context, imageID string, options do
 	switch err {
 	case deadline.ErrTimedOut:
 		logger.L().Ctx(ctx).Warning("Syft timed out", helpers.String("imageID", imageID))
-		domainSBOM.Status = domain.SBOMStatusTimedOut
+		domainSBOM.Status = domain.SBOMStatusIncomplete
 		return domainSBOM, nil
 	case nil:
 		// continue
@@ -142,7 +152,7 @@ func (s *SyftAdapter) CreateSBOM(ctx context.Context, imageID string, options do
 	return domainSBOM, err
 }
 
-func newFromRegistry(ctx context.Context, t *file.TempDirGenerator, sourceInput *source.Input, registryOptions image.RegistryOptions) (source.Source, error) {
+func newFromRegistry(t *file.TempDirGenerator, sourceInput *source.Input, registryOptions image.RegistryOptions, maxImageSize int64) (source.Source, error) {
 	imageTempDir, err := t.NewDirectory("oci-registry-image")
 	if err != nil {
 		return source.Source{}, err
@@ -156,7 +166,7 @@ func newFromRegistry(ctx context.Context, t *file.TempDirGenerator, sourceInput
 	if err != nil {
 		return source.Source{}, fmt.Errorf("unable to create platform reference=%q: %+v", sourceInput.UserInput, err)
 	}
-	descriptor, err := remote.Get(ref, prepareRemoteOptions(ctx, ref, registryOptions, platform)...)
+	descriptor, err := remote.Get(ref, prepareRemoteOptions(ref, registryOptions, platform)...)
 	if err != nil {
 		return source.Source{}, fmt.Errorf("failed to get image descriptor from registry: %+v", err)
 	}
@@ -188,7 +198,7 @@ func newFromRegistry(ctx context.Context, t *file.TempDirGenerator, sourceInput
 
 	img := image.New(imgRemote, t, imageTempDir, metadata...)
 
-	err = img.Read()
+	err = read(img, imgRemote, imageTempDir, maxImageSize)
 	if err != nil {
 		return source.Source{}, fmt.Errorf("could not read image: %+v", err)
 	}
@@ -208,8 +218,8 @@ func prepareReferenceOptions(registryOptions image.RegistryOptions) []name.Optio
 	return options
 }
 
-func prepareRemoteOptions(ctx context.Context, ref name.Reference, registryOptions image.RegistryOptions, p *image.Platform) (options []remote.Option) {
-	options = append(options, remote.WithContext(ctx))
+func prepareRemoteOptions(ref name.Reference, registryOptions image.RegistryOptions, p *image.Platform) (options []remote.Option) {
+	options = append(options, remote.WithContext(context.TODO()))
 
 	if registryOptions.InsecureSkipTLSVerify {
 		t := &http.Transport{
@@ -241,6 +251,102 @@ func prepareRemoteOptions(ctx context.Context, ref name.Reference, registryOptio
 	return options
 }
 
+func read(i *image.Image, imgRemote containerregistryV1.Image, imageTempDir string, maxImageSize int64) error {
+	var layers = make([]*image.Layer, 0)
+	var err error
+	i.Metadata, err = readImageMetadata(imgRemote)
+	if err != nil {
+		return err
+	}
+
+	v1Layers, err := imgRemote.Layers()
+	if err != nil {
+		return err
+	}
+
+	fileCatalog := image.NewFileCatalog()
+
+	for idx, v1Layer := range v1Layers {
+		layer := image.NewLayer(v1Layer)
+		err := layer.Read(fileCatalog, i.Metadata, idx, imageTempDir)
+		if err != nil {
+			return err
+		}
+		i.Metadata.Size += layer.Metadata.Size
+		// unfortunately we cannot check the size before we gunzip the layer
+		if i.Metadata.Size > maxImageSize {
+			return ErrImageTooLarge
+		}
+		layers = append(layers, layer)
+	}
+
+	i.Layers = layers
+
+	// in order to resolve symlinks all squashed trees must be available
+	err = squash(i, fileCatalog)
+
+	i.FileCatalog = fileCatalog
+	i.SquashedSearchContext = filetree.NewSearchContext(i.SquashedTree(), i.FileCatalog)
+
+	return err
+}
+
+func readImageMetadata(img containerregistryV1.Image) (image.Metadata, error) {
+	id, err := img.ConfigName()
+	if err != nil {
+		return image.Metadata{}, err
+	}
+
+	config, err := img.ConfigFile()
+	if err != nil {
+		return image.Metadata{}, err
+	}
+
+	mediaType, err := img.MediaType()
+	if err != nil {
+		return image.Metadata{}, err
+	}
+
+	rawConfig, err := img.RawConfigFile()
+	if err != nil {
+		return image.Metadata{}, err
+	}
+
+	return image.Metadata{
+		ID:        id.String(),
+		Config:    *config,
+		MediaType: mediaType,
+		RawConfig: rawConfig,
+	}, nil
+}
+
+func squash(i *image.Image, catalog *image.FileCatalog) error {
+	var lastSquashTree filetree.ReadWriter
+
+	for idx, layer := range i.Layers {
+		if idx == 0 {
+			lastSquashTree = layer.Tree.(filetree.ReadWriter)
+			layer.SquashedTree = layer.Tree
+			layer.SquashedSearchContext = filetree.NewSearchContext(layer.SquashedTree, catalog.Index)
+			continue
+		}
+
+		var unionTree = filetree.NewUnionFileTree()
+		unionTree.PushTree(lastSquashTree)
+		unionTree.PushTree(layer.Tree.(filetree.ReadWriter))
+
+		squashedTree, err := unionTree.Squash()
+		if err != nil {
+			return fmt.Errorf("failed to squash tree %d: %w", idx, err)
+		}
+
+		layer.SquashedTree = squashedTree
+		layer.SquashedSearchContext = filetree.NewSearchContext(layer.SquashedTree, catalog.Index)
+		lastSquashTree = squashedTree
+	}
+	return nil
+}
+
 // Version returns Syft's version which is used to tag SBOMs
 func (s *SyftAdapter) Version() string {
 	return tools.PackageVersion("github.com/anchore/syft")

adapters/v1/syft_test.go

@@ -21,11 +21,12 @@ func fileContent(path string) []byte {
 
 func Test_syftAdapter_CreateSBOM(t *testing.T) {
 	tests := []struct {
-		name    string
-		imageID string
-		format  string
-		options domain.RegistryOptions
-		wantErr bool
+		name         string
+		imageID      string
+		format       string
+		maxImageSize int64
+		options      domain.RegistryOptions
+		wantErr      bool
 	}{
 		{
 			name:    "empty image produces empty SBOM",
@@ -45,13 +46,20 @@ func Test_syftAdapter_CreateSBOM(t *testing.T) {
 				Credentials: []domain.RegistryCredentials{
 					{
 						Authority: "docker.io",
-						Username: "username",
-						Password: "password",
-						Token: "token",
+						Username:  "username",
+						Password:  "password",
+						Token:     "token",
 					},
 				},
 			},
 		},
+		{
+			name:         "big image produces error",
+			imageID:      "library/alpine@sha256:e2e16842c9b54d985bf1ef9242a313f36b856181f188de21313820e177002501",
+			format:       "null",
+			maxImageSize: 1,
+			wantErr:      true,
+		},
 		{
 			name:    "system tests image",
 			imageID: "public-registry.systest-ns-bpf7:5000/nginx:test",
@@ -61,7 +69,11 @@ func Test_syftAdapter_CreateSBOM(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := NewSyftAdapter(5 * time.Minute)
+			maxImageSize := int64(512 * 1024 * 1024)
+			if tt.maxImageSize > 0 {
+				maxImageSize = tt.maxImageSize
+			}
+			s := NewSyftAdapter(5*time.Minute, maxImageSize)
 			got, err := s.CreateSBOM(context.TODO(), tt.imageID, tt.options)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("CreateSBOM() error = %v, wantErr %v", err, tt.wantErr)
@@ -76,7 +88,7 @@ func Test_syftAdapter_CreateSBOM(t *testing.T) {
 }
 
 func Test_syftAdapter_Version(t *testing.T) {
-	s := NewSyftAdapter(5 * time.Minute)
+	s := NewSyftAdapter(5*time.Minute, 512*1024*1024)
 	version := s.Version()
 	assert.Assert(t, version != "")
 }
@@ -87,7 +99,7 @@ func Test_syftAdapter_transformations(t *testing.T) {
 	tools.EnsureSetup(t, err == nil)
 	spdxSBOM, err := domainToSpdx(*sbom.Content)
 	tools.EnsureSetup(t, err == nil)
-	s := NewSyftAdapter(5 * time.Minute)
+	s := NewSyftAdapter(5*time.Minute, 512*1024*1024)
 	domainSBOM, err := s.spdxToDomain(spdxSBOM)
 	tools.EnsureSetup(t, err == nil)
 	diff := deep.Equal(sbom.Content, domainSBOM)

cmd/http/main.go

@@ -50,7 +50,7 @@ func main() {
 			logger.L().Ctx(ctx).Fatal("storage initialization error", helpers.Error(err))
 		}
 	}
-	sbomAdapter := v1.NewSyftAdapter(c.ScanTimeout)
+	sbomAdapter := v1.NewSyftAdapter(c.ScanTimeout, c.MaxImageSize)
 	cveAdapter := v1.NewGrypeAdapter()
 	platform := v1.NewArmoAdapter(c.AccountID, c.BackendOpenAPI, c.EventReceiverRestURL)
 	service := services.NewScanService(sbomAdapter, storage, cveAdapter, storage, platform, c.Storage)

config/config.go

@@ -11,6 +11,7 @@ type Config struct {
 	BackendOpenAPI       string        `mapstructure:"backendOpenAPI"`
 	ClusterName          string        `mapstructure:"clusterName"`
 	EventReceiverRestURL string        `mapstructure:"eventReceiverRestURL"`
+	MaxImageSize         int64         `mapstructure:"maxImageSize"`
 	ScanConcurrency      int           `mapstructure:"scanConcurrency"`
 	ScanTimeout          time.Duration `mapstructure:"scanTimeout"`
 	Storage              bool          `mapstructure:"storage"`
@@ -22,6 +23,7 @@ func LoadConfig(path string) (Config, error) {
 	viper.SetConfigName("clusterData")
 	viper.SetConfigType("json")
 
+	viper.SetDefault("maxImageSize", 512*1024*1024)
 	viper.SetDefault("scanConcurrency", 1)
 	viper.SetDefault("scanTimeout", 5*time.Minute)
 

core/domain/sbom.go

@@ -5,7 +5,7 @@ import (
 )
 
 const (
-	SBOMStatusTimedOut = "timed out"
+	SBOMStatusIncomplete = "incomplete"
 )
 
 // SBOM contains an SPDX SBOM in JSON format with some metadata

core/services/scan.go

@@ -142,7 +142,7 @@ func (s *ScanService) ScanCVE(ctx context.Context) error {
 		}
 
 		// do not process timed out SBOM
-		if sbom.Status == domain.SBOMStatusTimedOut {
+		if sbom.Status == domain.SBOMStatusIncomplete {
 			return errors.New("SBOM incomplete due to timeout, skipping CVE scan")
 		}
 
@@ -232,7 +232,7 @@ func (s *ScanService) ScanRegistry(ctx context.Context) error {
 	}
 
 	// do not process timed out SBOM
-	if sbom.Status == domain.SBOMStatusTimedOut {
+	if sbom.Status == domain.SBOMStatusIncomplete {
 		return errors.New("SBOM incomplete due to timeout, skipping CVE scan")
 	}