External sources configuration (#1158)

cmd/syft/cli/options/packages.go

@@ -14,13 +14,20 @@ import (
 )
 
 type PackagesOptions struct {
-	Scope              string
-	Output             []string
-	OutputTemplatePath string
-	File               string
-	Platform           string
-	Exclude            []string
-	Catalogers         []string
+	Scope                  string
+	Output                 []string
+	OutputTemplatePath     string
+	File                   string
+	Platform               string
+	Host                   string
+	Username               string
+	Password               string
+	Dockerfile             string
+	Exclude                []string
+	OverwriteExistingImage bool
+	ImportTimeout          uint
+	Catalogers             []string
+	ExternalSourcesEnabled bool
 }
 
 var _ Interface = (*PackagesOptions)(nil)
@@ -47,9 +54,19 @@ func (o *PackagesOptions) AddFlags(cmd *cobra.Command, v *viper.Viper) error {
 	cmd.Flags().StringArrayVarP(&o.Catalogers, "catalogers", "", nil,
 		"enable one or more package catalogers")
 
+	cmd.Flags().BoolVarP(&o.OverwriteExistingImage, "overwrite-existing-image", "", false,
+		"overwrite an existing image during the upload to Anchore Enterprise")
+
+	cmd.Flags().UintVarP(&o.ImportTimeout, "import-timeout", "", 30,
+		"set a timeout duration (in seconds) for the upload to Anchore Enterprise")
+
+	cmd.Flags().BoolVarP(&o.ExternalSourcesEnabled, "external-sources-enabled", "", false,
+		"shut off any use of external sources during sbom generation (default false")
+
 	return bindPackageConfigOptions(cmd.Flags(), v)
 }
 
+//nolint:funlen
 func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error {
 	// Formatting & Input options //////////////////////////////////////////////
 
@@ -81,5 +98,9 @@ func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error {
 		return err
 	}
 
+	if err := v.BindPFlag("external_sources.external-sources-enabled", flags.Lookup("external-sources-enabled")); err != nil {
+		return err
+	}
+
 	return nil
 }

internal/config/application.go

@@ -56,6 +56,7 @@ type Application struct {
 	Exclusions         []string           `yaml:"exclude" json:"exclude" mapstructure:"exclude"`
 	Attest             attest             `yaml:"attest" json:"attest" mapstructure:"attest"`
 	Platform           string             `yaml:"platform" json:"platform" mapstructure:"platform"`
+	ExternalSources    ExternalSources    `yaml:"external_sources" json:"external_sources" mapstructure:"external_sources"`
 }
 
 func (cfg Application) ToCatalogerConfig() cataloger.Config {
@@ -65,7 +66,8 @@ func (cfg Application) ToCatalogerConfig() cataloger.Config {
 			IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives,
 			Scope:                    cfg.Package.Cataloger.ScopeOpt,
 		},
-		Catalogers: cfg.Catalogers,
+		Catalogers:             cfg.Catalogers,
+		ExternalSourcesEnabled: cfg.ExternalSources.ExternalSourcesEnabled,
 	}
 }
 

internal/config/datasources.go

@@ -0,0 +1,11 @@
+package config
+
+import "github.com/spf13/viper"
+
+type ExternalSources struct {
+	ExternalSourcesEnabled bool `yaml:"external-sources-enabled" json:"external-sources-enabled" mapstructure:"external-sources-enabled"`
+}
+
+func (e ExternalSources) loadDefaultValues(v *viper.Viper) {
+	v.SetDefault("external-sources-enabled", false)
+}

syft/pkg/cataloger/cataloger.go

@@ -23,7 +23,6 @@ import (
 	"github.com/anchore/syft/syft/pkg/cataloger/php"
 	"github.com/anchore/syft/syft/pkg/cataloger/portage"
 	"github.com/anchore/syft/syft/pkg/cataloger/python"
-	"github.com/anchore/syft/syft/pkg/cataloger/rekor"
 	"github.com/anchore/syft/syft/pkg/cataloger/rpm"
 	"github.com/anchore/syft/syft/pkg/cataloger/ruby"
 	"github.com/anchore/syft/syft/pkg/cataloger/rust"
@@ -105,7 +104,6 @@ func AllCatalogers(cfg Config) []pkg.Cataloger {
 		cpp.NewConanCataloger(),
 		portage.NewPortageCataloger(),
 		haskell.NewHackageCataloger(),
-		rekor.NewRekorCataloger(),
 	}, cfg.Catalogers)
 }
 

syft/pkg/cataloger/cataloger_test.go

@@ -3,11 +3,18 @@ package cataloger
 import (
 	"testing"
 
+<<<<<<< HEAD
 	"github.com/stretchr/testify/assert"
 
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/source"
+=======
+	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+	"github.com/stretchr/testify/assert"
+>>>>>>> 1329688 (External sources configuration (#1158))
 )
 
 var _ pkg.Cataloger = (*dummy)(nil)
@@ -24,12 +31,17 @@ func (d dummy) Catalog(_ source.FileResolver) ([]pkg.Package, []artifact.Relatio
 	panic("not implemented")
 }
 
+func (d dummy) UsesExternalSources() bool {
+	return false
+}
+
 func Test_filterCatalogers(t *testing.T) {
 	tests := []struct {
-		name       string
-		patterns   []string
-		catalogers []string
-		want       []string
+		name                   string
+		patterns               []string
+		ExternalSourcesEnabled bool
+		catalogers             []string
+		want                   []string
 	}{
 		{
 			name:     "no filtering",
@@ -144,14 +156,30 @@ func Test_filterCatalogers(t *testing.T) {
 				"go-module-binary-cataloger",
 			},
 		},
+		{ // Note: no catalogers with external sources are currently implemented
+			name:                   "external sources enabled",
+			patterns:               []string{"all"},
+			ExternalSourcesEnabled: true,
+			catalogers: []string{
+				"ruby-gemspec-cataloger",
+				"python-package-cataloger",
+				"rekor-cataloger",
+			},
+			want: []string{
+				"ruby-gemspec-cataloger",
+				"python-package-cataloger",
+				"rekor-cataloger",
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			var catalogers []pkg.Cataloger
 			for _, n := range tt.catalogers {
 				catalogers = append(catalogers, dummy{name: n})
 			}
-			got := filterCatalogers(catalogers, tt.patterns)
+			cfg := Config{Catalogers: tt.patterns, ExternalSourcesEnabled: tt.ExternalSourcesEnabled}
+			got := filterCatalogers(catalogers, cfg)
 			var gotNames []string
 			for _, g := range got {
 				gotNames = append(gotNames, g.Name())

syft/pkg/cataloger/common/generic_cataloger.go

@@ -39,6 +39,11 @@ func (c *GenericCataloger) Name() string {
 	return c.upstreamCataloger
 }
 
+// UsesExternalSources indicates that any GenericCatalogor does not use external sources
+func (c *GenericCataloger) UsesExternalSources() bool {
+	return false
+}
+
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
 func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	var packages []pkg.Package

syft/pkg/cataloger/config.go

@@ -5,8 +5,9 @@ import (
 )
 
 type Config struct {
-	Search     SearchConfig
-	Catalogers []string
+	Search                 SearchConfig
+	Catalogers             []string
+	ExternalSourcesEnabled bool
 }
 
 func DefaultConfig() Config {

syft/pkg/cataloger/golang/binary_cataloger.go

@@ -0,0 +1,66 @@
+/*
+Package golang provides a concrete Cataloger implementation for go.mod files.
+*/
+package golang
+
+import (
+	"fmt"
+
+	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/pkg/cataloger/internal/unionreader"
+	"github.com/anchore/syft/syft/source"
+)
+
+const catalogerName = "go-module-binary-cataloger"
+
+type Cataloger struct{}
+
+// NewGoModuleBinaryCataloger returns a new Golang cataloger object.
+func NewGoModuleBinaryCataloger() *Cataloger {
+	return &Cataloger{}
+}
+
+// Name returns a string that uniquely describes a cataloger
+func (c *Cataloger) Name() string {
+	return catalogerName
+}
+
+// UsesExternalSources indicates that the golang binary cataloger does not use external sources
+func (c *Cataloger) UsesExternalSources() bool {
+	return false
+}
+
+// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation.
+func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
+	var pkgs []pkg.Package
+
+	fileMatches, err := resolver.FilesByMIMEType(internal.ExecutableMIMETypeSet.List()...)
+	if err != nil {
+		return pkgs, nil, fmt.Errorf("failed to find bin by mime types: %w", err)
+	}
+
+	for _, location := range fileMatches {
+		readerCloser, err := resolver.FileContentsByLocation(location)
+		if err != nil {
+			log.Warnf("golang cataloger: opening file: %v", err)
+			continue
+		}
+
+		reader, err := unionreader.GetUnionReader(readerCloser)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		mods, archs := scanFile(reader, location.RealPath)
+		internal.CloseAndLogError(readerCloser, location.RealPath)
+
+		for i, mod := range mods {
+			pkgs = append(pkgs, buildGoPkgInfo(location, mod, archs[i])...)
+		}
+	}
+
+	return pkgs, nil, nil
+}

syft/pkg/cataloger/portage/cataloger.go

@@ -37,6 +37,11 @@ func (c *Cataloger) Name() string {
 	return "portage-cataloger"
 }
 
+// UsesExternalSources indicates that the portage cataloger does not use external sources
+func (c *Cataloger) UsesExternalSources() bool {
+	return false
+}
+
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing portage support files.
 func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	dbFileMatches, err := resolver.FilesByGlob(pkg.PortageDBGlob)

syft/pkg/cataloger/python/package_cataloger.go

@@ -32,6 +32,11 @@ func (c *PackageCataloger) Name() string {
 	return "python-package-cataloger"
 }
 
+// UsesExternalSources indicates that the python package cataloger does not use external sources
+func (c *PackageCataloger) UsesExternalSources() bool {
+	return false
+}
+
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations.
 func (c *PackageCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	var fileMatches []source.Location

syft/pkg/cataloger/rpm/db_cataloger.go

@@ -27,6 +27,11 @@ func (c *DBCataloger) Name() string {
 	return dbCatalogerName
 }
 
+// UsesExternalSources indicates that the rpmdb cataloger does not use external sources
+func (c *Cataloger) UsesExternalSources() bool {
+	return false
+}
+
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation.
 func (c *DBCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	fileMatches, err := resolver.FilesByGlob(pkg.RpmDBGlob)

syft/pkg/cataloger/rust/audit_binary_cataloger.go

@@ -28,6 +28,11 @@ func (c *Cataloger) Name() string {
 	return catalogerName
 }
 
+// UsesExternalSources indicates that the audit binary cataloger does not use external sources
+func (c *Cataloger) UsesExternalSources() bool {
+	return false
+}
+
 // Catalog identifies executables then attempts to read Rust dependency information from them
 func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	var pkgs []pkg.Package