3 Major features for SPDX and bugfixes

cmd/bom/cmd/generate.go

@@ -19,11 +19,13 @@ package cmd
 import (
 	"fmt"
 	"net/url"
+	"os"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"k8s.io/release/pkg/spdx"
+	"sigs.k8s.io/release-utils/util"
 )
 
 var genOpts = &generateOptions{}
@@ -46,28 +48,57 @@ of analyzers designed to add more sense to common base images.
 	SilenceErrors:     true,
 	PersistentPreRunE: initLogging,
 	RunE: func(cmd *cobra.Command, args []string) error {
+		for i, arg := range args {
+			if util.Exists(arg) {
+				file, err := os.Open(arg)
+				if err != nil {
+					return errors.Wrapf(err, "checking argument %d", i)
+				}
+				fileInfo, err := file.Stat()
+				if err != nil {
+					return errors.Wrapf(err, "calling stat on argument %d", i)
+				}
+				if fileInfo.IsDir() {
+					genOpts.directories = append(genOpts.directories, arg)
+				}
+			}
+		}
 		return generateBOM(genOpts)
 	},
 }
 
 type generateOptions struct {
-	analyze    bool
-	namespace  string
-	outputFile string
-	images     []string
-	tarballs   []string
-	files      []string
+	analyze        bool
+	noGitignore    bool
+	noGoModules    bool
+	namespace      string
+	outputFile     string
+	images         []string
+	tarballs       []string
+	files          []string
+	directories    []string
+	ignorePatterns []string
 }
 
 // Validate verify options consistency
 func (opts *generateOptions) Validate() error {
-	if len(opts.images) == 0 && len(opts.files) == 0 && len(opts.tarballs) == 0 {
+	if len(opts.images) == 0 && len(opts.files) == 0 && len(opts.tarballs) == 0 && len(opts.directories) == 0 {
 		return errors.New("to generate a SPDX BOM you have to provide at least one image or file")
 	}
 
 	// A namespace URL is required
 	if opts.namespace == "" {
-		return errors.New("A namespace (URL) must be defined to have a compliant SPDX BOM")
+		msg := "\nYou did not specify a namespace for your document. This is an error.\n"
+		msg += "To produce a valid SPDX SBOM, the document has to have an URI as its\n"
+		msg += "namespace.\n\nYou can use http://example.com/ for now if you are testing but your\n"
+		msg += "final document must have the namespace URI pointing to the location where\n"
+		msg += "you SBOM will be referenced in the future.\n\n"
+		msg += "For more details, check the SPDX documentation here:\n"
+		msg += "https://spdx.github.io/spdx-spec/2-document-creation-information/#25-spdx-document-namespace\n\n"
+		msg += "Hint: --namespace is your friend here\n\n"
+		logrus.Info(msg)
+
+		return errors.New("A namespace URI must be defined to have a compliant SPDX BOM")
 	}
 
 	// CHeck namespace is a valid URL
@@ -102,6 +133,35 @@ func init() {
 		"list of docker archive tarballs to include in the manifest",
 	)
 
+	generateCmd.PersistentFlags().StringSliceVarP(
+		&genOpts.directories,
+		"dirs",
+		"d",
+		[]string{},
+		"list of directories to include in the manifest as packages",
+	)
+
+	generateCmd.PersistentFlags().StringSliceVar(
+		&genOpts.ignorePatterns,
+		"ignore",
+		[]string{},
+		"list of regexp patterns to ignore when scanning directories",
+	)
+
+	generateCmd.PersistentFlags().BoolVar(
+		&genOpts.noGitignore,
+		"no-gitignore",
+		false,
+		"don't use exclusions from .gitignore files",
+	)
+
+	generateCmd.PersistentFlags().BoolVar(
+		&genOpts.noGitignore,
+		"no-gomod",
+		false,
+		"don't perform go.mod analysis, sbom will not include data about go packages",
+	)
+
 	generateCmd.PersistentFlags().StringVarP(
 		&genOpts.namespace,
 		"namespace",
@@ -134,14 +194,22 @@ func generateBOM(opts *generateOptions) error {
 	logrus.Info("Generating SPDX Bill of Materials")
 
 	builder := spdx.NewDocBuilder()
-	doc, err := builder.Generate(&spdx.DocGenerateOptions{
-		Tarballs:      opts.tarballs,
-		Files:         opts.files,
-		Images:        opts.images,
-		OutputFile:    opts.outputFile,
-		Namespace:     "",
-		AnalyseLayers: opts.analyze,
-	})
+	builderOpts := &spdx.DocGenerateOptions{
+		Tarballs:         opts.tarballs,
+		Files:            opts.files,
+		Images:           opts.images,
+		Directories:      opts.directories,
+		OutputFile:       opts.outputFile,
+		Namespace:        opts.namespace,
+		AnalyseLayers:    opts.analyze,
+		ProcessGoModules: !opts.noGoModules,
+	}
+
+	// We only replace the ignore patterns one or more where defined
+	if len(opts.ignorePatterns) > 0 {
+		builderOpts.IgnorePatterns = opts.ignorePatterns
+	}
+	doc, err := builder.Generate(builderOpts)
 	if err != nil {
 		return errors.Wrap(err, "generating doc")
 	}

go.mod

@@ -33,8 +33,10 @@ require (
 	github.com/spiegel-im-spiegel/go-cvss v0.4.0
 	github.com/stretchr/testify v1.7.0
 	github.com/yuin/goldmark v1.3.7
+	golang.org/x/mod v0.4.2
 	golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420
 	golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c
+	golang.org/x/tools v0.1.1
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
 	google.golang.org/api v0.46.0
 	google.golang.org/genproto v0.0.0-20210517163617-5e0236093d7a

pkg/license/catalog.go

@@ -33,7 +33,7 @@ type CatalogOptions struct {
 // are in the temporary OS directory and are created if the do not exist
 var DefaultCatalogOpts = &CatalogOptions{}
 
-// NewSPDXWithOptions returns a SPDX object with the specified options
+// NewCatalogWithOptions returns a SPDX object with the specified options
 func NewCatalogWithOptions(opts *CatalogOptions) (catalog *Catalog, err error) {
 	// Create the license downloader
 	doptions := DefaultDownloaderOpts
@@ -76,7 +76,7 @@ type Catalog struct {
 
 // WriteLicensesAsText writes the SPDX license collection to text files
 func (catalog *Catalog) WriteLicensesAsText(targetDir string) error {
-	logrus.Info("Writing SPDX licenses to " + targetDir)
+	logrus.Infof("Writing %d SPDX licenses to %s", len(catalog.List.Licenses), targetDir)
 	if catalog.List.Licenses == nil {
 		return errors.New("unable to write licenses, they have not been loaded yet")
 	}

pkg/license/download.go

@@ -147,25 +147,28 @@ func (ddi *DefaultDownloaderImpl) GetLicenses() (licenses *List, err error) {
 	// Create a new Throttler that will get `parallelDownloads` urls at a time
 	t := throttler.New(ddi.Options.parallelDownloads, len(licenseList.LicenseData))
 	for _, l := range licenseList.LicenseData {
-		licURL := l.Reference
+		licURL := l.DetailsURL
 		// If the license URLs have a local reference
 		if strings.HasPrefix(licURL, "./") {
 			licURL = LicenseDataURL + strings.TrimPrefix(licURL, "./")
 		}
 		// Launch a goroutine to fetch the URL.
 		go func(url string) {
-			var err error
+			var lic *License
 			defer t.Done(err)
-			l, err := ddi.getLicenseFromURL(url)
+			lic, err = ddi.getLicenseFromURL(url)
 			if err != nil {
+				logrus.Error(err)
 				return
 			}
 			logrus.Debugf("Got license: %s from %s", l.LicenseID, url)
-			licenseList.Add(l)
+			licenseList.Add(lic)
 		}(licURL)
 		t.Throttle()
 	}
 
+	logrus.Infof("Downloaded %d licenses", len(licenseList.Licenses))
+
 	// If the throttler collected errors, return those
 	if t.Err() != nil {
 		return nil, t.Err()

pkg/license/implementation.go

@@ -167,14 +167,16 @@ func (d *ReaderDefaultImpl) Initialize(opts *ReaderOptions) error {
 		return errors.Wrap(err, "loading licenses")
 	}
 
+	logrus.Infof("Writing license data to %s", opts.CachePath())
+
 	// Write the licenses to disk as th classifier will need them
 	if err := catalog.WriteLicensesAsText(opts.LicensesPath()); err != nil {
 		return errors.Wrap(err, "writing license data to disk")
 	}
 
 	// Create the implementation's classifier
 	d.lc = licenseclassifier.NewClassifier(opts.ConfidenceThreshold)
-	return errors.Wrap(d.lc.LoadLicenses(opts.LicensesPath()), "loading licenses at init")
+	return errors.Wrap(d.lc.LoadLicenses(opts.CachePath()), "loading licenses at init")
 }
 
 // Classifier returns the license classifier

pkg/license/license.go

@@ -28,7 +28,6 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-	"sigs.k8s.io/release-utils/util"
 )
 
 const (
@@ -120,32 +119,6 @@ func (ro *ReaderOptions) Validate() error {
 		return errors.Wrap(err, "checking working directory")
 	}
 
-	// Check the cache directory
-	if !util.Exists(ro.CacheDir) {
-		if ro.CacheDir == "" {
-			if err := os.MkdirAll(
-				filepath.Join(ro.WorkDir, defaultCacheSubDir), os.FileMode(0o755),
-			); err != nil {
-				return errors.Wrap(err, "creating cache directory")
-			}
-		} else {
-			return errors.New("specified cache directory does not exist")
-		}
-	}
-
-	// Check the licenses directory
-	if !util.Exists(ro.LicenseDir) {
-		if ro.LicenseDir == "" {
-			if err := os.MkdirAll(
-				filepath.Join(ro.WorkDir, defaultLicenseSubDir), os.FileMode(0o755),
-			); err != nil {
-				return errors.Wrap(err, "creating licenses directory")
-			}
-		} else {
-			return errors.New("specified licenses directory does not exist")
-		}
-	}
-
 	// TODO check dirs
 	return nil
 }
@@ -159,7 +132,7 @@ func (ro *ReaderOptions) CachePath() string {
 	return filepath.Join(ro.WorkDir, defaultCacheSubDir)
 }
 
-// LicensesPath return the full path to the downloads cache
+// LicensesPath return the full path the dir where the licenses are
 func (ro *ReaderOptions) LicensesPath() string {
 	if ro.LicenseDir != "" {
 		return ro.LicenseDir

pkg/spdx/builder.go

@@ -61,18 +61,22 @@ func (db *DocBuilder) Generate(genopts *DocGenerateOptions) (*Document, error) {
 }
 
 type DocGenerateOptions struct {
-	Tarballs      []string // A slice of tar paths
-	Files         []string // A slice of naked files to include in the bom
-	Images        []string // A slice of docker images
-	OutputFile    string   // Output location
-	Namespace     string   // Namespace for the document (a unique URI)
-	AnalyseLayers bool     // A flag that controls if deep layer analysis should be performed
+	AnalyseLayers    bool     // A flag that controls if deep layer analysis should be performed
+	NoGitignore      bool     // Do not read exclusions from gitignore file
+	ProcessGoModules bool     // Analyze go.mod to include data about packages
+	OutputFile       string   // Output location
+	Namespace        string   // Namespace for the document (a unique URI)
+	Tarballs         []string // A slice of tar paths
+	Files            []string // A slice of naked files to include in the bom
+	Images           []string // A slice of docker images
+	Directories      []string // A slice of directories to convert into packages
+	IgnorePatterns   []string // a slice of regexp patterns to ignore when scanning dirs
 }
 
 func (o *DocGenerateOptions) Validate() error {
-	if len(o.Tarballs) == 0 && len(o.Files) == 0 && len(o.Images) == 0 {
+	if len(o.Tarballs) == 0 && len(o.Files) == 0 && len(o.Images) == 0 && len(o.Directories) == 0 {
 		return errors.New(
-			"To build a document at least an image, tarball or a file has to be specified",
+			"To build a document at least an image, tarball, directory or a file has to be specified",
 		)
 	}
 	return nil
@@ -104,7 +108,11 @@ func (builder defaultDocBuilderImpl) GenerateDoc(
 	}
 
 	spdx := NewSPDX()
-	spdx.options.AnalyzeLayers = genopts.AnalyseLayers
+	if len(genopts.IgnorePatterns) > 0 {
+		spdx.Options().IgnorePatterns = genopts.IgnorePatterns
+	}
+	spdx.Options().AnalyzeLayers = genopts.AnalyseLayers
+	spdx.Options().ProcessGoModules = genopts.ProcessGoModules
 
 	if !util.Exists(opts.WorkDir) {
 		if err := os.MkdirAll(opts.WorkDir, os.FileMode(0o755)); err != nil {
@@ -123,8 +131,19 @@ func (builder defaultDocBuilderImpl) GenerateDoc(
 	doc.Namespace = genopts.Namespace
 
 	if genopts.Namespace == "" {
-		logrus.Warn("Document namespace is empty, a mock URI will be supplied but the doc will not be valid")
-		doc.Namespace = "http://example.com/"
+		return nil, errors.New("unable to generate doc, namespace URI is not defined")
+	}
+
+	for _, i := range genopts.Directories {
+		logrus.Infof("Processing directory %s", i)
+		pkg, err := spdx.PackageFromDirectory(i)
+		if err != nil {
+			return nil, errors.Wrap(err, "generating package from directory")
+		}
+
+		if err := doc.AddPackage(pkg); err != nil {
+			return nil, errors.Wrap(err, "adding directory package to document")
+		}
 	}
 
 	for _, i := range genopts.Images {

pkg/spdx/document.go

@@ -133,7 +133,7 @@ func (d *Document) Render() (doc string, err error) {
 	}
 
 	if d.Name == "" {
-		d.Name = "BOM-SPDX-" + uuid.New().String()
+		d.Name = "SBOM-SPDX-" + uuid.New().String()
 		logrus.Warnf("Document has no name defined, automatically set to " + d.Name)
 	}
 

pkg/spdx/file.go

@@ -41,7 +41,7 @@ var fileTemplate = `{{ if .Name }}FileName: {{ .Name }}
 {{- end -}}
 {{- end -}}
 LicenseConcluded: {{ if .LicenseConcluded }}{{ .LicenseConcluded }}{{ else }}NOASSERTION{{ end }}
-LicenseInfoInFile: {{ if .LicenseInfoInFile }}LicenseInfoInFile: {{ .LicenseInfoInFile }}{{ else }}NOASSERTION{{ end }}
+LicenseInfoInFile: {{ if .LicenseInfoInFile }}{{ .LicenseInfoInFile }}{{ else }}NOASSERTION{{ end }}
 FileCopyrightText: {{ if .CopyrightText }}<text>{{ .CopyrightText }}
 </text>{{ else }}NOASSERTION{{ end }}
 
@@ -113,6 +113,18 @@ func (f *File) ReadChecksums(filePath string) error {
 
 // Render renders the document fragment of a file
 func (f *File) Render() (docFragment string, err error) {
+	// If we have not yet checksummed the file, do it now:
+	if f.Checksum == nil || len(f.Checksum) == 0 {
+		if f.SourceFile != "" {
+			if err := f.ReadSourceFile(f.SourceFile); err != nil {
+				return "", errors.Wrap(err, "checksumming file")
+			}
+		} else {
+			logrus.Warnf(
+				"File %s does not have checksums, SBOM will not be SPDX compliant", f.ID,
+			)
+		}
+	}
 	var buf bytes.Buffer
 	tmpl, err := template.New("file").Parse(fileTemplate)
 	if err != nil {