diff --git a/cmd/bom/cmd/generate.go b/cmd/bom/cmd/generate.go index 9ab092d05d5..a29e45d28d7 100644 --- a/cmd/bom/cmd/generate.go +++ b/cmd/bom/cmd/generate.go @@ -19,11 +19,13 @@ package cmd import ( "fmt" "net/url" + "os" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "k8s.io/release/pkg/spdx" + "sigs.k8s.io/release-utils/util" ) var genOpts = &generateOptions{} @@ -46,28 +48,57 @@ of analyzers designed to add more sense to common base images. SilenceErrors: true, PersistentPreRunE: initLogging, RunE: func(cmd *cobra.Command, args []string) error { + for i, arg := range args { + if util.Exists(arg) { + file, err := os.Open(arg) + if err != nil { + return errors.Wrapf(err, "checking argument %d", i) + } + fileInfo, err := file.Stat() + if err != nil { + return errors.Wrapf(err, "calling stat on argument %d", i) + } + if fileInfo.IsDir() { + genOpts.directories = append(genOpts.directories, arg) + } + } + } return generateBOM(genOpts) }, } type generateOptions struct { - analyze bool - namespace string - outputFile string - images []string - tarballs []string - files []string + analyze bool + noGitignore bool + noGoModules bool + namespace string + outputFile string + images []string + tarballs []string + files []string + directories []string + ignorePatterns []string } // Validate verify options consistency func (opts *generateOptions) Validate() error { - if len(opts.images) == 0 && len(opts.files) == 0 && len(opts.tarballs) == 0 { + if len(opts.images) == 0 && len(opts.files) == 0 && len(opts.tarballs) == 0 && len(opts.directories) == 0 { return errors.New("to generate a SPDX BOM you have to provide at least one image or file") } // A namespace URL is required if opts.namespace == "" { - return errors.New("A namespace (URL) must be defined to have a compliant SPDX BOM") + msg := "\nYou did not specify a namespace for your document. This is an error.\n" + msg += "To produce a valid SPDX SBOM, the document has to have an URI as its\n" + msg += "namespace.\n\nYou can use http://example.com/ for now if you are testing but your\n" + msg += "final document must have the namespace URI pointing to the location where\n" + msg += "you SBOM will be referenced in the future.\n\n" + msg += "For more details, check the SPDX documentation here:\n" + msg += "https://spdx.github.io/spdx-spec/2-document-creation-information/#25-spdx-document-namespace\n\n" + msg += "Hint: --namespace is your friend here\n\n" + logrus.Info(msg) + + return errors.New("A namespace URI must be defined to have a compliant SPDX BOM") } // CHeck namespace is a valid URL @@ -102,6 +133,35 @@ func init() { "list of docker archive tarballs to include in the manifest", ) + generateCmd.PersistentFlags().StringSliceVarP( + &genOpts.directories, + "dirs", + "d", + []string{}, + "list of directories to include in the manifest as packages", + ) + + generateCmd.PersistentFlags().StringSliceVar( + &genOpts.ignorePatterns, + "ignore", + []string{}, + "list of regexp patterns to ignore when scanning directories", + ) + + generateCmd.PersistentFlags().BoolVar( + &genOpts.noGitignore, + "no-gitignore", + false, + "don't use exclusions from .gitignore files", + ) + + generateCmd.PersistentFlags().BoolVar( + &genOpts.noGitignore, + "no-gomod", + false, + "don't perform go.mod analysis, sbom will not include data about go packages", + ) + generateCmd.PersistentFlags().StringVarP( &genOpts.namespace, "namespace", @@ -134,14 +194,22 @@ func generateBOM(opts *generateOptions) error { logrus.Info("Generating SPDX Bill of Materials") builder := spdx.NewDocBuilder() - doc, err := builder.Generate(&spdx.DocGenerateOptions{ - Tarballs: opts.tarballs, - Files: opts.files, - Images: opts.images, - OutputFile: opts.outputFile, - Namespace: "", - AnalyseLayers: opts.analyze, - }) + builderOpts := &spdx.DocGenerateOptions{ + Tarballs: opts.tarballs, + Files: opts.files, + Images: opts.images, + Directories: opts.directories, + OutputFile: opts.outputFile, + Namespace: opts.namespace, + AnalyseLayers: opts.analyze, + ProcessGoModules: !opts.noGoModules, + } + + // We only replace the ignore patterns one or more where defined + if len(opts.ignorePatterns) > 0 { + builderOpts.IgnorePatterns = opts.ignorePatterns + } + doc, err := builder.Generate(builderOpts) if err != nil { return errors.Wrap(err, "generating doc") } diff --git a/go.mod b/go.mod index f02d993e526..165b250f6c7 100644 --- a/go.mod +++ b/go.mod @@ -33,8 +33,10 @@ require ( github.com/spiegel-im-spiegel/go-cvss v0.4.0 github.com/stretchr/testify v1.7.0 github.com/yuin/goldmark v1.3.7 + golang.org/x/mod v0.4.2 golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c + golang.org/x/tools v0.1.1 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 google.golang.org/api v0.46.0 google.golang.org/genproto v0.0.0-20210517163617-5e0236093d7a diff --git a/pkg/license/catalog.go b/pkg/license/catalog.go index 1f9d690490b..6819059c281 100644 --- a/pkg/license/catalog.go +++ b/pkg/license/catalog.go @@ -33,7 +33,7 @@ type CatalogOptions struct { // are in the temporary OS directory and are created if the do not exist var DefaultCatalogOpts = &CatalogOptions{} -// NewSPDXWithOptions returns a SPDX object with the specified options +// NewCatalogWithOptions returns a SPDX object with the specified options func NewCatalogWithOptions(opts *CatalogOptions) (catalog *Catalog, err error) { // Create the license downloader doptions := DefaultDownloaderOpts @@ -76,7 +76,7 @@ type Catalog struct { // WriteLicensesAsText writes the SPDX license collection to text files func (catalog *Catalog) WriteLicensesAsText(targetDir string) error { - logrus.Info("Writing SPDX licenses to " + targetDir) + logrus.Infof("Writing %d SPDX licenses to %s", len(catalog.List.Licenses), targetDir) if catalog.List.Licenses == nil { return errors.New("unable to write licenses, they have not been loaded yet") } diff --git a/pkg/license/download.go b/pkg/license/download.go index f84a71001f0..26ad2614f54 100644 --- a/pkg/license/download.go +++ b/pkg/license/download.go @@ -147,25 +147,28 @@ func (ddi *DefaultDownloaderImpl) GetLicenses() (licenses *List, err error) { // Create a new Throttler that will get `parallelDownloads` urls at a time t := throttler.New(ddi.Options.parallelDownloads, len(licenseList.LicenseData)) for _, l := range licenseList.LicenseData { - licURL := l.Reference + licURL := l.DetailsURL // If the license URLs have a local reference if strings.HasPrefix(licURL, "./") { licURL = LicenseDataURL + strings.TrimPrefix(licURL, "./") } // Launch a goroutine to fetch the URL. go func(url string) { - var err error + var lic *License defer t.Done(err) - l, err := ddi.getLicenseFromURL(url) + lic, err = ddi.getLicenseFromURL(url) if err != nil { + logrus.Error(err) return } logrus.Debugf("Got license: %s from %s", l.LicenseID, url) - licenseList.Add(l) + licenseList.Add(lic) }(licURL) t.Throttle() } + logrus.Infof("Downloaded %d licenses", len(licenseList.Licenses)) + // If the throttler collected errors, return those if t.Err() != nil { return nil, t.Err() diff --git a/pkg/license/implementation.go b/pkg/license/implementation.go index 3915528b5f5..cb5c8ac3532 100644 --- a/pkg/license/implementation.go +++ b/pkg/license/implementation.go @@ -167,6 +167,8 @@ func (d *ReaderDefaultImpl) Initialize(opts *ReaderOptions) error { return errors.Wrap(err, "loading licenses") } + logrus.Infof("Writing license data to %s", opts.CachePath()) + // Write the licenses to disk as th classifier will need them if err := catalog.WriteLicensesAsText(opts.LicensesPath()); err != nil { return errors.Wrap(err, "writing license data to disk") @@ -174,7 +176,7 @@ func (d *ReaderDefaultImpl) Initialize(opts *ReaderOptions) error { // Create the implementation's classifier d.lc = licenseclassifier.NewClassifier(opts.ConfidenceThreshold) - return errors.Wrap(d.lc.LoadLicenses(opts.LicensesPath()), "loading licenses at init") + return errors.Wrap(d.lc.LoadLicenses(opts.CachePath()), "loading licenses at init") } // Classifier returns the license classifier diff --git a/pkg/license/license.go b/pkg/license/license.go index 6f6793ad0be..29fbbb967b8 100644 --- a/pkg/license/license.go +++ b/pkg/license/license.go @@ -28,7 +28,6 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" - "sigs.k8s.io/release-utils/util" ) const ( @@ -120,32 +119,6 @@ func (ro *ReaderOptions) Validate() error { return errors.Wrap(err, "checking working directory") } - // Check the cache directory - if !util.Exists(ro.CacheDir) { - if ro.CacheDir == "" { - if err := os.MkdirAll( - filepath.Join(ro.WorkDir, defaultCacheSubDir), os.FileMode(0o755), - ); err != nil { - return errors.Wrap(err, "creating cache directory") - } - } else { - return errors.New("specified cache directory does not exist") - } - } - - // Check the licenses directory - if !util.Exists(ro.LicenseDir) { - if ro.LicenseDir == "" { - if err := os.MkdirAll( - filepath.Join(ro.WorkDir, defaultLicenseSubDir), os.FileMode(0o755), - ); err != nil { - return errors.Wrap(err, "creating licenses directory") - } - } else { - return errors.New("specified licenses directory does not exist") - } - } - // TODO check dirs return nil } @@ -159,7 +132,7 @@ func (ro *ReaderOptions) CachePath() string { return filepath.Join(ro.WorkDir, defaultCacheSubDir) } -// LicensesPath return the full path to the downloads cache +// LicensesPath return the full path the dir where the licenses are func (ro *ReaderOptions) LicensesPath() string { if ro.LicenseDir != "" { return ro.LicenseDir diff --git a/pkg/spdx/builder.go b/pkg/spdx/builder.go index ee2f62a6619..f397d84b20e 100644 --- a/pkg/spdx/builder.go +++ b/pkg/spdx/builder.go @@ -61,18 +61,22 @@ func (db *DocBuilder) Generate(genopts *DocGenerateOptions) (*Document, error) { } type DocGenerateOptions struct { - Tarballs []string // A slice of tar paths - Files []string // A slice of naked files to include in the bom - Images []string // A slice of docker images - OutputFile string // Output location - Namespace string // Namespace for the document (a unique URI) - AnalyseLayers bool // A flag that controls if deep layer analysis should be performed + AnalyseLayers bool // A flag that controls if deep layer analysis should be performed + NoGitignore bool // Do not read exclusions from gitignore file + ProcessGoModules bool // Analyze go.mod to include data about packages + OutputFile string // Output location + Namespace string // Namespace for the document (a unique URI) + Tarballs []string // A slice of tar paths + Files []string // A slice of naked files to include in the bom + Images []string // A slice of docker images + Directories []string // A slice of directories to convert into packages + IgnorePatterns []string // a slice of regexp patterns to ignore when scanning dirs } func (o *DocGenerateOptions) Validate() error { - if len(o.Tarballs) == 0 && len(o.Files) == 0 && len(o.Images) == 0 { + if len(o.Tarballs) == 0 && len(o.Files) == 0 && len(o.Images) == 0 && len(o.Directories) == 0 { return errors.New( - "To build a document at least an image, tarball or a file has to be specified", + "To build a document at least an image, tarball, directory or a file has to be specified", ) } return nil @@ -104,7 +108,11 @@ func (builder defaultDocBuilderImpl) GenerateDoc( } spdx := NewSPDX() - spdx.options.AnalyzeLayers = genopts.AnalyseLayers + if len(genopts.IgnorePatterns) > 0 { + spdx.Options().IgnorePatterns = genopts.IgnorePatterns + } + spdx.Options().AnalyzeLayers = genopts.AnalyseLayers + spdx.Options().ProcessGoModules = genopts.ProcessGoModules if !util.Exists(opts.WorkDir) { if err := os.MkdirAll(opts.WorkDir, os.FileMode(0o755)); err != nil { @@ -123,8 +131,19 @@ func (builder defaultDocBuilderImpl) GenerateDoc( doc.Namespace = genopts.Namespace if genopts.Namespace == "" { - logrus.Warn("Document namespace is empty, a mock URI will be supplied but the doc will not be valid") - doc.Namespace = "http://example.com/" + return nil, errors.New("unable to generate doc, namespace URI is not defined") + } + + for _, i := range genopts.Directories { + logrus.Infof("Processing directory %s", i) + pkg, err := spdx.PackageFromDirectory(i) + if err != nil { + return nil, errors.Wrap(err, "generating package from directory") + } + + if err := doc.AddPackage(pkg); err != nil { + return nil, errors.Wrap(err, "adding directory package to document") + } } for _, i := range genopts.Images { diff --git a/pkg/spdx/document.go b/pkg/spdx/document.go index 2fc43dbf49c..4c3fdfeebe5 100644 --- a/pkg/spdx/document.go +++ b/pkg/spdx/document.go @@ -133,7 +133,7 @@ func (d *Document) Render() (doc string, err error) { } if d.Name == "" { - d.Name = "BOM-SPDX-" + uuid.New().String() + d.Name = "SBOM-SPDX-" + uuid.New().String() logrus.Warnf("Document has no name defined, automatically set to " + d.Name) } diff --git a/pkg/spdx/file.go b/pkg/spdx/file.go index 7b547b2a7f9..2d17710f817 100644 --- a/pkg/spdx/file.go +++ b/pkg/spdx/file.go @@ -41,7 +41,7 @@ var fileTemplate = `{{ if .Name }}FileName: {{ .Name }} {{- end -}} {{- end -}} LicenseConcluded: {{ if .LicenseConcluded }}{{ .LicenseConcluded }}{{ else }}NOASSERTION{{ end }} -LicenseInfoInFile: {{ if .LicenseInfoInFile }}LicenseInfoInFile: {{ .LicenseInfoInFile }}{{ else }}NOASSERTION{{ end }} +LicenseInfoInFile: {{ if .LicenseInfoInFile }}{{ .LicenseInfoInFile }}{{ else }}NOASSERTION{{ end }} FileCopyrightText: {{ if .CopyrightText }}{{ .CopyrightText }} {{ else }}NOASSERTION{{ end }} @@ -113,6 +113,18 @@ func (f *File) ReadChecksums(filePath string) error { // Render renders the document fragment of a file func (f *File) Render() (docFragment string, err error) { + // If we have not yet checksummed the file, do it now: + if f.Checksum == nil || len(f.Checksum) == 0 { + if f.SourceFile != "" { + if err := f.ReadSourceFile(f.SourceFile); err != nil { + return "", errors.Wrap(err, "checksumming file") + } + } else { + logrus.Warnf( + "File %s does not have checksums, SBOM will not be SPDX compliant", f.ID, + ) + } + } var buf bytes.Buffer tmpl, err := template.New("file").Parse(fileTemplate) if err != nil { diff --git a/pkg/spdx/gomod.go b/pkg/spdx/gomod.go new file mode 100644 index 00000000000..c47b0024829 --- /dev/null +++ b/pkg/spdx/gomod.go @@ -0,0 +1,297 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package spdx + +import ( + "os" + "path/filepath" + "regexp" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/mod/modfile" + "golang.org/x/tools/go/vcs" + "k8s.io/release/pkg/license" + "sigs.k8s.io/release-utils/util" +) + +const ( + downloadDir = spdxTempDir + "/gomod-scanner" + GoModFileName = "go.mod" +) + +// NewGoModule returns a new go module from the specified path +func NewGoModuleFromPath(path string) (*GoModule, error) { + mod := NewGoModule() + mod.opts.Path = path + if err := mod.Open(); err != nil { + return nil, errors.Wrap(err, "opening new module path") + } + return mod, nil +} + +func NewGoModule() *GoModule { + return &GoModule{ + opts: &GoModuleOptions{}, + impl: &GoModDefaultImpl{}, + } +} + +// GoModule abstracts the go module data of a project +type GoModule struct { + impl GoModImplementation + GoMod *modfile.File + opts *GoModuleOptions // Options + Packages []*GoPackage // maps of package download locations +} + +type GoModuleOptions struct { + Path string // Path to the dir where go.mod resides +} + +// GoPackage basic pkg data we need +type GoPackage struct { + ImportPath string + Revision string + LocalDir string + LicenseID string +} + +// SPDXPackage builds a spdx package from the go package data +func (pkg *GoPackage) ToSPDXPackage() (*Package, error) { + repo, err := vcs.RepoRootForImportPath(pkg.ImportPath, true) + if err != nil { + return nil, errors.Wrap(err, "building repository from package import path") + } + spdxPackage := NewPackage() + spdxPackage.Name = pkg.ImportPath + spdxPackage.DownloadLocation = repo.Repo + spdxPackage.LicenseConcluded = pkg.LicenseID + spdxPackage.Version = pkg.Revision + return spdxPackage, nil +} + +type GoModImplementation interface { + OpenModule(*GoModuleOptions) (*modfile.File, error) + BuildPackageList(*modfile.File) ([]*GoPackage, error) + DownloadPackage(*GoPackage, *GoModuleOptions, bool) error + RemoveDownloads([]*GoPackage) error + LicenseReader() (*license.Reader, error) + ScanPackageLicense(*GoPackage, *license.Reader, *GoModuleOptions) error +} + +// Initializes a go module from the specified path +func (mod *GoModule) Open() error { + gomod, err := mod.impl.OpenModule(mod.opts) + if err != nil { + return errors.Wrap(err, "opening module") + } + mod.GoMod = gomod + + // Build the package list + pkgs, err := mod.impl.BuildPackageList(mod.GoMod) + if err != nil { + return errors.Wrap(err, "building module package list") + } + mod.Packages = pkgs + return nil +} + +// RemoveDownloads cleans all downloads +func (mod *GoModule) RemoveDownloads() error { + return mod.impl.RemoveDownloads(mod.Packages) +} + +// DownloadPackages downloads all the module's packages to the local disk +func (mod *GoModule) DownloadPackages() error { + logrus.Infof("Downloading source code for %d packages", len(mod.Packages)) + if mod.Packages == nil { + return errors.New("Unable to download packages, package list is nil") + } + + for _, pkg := range mod.Packages { + if err := mod.impl.DownloadPackage(pkg, mod.opts, true); err != nil { + return err + } + } + return nil +} + +// ScanLicenses scans the licenses and populats the fields +func (mod *GoModule) ScanLicenses() error { + if mod.Packages == nil { + return errors.New("Unable to scan lincese files, package list is nil") + } + + reader, err := mod.impl.LicenseReader() + if err != nil { + return errors.Wrap(err, "creating license scanner") + } + + // Do a quick re-check for missing downloads + // todo: paralelize this. urgently. + for _, pkg := range mod.Packages { + // Call download with no force in case local data is missing + if err := mod.impl.DownloadPackage(pkg, mod.opts, false); err != nil { + // If we're unable to download the module we dont treat it as + // fatal, package will remain without license info but we go + // on scanning the rest of the packages. + logrus.Error(err) + continue + } + + if err := mod.impl.ScanPackageLicense(pkg, reader, mod.opts); err != nil { + return errors.Wrapf(err, "scanning package %s for licensing info", pkg.ImportPath) + } + } + + return nil +} + +type GoModDefaultImpl struct { + licenseReader *license.Reader +} + +// OpenModule opens the go,mod file for the module and parses it +func (di *GoModDefaultImpl) OpenModule(opts *GoModuleOptions) (*modfile.File, error) { + modData, err := os.ReadFile(filepath.Join(opts.Path, GoModFileName)) + if err != nil { + return nil, errors.Wrap(err, "reading module's go.mod file") + } + gomod, err := modfile.ParseLax("file", modData, nil) + if err != nil { + return nil, errors.Wrap(err, "reading go.mod") + } + logrus.Infof( + "Parsed go.mod file for %s, found %d packages", + gomod.Module.Mod.Path, + len(gomod.Require), + ) + return gomod, nil +} + +// BuildPackageList builds a slice of packages to assign to the module +func (di *GoModDefaultImpl) BuildPackageList(gomod *modfile.File) ([]*GoPackage, error) { + pkgs := []*GoPackage{} + for _, req := range gomod.Require { + pkgs = append(pkgs, &GoPackage{ + ImportPath: req.Mod.Path, + Revision: req.Mod.Version, + }) + } + return pkgs, nil +} + +// DownloadPackage takes a pkg, downloads it from its src and sets +// the download dir in the LocalDir field +func (di *GoModDefaultImpl) DownloadPackage(pkg *GoPackage, opts *GoModuleOptions, force bool) error { + logrus.Infof("Downloading package %s@%s", pkg.ImportPath, pkg.Revision) + repo, err := vcs.RepoRootForImportPath(pkg.ImportPath, true) + if err != nil { + return errors.Wrapf(err, "Fetching package %s from %s", pkg.ImportPath, repo.Repo) + } + + if pkg.LocalDir != "" && util.Exists(pkg.LocalDir) && !force { + logrus.Infof("Not downloading %s as it already has local data", pkg.ImportPath) + return nil + } + + if !util.Exists(filepath.Join(os.TempDir(), downloadDir)) { + if err := os.MkdirAll( + filepath.Join(os.TempDir(), downloadDir), os.FileMode(0o755), + ); err != nil { + return errors.Wrap(err, "creating parent tmpdir") + } + } + + // Create tempdir + tmpDir, err := os.MkdirTemp(filepath.Join(os.TempDir(), downloadDir), "package-download-") + if err != nil { + return errors.Wrap(err, "creating temporary dir") + } + // Create a clone of the module repo at the revision + rev := pkg.Revision + m := regexp.MustCompile(`v\d+\.\d+\.\d+-[0-9.]+-([a-f0-9]+)`).FindStringSubmatch(pkg.Revision) + if len(m) > 1 { + rev = m[1] + logrus.Infof("Using commit %s as revision for download", rev) + } + if err := repo.VCS.CreateAtRev(tmpDir, repo.Repo, rev); err != nil { + return errors.Wrapf(err, "creating local clone of %s", repo.Repo) + } + + logrus.Infof("Go Package %s (rev %s) downloaded to %s", pkg.ImportPath, pkg.Revision, tmpDir) + pkg.LocalDir = tmpDir + return nil +} + +// RemoveDownloads takes a list of packages and remove its downloads +func (di *GoModDefaultImpl) RemoveDownloads(packageList []*GoPackage) error { + for _, pkg := range packageList { + if pkg.ImportPath != "" && util.Exists(pkg.LocalDir) { + if err := os.RemoveAll(pkg.ImportPath); err != nil { + return errors.Wrap(err, "removing package data") + } + } + } + return nil +} + +// LicenseReader returns a license reader +func (di *GoModDefaultImpl) LicenseReader() (*license.Reader, error) { + if di.licenseReader == nil { + opts := license.DefaultReaderOptions + opts.CacheDir = filepath.Join(os.TempDir(), spdxLicenseDlCache) + opts.LicenseDir = filepath.Join(os.TempDir(), spdxLicenseData) + if !util.Exists(opts.CacheDir) { + if err := os.MkdirAll(opts.CacheDir, os.FileMode(0o755)); err != nil { + return nil, errors.Wrap(err, "creating dir") + } + } + reader, err := license.NewReaderWithOptions(opts) + if err != nil { + return nil, errors.Wrap(err, "creating reader") + } + + di.licenseReader = reader + } + return di.licenseReader, nil +} + +// ScanPackageLicense scans a package for licensing info +func (di *GoModDefaultImpl) ScanPackageLicense( + pkg *GoPackage, reader *license.Reader, opts *GoModuleOptions) error { + licenselist, _, err := reader.ReadLicenses(pkg.LocalDir) + if err != nil { + return errors.Wrapf(err, "scanning package %s for licensing information", pkg.ImportPath) + } + + if len(licenselist) > 1 { + logrus.Warnf("Package %s has %d licenses, picking the first", pkg.ImportPath, len(licenselist)) + } + + if len(licenselist) != 0 { + logrus.Infof( + "Package %s license is %s", pkg.ImportPath, + licenselist[0].License.LicenseID, + ) + pkg.LicenseID = licenselist[0].License.LicenseID + } else { + logrus.Infof("Could not find licensing information for package %s", pkg.ImportPath) + } + return nil +} diff --git a/pkg/spdx/imageanalyzer.go b/pkg/spdx/imageanalyzer.go index b00316bd253..ed4870067b4 100644 --- a/pkg/spdx/imageanalyzer.go +++ b/pkg/spdx/imageanalyzer.go @@ -36,7 +36,7 @@ type ImageAnalyzer struct { func NewImageAnalyzer() *ImageAnalyzer { // Default options for all analyzers opts := &ContainerLayerAnalyzerOptions{ - LicenseCacheDir: filepath.Join(os.TempDir(), spdxLicenseCacheDir), + LicenseCacheDir: filepath.Join(os.TempDir(), spdxLicenseData), } // Create the instance with all the drivers we have so far diff --git a/pkg/spdx/imageanalyzer_gorunner.go b/pkg/spdx/imageanalyzer_gorunner.go index 93808414b27..bd838a8a257 100644 --- a/pkg/spdx/imageanalyzer_gorunner.go +++ b/pkg/spdx/imageanalyzer_gorunner.go @@ -103,7 +103,7 @@ func (h *goRunnerHandler) licenseReader(o *ContainerLayerAnalyzerOptions) (*lice logrus.Info("Initializing licence reader with default options") // We use a default license cache opts := license.DefaultReaderOptions - ldir := filepath.Join(os.TempDir(), "spdx-license-reader-licenses") + ldir := filepath.Join(os.TempDir(), spdxLicenseDlCache) // ... unless overridden by the options if o.LicenseCacheDir != "" { ldir = o.LicenseCacheDir @@ -116,6 +116,7 @@ func (h *goRunnerHandler) licenseReader(o *ContainerLayerAnalyzerOptions) (*lice } } opts.CacheDir = ldir + opts.LicenseDir = filepath.Join(os.TempDir(), spdxLicenseData) // Create the new reader reader, err := license.NewReaderWithOptions(opts) if err != nil { diff --git a/pkg/spdx/implementation.go b/pkg/spdx/implementation.go index e19bc6a353e..0edcc58720b 100644 --- a/pkg/spdx/implementation.go +++ b/pkg/spdx/implementation.go @@ -20,20 +20,24 @@ package spdx import ( "archive/tar" + "bufio" "crypto/sha1" "encoding/json" "fmt" "io" + "io/fs" "os" "path/filepath" "strings" + gitignore "github.com/go-git/go-git/v5/plumbing/format/gitignore" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "k8s.io/release/pkg/license" "sigs.k8s.io/release-utils/util" ) @@ -44,6 +48,12 @@ type spdxImplementation interface { ReadArchiveManifest(string) (*ArchiveManifest, error) PullImagesToArchive(string, string) error PackageFromLayerTarBall(string, *TarballOptions) (*Package, error) + GetDirectoryTree(string) ([]string, error) + IgnorePatterns(string, []string, bool) ([]gitignore.Pattern, error) + ApplyIgnorePatterns([]string, []gitignore.Pattern) []string + GetGoDependencies(string, bool) ([]*Package, error) + GetDirectoryLicense(*license.Reader, string, *Options) (*license.License, error) + LicenseReader(*Options) (*license.Reader, error) } type spdxDefaultImplementation struct{} @@ -177,3 +187,149 @@ func (di *spdxDefaultImplementation) PackageFromLayerTarBall( return pkg, nil } + +// GetDirectoryTree traverses a directory and return a slice of strings with all files +func (di *spdxDefaultImplementation) GetDirectoryTree(dirPath string) ([]string, error) { + fileList := []string{} + + if err := fs.WalkDir(os.DirFS(dirPath), ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + + if d.Type() == os.ModeSymlink { + return nil + } + + fileList = append(fileList, path) + return nil + }); err != nil { + return nil, errors.Wrap(err, "buiding directory tree") + } + return fileList, nil +} + +// IgnorePatterns return a list of gitignore patterns +func (di *spdxDefaultImplementation) IgnorePatterns( + dirPath string, extraPatterns []string, skipGitIgnore bool, +) ([]gitignore.Pattern, error) { + patterns := []gitignore.Pattern{} + for _, s := range extraPatterns { + patterns = append(patterns, gitignore.ParsePattern(s, nil)) + } + + if skipGitIgnore { + logrus.Debug("Not using patterns in .gitignore") + return patterns, nil + } + + if util.Exists(filepath.Join(dirPath, gitIgnoreFile)) { + f, err := os.Open(filepath.Join(dirPath, gitIgnoreFile)) + if err != nil { + return nil, errors.Wrap(err, "opening gitignore file") + } + defer f.Close() + + // When using .gitignore files, we alwas add the .git directory + // to match git's behavior + patterns = append(patterns, gitignore.ParsePattern(".git/", nil)) + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + s := scanner.Text() + if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 { + logrus.Debugf("Loaded .gitignore pattern: >>%s<<", s) + patterns = append(patterns, gitignore.ParsePattern(s, nil)) + } + } + } + + logrus.Debugf( + "Loaded %d patterns from .gitignore (+ %d extra) at root of directory", len(patterns), len(extraPatterns), + ) + return patterns, nil +} + +// ApplyIgnorePatterns applies the gitignore patterns to a list of files, removing matched +func (di *spdxDefaultImplementation) ApplyIgnorePatterns( + fileList []string, patterns []gitignore.Pattern, +) (filteredList []string) { + // We will return a new file list + filteredList = []string{} + + // Build the new gitignore matcher + matcher := gitignore.NewMatcher(patterns) + + // Cycle all files, removing those matched: + for _, file := range fileList { + if matcher.Match(strings.Split(file, string(filepath.Separator)), false) { + logrus.Debugf("File ignored by .gitignore: %s", file) + } else { + filteredList = append(filteredList, file) + } + } + return filteredList +} + +// GetGoDependencies +func (di *spdxDefaultImplementation) GetGoDependencies(path string, scanLicenses bool) ([]*Package, error) { + // Open the directory as a go module: + mod, err := NewGoModuleFromPath(path) + if err != nil { + return nil, errors.Wrap(err, "creating a mod from the specified path") + } + defer mod.GoMod.Cleanup() + + if scanLicenses { + if err := mod.ScanLicenses(); err != nil { + return nil, errors.Wrap(err, "scanning go module licenses") + } + } + + spdxPackages := []*Package{} + for _, goPkg := range mod.Packages { + spdxPkg, err := goPkg.ToSPDXPackage() + if err != nil { + return nil, errors.Wrap(err, "converting go module to spdx package") + } + spdxPackages = append(spdxPackages, spdxPkg) + } + + return spdxPackages, nil +} + +func (di *spdxDefaultImplementation) LicenseReader(spdxOpts *Options) (*license.Reader, error) { + opts := license.DefaultReaderOptions + opts.CacheDir = spdxOpts.LicenseCacheDir + // Create the new reader + reader, err := license.NewReaderWithOptions(opts) + if err != nil { + return nil, errors.Wrap(err, "creating reusable license reader") + } + return reader, nil +} + +// GetDirectoryLicense takes a path and scans +// the files in it to determine licensins information +func (di *spdxDefaultImplementation) GetDirectoryLicense( + reader *license.Reader, path string, spdxOpts *Options, +) (*license.License, error) { + // Perhaps here we should take into account thre results + licenselist, _, err := reader.ReadLicenses(path) + if err != nil { + return nil, errors.Wrap(err, "scanning directory for licensing information") + } + if len(licenselist) == 0 { + logrus.Warn("No license info could be determined, SPDX SBOM will not be valid") + return nil, nil + } + logrus.Infof("Determined license %s for directory %s", licenselist[0].License.LicenseID, path) + + if len(licenselist) > 1 { + logrus.Warnf("Found %d licenses in directory, picking the first", len(licenselist)) + } + return licenselist[0].License, nil +} diff --git a/pkg/spdx/package.go b/pkg/spdx/package.go index e5768ed4b9c..61be87ce759 100644 --- a/pkg/spdx/package.go +++ b/pkg/spdx/package.go @@ -50,30 +50,32 @@ FilesAnalyzed: {{ .FilesAnalyzed }} PackageLicenseConcluded: {{ if .LicenseConcluded }}{{ .LicenseConcluded }}{{ else }}NOASSERTION{{ end }} {{ if .FileName }}PackageFileName: {{ .FileName }} {{ end -}} -{{ if .LicenseInfoFromFiles }}PackageLicenseInfoFromFiles: {{ .LicenseInfoFromFiles }} +{{ if .LicenseInfoFromFiles }}{{- range $key, $value := .LicenseInfoFromFiles -}}PackageLicenseInfoFromFiles: {{ $value }} +{{ end -}} {{ end -}} {{ if .Version }}PackageVersion: {{ .Version }} {{ end -}} PackageLicenseDeclared: {{ if .LicenseDeclared }}{{ .LicenseDeclared }}{{ else }}NOASSERTION{{ end }} PackageCopyrightText: {{ if .CopyrightText }}{{ .CopyrightText }} {{ else }}NOASSERTION{{ end }} + ` // Package groups a set of files type Package struct { - FilesAnalyzed bool // true - Name string // hello-go-src - ID string // SPDXRef-Package-hello-go-src - DownloadLocation string // git@github.com:swinslow/spdx-examples.git#example6/content/src - VerificationCode string // 6486e016b01e9ec8a76998cefd0705144d869234 - LicenseConcluded string // LicenseID o NOASSERTION - LicenseInfoFromFiles string // GPL-3.0-or-later - LicenseDeclared string // GPL-3.0-or-later - LicenseComments string // record any relevant background information or analysis that went in to arriving at the Concluded License - CopyrightText string // string NOASSERTION - Version string // Package version - FileName string // Name of the package - SourceFile string // Source file for the package (taball for images, rpm, deb, etc) + FilesAnalyzed bool // true + Name string // hello-go-src + ID string // SPDXRef-Package-hello-go-src + DownloadLocation string // git@github.com:swinslow/spdx-examples.git#example6/content/src + VerificationCode string // 6486e016b01e9ec8a76998cefd0705144d869234 + LicenseConcluded string // LicenseID o NOASSERTION + LicenseInfoFromFiles []string // GPL-3.0-or-later + LicenseDeclared string // GPL-3.0-or-later + LicenseComments string // record any relevant background information or analysis that went in to arriving at the Concluded License + CopyrightText string // string NOASSERTION + Version string // Package version + FileName string // Name of the package + SourceFile string // Source file for the package (taball for images, rpm, deb, etc) // Supplier: the actual distribution source for the package/directory Supplier struct { @@ -87,9 +89,10 @@ type Package struct { Organization string // organization name and optional () } // Subpackages contained - Packages map[string]*Package // Sub packages conatined in this pkg - Files map[string]*File // List of files - Checksum map[string]string // Checksum of the package + Packages map[string]*Package // Sub packages conatined in this pkg + Files map[string]*File // List of files + Checksum map[string]string // Checksum of the package + Dependencies map[string]*Package // Packages marked as dependencies options *PackageOptions // Options } @@ -144,7 +147,7 @@ func (p *Package) AddFile(file *File) error { return errors.New("unable to generate file ID, filename not set") } if p.Name == "" { - return errors.New("unable to generate file ID, filename not set") + return errors.New("unable to generate file ID, package not set") } h := sha1.New() if _, err := h.Write([]byte(p.Name + ":" + file.Name)); err != nil { @@ -156,14 +159,13 @@ func (p *Package) AddFile(file *File) error { return nil } -// AddPackage adds a new subpackage to a package -func (p *Package) AddPackage(pkg *Package) error { - if p.Packages == nil { - p.Packages = map[string]*Package{} - } +// preProcessSubPackage performs a basic check on a package +// to ensure it can be added as a subpackage, trying to infer +// missing data when possible +func (p *Package) preProcessSubPackage(pkg *Package) error { if pkg.ID == "" { // If we so not have an ID but have a name generate it fro there - reg := regexp.MustCompile("[^a-zA-Z0-9-]+") + reg := regexp.MustCompile(validNameCharsRe) id := reg.ReplaceAllString(pkg.Name, "") if id != "" { pkg.ID = "SPDXRef-Package-" + id @@ -173,13 +175,44 @@ func (p *Package) AddPackage(pkg *Package) error { return errors.New("package name is needed to add a new package") } if _, ok := p.Packages[pkg.ID]; ok { - return errors.New("a package named " + pkg.ID + " already exists in the document") + return errors.New("a package named " + pkg.ID + " already exists as a subpackage") + } + + if _, ok := p.Dependencies[pkg.ID]; ok { + return errors.New("a package named " + pkg.ID + " already exists as a dependency") + } + + return nil +} + +// AddPackage adds a new subpackage to a package +func (p *Package) AddPackage(pkg *Package) error { + if p.Packages == nil { + p.Packages = map[string]*Package{} + } + + if err := p.preProcessSubPackage(pkg); err != nil { + return errors.Wrap(err, "performing subpackage preprocessing") } p.Packages[pkg.ID] = pkg return nil } +// AddDependency adds a new subpackage as a dependency +func (p *Package) AddDependency(pkg *Package) error { + if p.Dependencies == nil { + p.Dependencies = map[string]*Package{} + } + + if err := p.preProcessSubPackage(pkg); err != nil { + return errors.Wrap(err, "performing subpackage preprocessing") + } + + p.Dependencies[pkg.ID] = pkg + return nil +} + // Render renders the document fragment of the package func (p *Package) Render() (docFragment string, err error) { var buf bytes.Buffer @@ -189,6 +222,7 @@ func (p *Package) Render() (docFragment string, err error) { } // If files were analyzed, calculate the verification + filesTagList := map[string]*struct{}{} if p.FilesAnalyzed { if len(p.Files) == 0 { return docFragment, errors.New("unable to get package verification code, package has no files") @@ -202,6 +236,11 @@ func (p *Package) Render() (docFragment string, err error) { return docFragment, errors.New("unable to render package, files were analyzed but some do not have sha1 checksum") } shaList = append(shaList, f.Checksum["SHA1"]) + + // Collect the license tags + if f.LicenseInfoInFile != "" { + filesTagList[f.LicenseInfoInFile] = nil + } } sort.Strings(shaList) h := sha1.New() @@ -209,6 +248,16 @@ func (p *Package) Render() (docFragment string, err error) { return docFragment, errors.Wrap(err, "getting sha1 verification of files") } p.VerificationCode = fmt.Sprintf("%x", h.Sum(nil)) + + for tag := range filesTagList { + if tag != "NONE" && tag != "NOASSERTION" { + p.LicenseInfoFromFiles = append(p.LicenseInfoFromFiles, tag) + } + } + + if len(filesTagList) == 0 { + p.LicenseInfoFromFiles = append(p.LicenseInfoFromFiles, "NONE") + } } // Run the template to verify the output. @@ -239,5 +288,18 @@ func (p *Package) Render() (docFragment string, err error) { docFragment += fmt.Sprintf("Relationship: %s CONTAINS %s\n\n", p.ID, pkg.ID) } } + + // Print the contained dependencies + if p.Dependencies != nil { + for _, pkg := range p.Dependencies { + pkgDoc, err := pkg.Render() + if err != nil { + return "", errors.Wrap(err, "rendering pkg "+pkg.Name) + } + + docFragment += pkgDoc + docFragment += fmt.Sprintf("Relationship: %s DEPENDS_ON %s\n\n", p.ID, pkg.ID) + } + } return docFragment, nil } diff --git a/pkg/spdx/spdx.go b/pkg/spdx/spdx.go index 07ea678494f..08d730bc434 100644 --- a/pkg/spdx/spdx.go +++ b/pkg/spdx/spdx.go @@ -19,7 +19,9 @@ package spdx import ( "os" "path/filepath" + "regexp" + "github.com/google/uuid" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -29,7 +31,11 @@ import ( const ( defaultDocumentAuthor = "Kubernetes Release Managers (release-managers@kubernetes.io)" archiveManifestFilename = "manifest.json" - spdxLicenseCacheDir = "spdx/lic" + spdxTempDir = "spdx" + spdxLicenseData = spdxTempDir + "/licenses" + spdxLicenseDlCache = spdxTempDir + "/downloadCache" + gitIgnoreFile = ".gitignore" + validNameCharsRe = `[^a-zA-Z0-9-]+` ) type SPDX struct { @@ -49,8 +55,11 @@ func (spdx *SPDX) SetImplementation(impl spdxImplementation) { } type Options struct { - LicenseCacheDir string // Directory to cache SPDX license information - AnalyzeLayers bool + AnalyzeLayers bool + NoGitignore bool // Do not read exclusions from gitignore file + ProcessGoModules bool // If true, spdx will check if dirs are go modules and analize the packages + LicenseCacheDir string // Directory to cache SPDX license information + IgnorePatterns []string // Patterns to ignore when scanning file } func (spdx *SPDX) Options() *Options { @@ -58,8 +67,10 @@ func (spdx *SPDX) Options() *Options { } var defaultSPDXOptions = Options{ - LicenseCacheDir: filepath.Join(os.TempDir(), spdxLicenseCacheDir), - AnalyzeLayers: true, + LicenseCacheDir: filepath.Join(os.TempDir(), spdxLicenseDlCache), + AnalyzeLayers: true, + ProcessGoModules: true, + IgnorePatterns: []string{}, } type ArchiveManifest struct { @@ -73,6 +84,91 @@ type TarballOptions struct { ExtractDir string // Directory where the docker tar archive will be extracted } +// PackageFromDirectory indexes all files in a directory and builds a +// SPDX package describing its contents +func (spdx *SPDX) PackageFromDirectory(dirPath string) (pkg *Package, err error) { + fileList, err := spdx.impl.GetDirectoryTree(dirPath) + if err != nil { + return nil, errors.Wrap(err, "building directory tree") + } + reader, err := spdx.impl.LicenseReader(spdx.Options()) + if err != nil { + return nil, errors.Wrap(err, "creating license reader") + } + licenseTag := "" + lic, err := spdx.impl.GetDirectoryLicense(reader, dirPath, spdx.Options()) + if err != nil { + return nil, errors.Wrap(err, "scanning directory for licenses") + } + if lic == nil { + logrus.Warn(err, "Licenseclassifier could not find a license for directory") + } else { + licenseTag = lic.LicenseID + } + + // Build a list of patterns from those found in the .gitignore file and + // posssibly others passed in the options: + patterns, err := spdx.impl.IgnorePatterns( + dirPath, spdx.Options().IgnorePatterns, spdx.Options().NoGitignore, + ) + if err != nil { + return nil, errors.Wrap(err, "building ignore patterns list") + } + + // Apply the ignore patterns to the list of files + fileList = spdx.impl.ApplyIgnorePatterns(fileList, patterns) + + pkg = NewPackage() + pkg.FilesAnalyzed = true + pkg.Name = filepath.Base(dirPath) + // If the package file will result in an empty ID, generate one + reg := regexp.MustCompile(validNameCharsRe) + if reg.ReplaceAllString(pkg.Name, "") == "" { + pkg.Name = uuid.NewString() + } + pkg.LicenseConcluded = licenseTag + + // todo: parallellize + for _, path := range fileList { + f := NewFile() + f.Name = path + f.FileName = path + f.SourceFile = filepath.Join(dirPath, path) + lic, err := reader.LicenseFromFile(f.SourceFile) + if err != nil { + return nil, errors.Wrap(err, "scanning file for license") + } + if lic != nil { + f.LicenseInfoInFile = lic.LicenseID + } else { + f.LicenseInfoInFile = "NONE" + } + f.LicenseConcluded = licenseTag + if err := f.ReadSourceFile(filepath.Join(dirPath, path)); err != nil { + return nil, errors.Wrap(err, "checksumming file") + } + if err := pkg.AddFile(f); err != nil { + return nil, errors.Wrapf(err, "adding %s as file to the spdx package", path) + } + } + + if util.Exists(filepath.Join(dirPath, GoModFileName)) && spdx.Options().ProcessGoModules { + logrus.Info("Directory contains a go module. Scanning go packages") + deps, err := spdx.impl.GetGoDependencies(dirPath, true) + if err != nil { + return nil, errors.Wrap(err, "scanning go packages") + } + for _, dep := range deps { + if err := pkg.AddDependency(dep); err != nil { + return nil, errors.Wrap(err, "adding go dependency") + } + } + } + + // Add files into the package + return pkg, nil +} + // PackageFromImageTarball returns a SPDX package from a tarball func (spdx *SPDX) PackageFromImageTarball( tarPath string, opts *TarballOptions, diff --git a/pkg/spdx/spdxfakes/fake_spdx_implementation.go b/pkg/spdx/spdxfakes/fake_spdx_implementation.go index 7956d5f3b18..f94fb7bb944 100644 --- a/pkg/spdx/spdxfakes/fake_spdx_implementation.go +++ b/pkg/spdx/spdxfakes/fake_spdx_implementation.go @@ -1,29 +1,27 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - // Code generated by counterfeiter. DO NOT EDIT. package spdxfakes import ( "sync" + "github.com/go-git/go-git/v5/plumbing/format/gitignore" + "k8s.io/release/pkg/license" "k8s.io/release/pkg/spdx" ) type FakeSpdxImplementation struct { + ApplyIgnorePatternsStub func([]string, []gitignore.Pattern) []string + applyIgnorePatternsMutex sync.RWMutex + applyIgnorePatternsArgsForCall []struct { + arg1 []string + arg2 []gitignore.Pattern + } + applyIgnorePatternsReturns struct { + result1 []string + } + applyIgnorePatternsReturnsOnCall map[int]struct { + result1 []string + } ExtractTarballTmpStub func(string) (string, error) extractTarballTmpMutex sync.RWMutex extractTarballTmpArgsForCall []struct { @@ -37,6 +35,76 @@ type FakeSpdxImplementation struct { result1 string result2 error } + GetDirectoryLicenseStub func(*license.Reader, string, *spdx.Options) (*license.License, error) + getDirectoryLicenseMutex sync.RWMutex + getDirectoryLicenseArgsForCall []struct { + arg1 *license.Reader + arg2 string + arg3 *spdx.Options + } + getDirectoryLicenseReturns struct { + result1 *license.License + result2 error + } + getDirectoryLicenseReturnsOnCall map[int]struct { + result1 *license.License + result2 error + } + GetDirectoryTreeStub func(string) ([]string, error) + getDirectoryTreeMutex sync.RWMutex + getDirectoryTreeArgsForCall []struct { + arg1 string + } + getDirectoryTreeReturns struct { + result1 []string + result2 error + } + getDirectoryTreeReturnsOnCall map[int]struct { + result1 []string + result2 error + } + GetGoDependenciesStub func(string, bool) ([]*spdx.Package, error) + getGoDependenciesMutex sync.RWMutex + getGoDependenciesArgsForCall []struct { + arg1 string + arg2 bool + } + getGoDependenciesReturns struct { + result1 []*spdx.Package + result2 error + } + getGoDependenciesReturnsOnCall map[int]struct { + result1 []*spdx.Package + result2 error + } + IgnorePatternsStub func(string, []string, bool) ([]gitignore.Pattern, error) + ignorePatternsMutex sync.RWMutex + ignorePatternsArgsForCall []struct { + arg1 string + arg2 []string + arg3 bool + } + ignorePatternsReturns struct { + result1 []gitignore.Pattern + result2 error + } + ignorePatternsReturnsOnCall map[int]struct { + result1 []gitignore.Pattern + result2 error + } + LicenseReaderStub func(*spdx.Options) (*license.Reader, error) + licenseReaderMutex sync.RWMutex + licenseReaderArgsForCall []struct { + arg1 *spdx.Options + } + licenseReaderReturns struct { + result1 *license.Reader + result2 error + } + licenseReaderReturnsOnCall map[int]struct { + result1 *license.Reader + result2 error + } PackageFromLayerTarBallStub func(string, *spdx.TarballOptions) (*spdx.Package, error) packageFromLayerTarBallMutex sync.RWMutex packageFromLayerTarBallArgsForCall []struct { @@ -80,6 +148,78 @@ type FakeSpdxImplementation struct { invocationsMutex sync.RWMutex } +func (fake *FakeSpdxImplementation) ApplyIgnorePatterns(arg1 []string, arg2 []gitignore.Pattern) []string { + var arg1Copy []string + if arg1 != nil { + arg1Copy = make([]string, len(arg1)) + copy(arg1Copy, arg1) + } + var arg2Copy []gitignore.Pattern + if arg2 != nil { + arg2Copy = make([]gitignore.Pattern, len(arg2)) + copy(arg2Copy, arg2) + } + fake.applyIgnorePatternsMutex.Lock() + ret, specificReturn := fake.applyIgnorePatternsReturnsOnCall[len(fake.applyIgnorePatternsArgsForCall)] + fake.applyIgnorePatternsArgsForCall = append(fake.applyIgnorePatternsArgsForCall, struct { + arg1 []string + arg2 []gitignore.Pattern + }{arg1Copy, arg2Copy}) + stub := fake.ApplyIgnorePatternsStub + fakeReturns := fake.applyIgnorePatternsReturns + fake.recordInvocation("ApplyIgnorePatterns", []interface{}{arg1Copy, arg2Copy}) + fake.applyIgnorePatternsMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeSpdxImplementation) ApplyIgnorePatternsCallCount() int { + fake.applyIgnorePatternsMutex.RLock() + defer fake.applyIgnorePatternsMutex.RUnlock() + return len(fake.applyIgnorePatternsArgsForCall) +} + +func (fake *FakeSpdxImplementation) ApplyIgnorePatternsCalls(stub func([]string, []gitignore.Pattern) []string) { + fake.applyIgnorePatternsMutex.Lock() + defer fake.applyIgnorePatternsMutex.Unlock() + fake.ApplyIgnorePatternsStub = stub +} + +func (fake *FakeSpdxImplementation) ApplyIgnorePatternsArgsForCall(i int) ([]string, []gitignore.Pattern) { + fake.applyIgnorePatternsMutex.RLock() + defer fake.applyIgnorePatternsMutex.RUnlock() + argsForCall := fake.applyIgnorePatternsArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeSpdxImplementation) ApplyIgnorePatternsReturns(result1 []string) { + fake.applyIgnorePatternsMutex.Lock() + defer fake.applyIgnorePatternsMutex.Unlock() + fake.ApplyIgnorePatternsStub = nil + fake.applyIgnorePatternsReturns = struct { + result1 []string + }{result1} +} + +func (fake *FakeSpdxImplementation) ApplyIgnorePatternsReturnsOnCall(i int, result1 []string) { + fake.applyIgnorePatternsMutex.Lock() + defer fake.applyIgnorePatternsMutex.Unlock() + fake.ApplyIgnorePatternsStub = nil + if fake.applyIgnorePatternsReturnsOnCall == nil { + fake.applyIgnorePatternsReturnsOnCall = make(map[int]struct { + result1 []string + }) + } + fake.applyIgnorePatternsReturnsOnCall[i] = struct { + result1 []string + }{result1} +} + func (fake *FakeSpdxImplementation) ExtractTarballTmp(arg1 string) (string, error) { fake.extractTarballTmpMutex.Lock() ret, specificReturn := fake.extractTarballTmpReturnsOnCall[len(fake.extractTarballTmpArgsForCall)] @@ -144,6 +284,336 @@ func (fake *FakeSpdxImplementation) ExtractTarballTmpReturnsOnCall(i int, result }{result1, result2} } +func (fake *FakeSpdxImplementation) GetDirectoryLicense(arg1 *license.Reader, arg2 string, arg3 *spdx.Options) (*license.License, error) { + fake.getDirectoryLicenseMutex.Lock() + ret, specificReturn := fake.getDirectoryLicenseReturnsOnCall[len(fake.getDirectoryLicenseArgsForCall)] + fake.getDirectoryLicenseArgsForCall = append(fake.getDirectoryLicenseArgsForCall, struct { + arg1 *license.Reader + arg2 string + arg3 *spdx.Options + }{arg1, arg2, arg3}) + stub := fake.GetDirectoryLicenseStub + fakeReturns := fake.getDirectoryLicenseReturns + fake.recordInvocation("GetDirectoryLicense", []interface{}{arg1, arg2, arg3}) + fake.getDirectoryLicenseMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeSpdxImplementation) GetDirectoryLicenseCallCount() int { + fake.getDirectoryLicenseMutex.RLock() + defer fake.getDirectoryLicenseMutex.RUnlock() + return len(fake.getDirectoryLicenseArgsForCall) +} + +func (fake *FakeSpdxImplementation) GetDirectoryLicenseCalls(stub func(*license.Reader, string, *spdx.Options) (*license.License, error)) { + fake.getDirectoryLicenseMutex.Lock() + defer fake.getDirectoryLicenseMutex.Unlock() + fake.GetDirectoryLicenseStub = stub +} + +func (fake *FakeSpdxImplementation) GetDirectoryLicenseArgsForCall(i int) (*license.Reader, string, *spdx.Options) { + fake.getDirectoryLicenseMutex.RLock() + defer fake.getDirectoryLicenseMutex.RUnlock() + argsForCall := fake.getDirectoryLicenseArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeSpdxImplementation) GetDirectoryLicenseReturns(result1 *license.License, result2 error) { + fake.getDirectoryLicenseMutex.Lock() + defer fake.getDirectoryLicenseMutex.Unlock() + fake.GetDirectoryLicenseStub = nil + fake.getDirectoryLicenseReturns = struct { + result1 *license.License + result2 error + }{result1, result2} +} + +func (fake *FakeSpdxImplementation) GetDirectoryLicenseReturnsOnCall(i int, result1 *license.License, result2 error) { + fake.getDirectoryLicenseMutex.Lock() + defer fake.getDirectoryLicenseMutex.Unlock() + fake.GetDirectoryLicenseStub = nil + if fake.getDirectoryLicenseReturnsOnCall == nil { + fake.getDirectoryLicenseReturnsOnCall = make(map[int]struct { + result1 *license.License + result2 error + }) + } + fake.getDirectoryLicenseReturnsOnCall[i] = struct { + result1 *license.License + result2 error + }{result1, result2} +} + +func (fake *FakeSpdxImplementation) GetDirectoryTree(arg1 string) ([]string, error) { + fake.getDirectoryTreeMutex.Lock() + ret, specificReturn := fake.getDirectoryTreeReturnsOnCall[len(fake.getDirectoryTreeArgsForCall)] + fake.getDirectoryTreeArgsForCall = append(fake.getDirectoryTreeArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.GetDirectoryTreeStub + fakeReturns := fake.getDirectoryTreeReturns + fake.recordInvocation("GetDirectoryTree", []interface{}{arg1}) + fake.getDirectoryTreeMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeSpdxImplementation) GetDirectoryTreeCallCount() int { + fake.getDirectoryTreeMutex.RLock() + defer fake.getDirectoryTreeMutex.RUnlock() + return len(fake.getDirectoryTreeArgsForCall) +} + +func (fake *FakeSpdxImplementation) GetDirectoryTreeCalls(stub func(string) ([]string, error)) { + fake.getDirectoryTreeMutex.Lock() + defer fake.getDirectoryTreeMutex.Unlock() + fake.GetDirectoryTreeStub = stub +} + +func (fake *FakeSpdxImplementation) GetDirectoryTreeArgsForCall(i int) string { + fake.getDirectoryTreeMutex.RLock() + defer fake.getDirectoryTreeMutex.RUnlock() + argsForCall := fake.getDirectoryTreeArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeSpdxImplementation) GetDirectoryTreeReturns(result1 []string, result2 error) { + fake.getDirectoryTreeMutex.Lock() + defer fake.getDirectoryTreeMutex.Unlock() + fake.GetDirectoryTreeStub = nil + fake.getDirectoryTreeReturns = struct { + result1 []string + result2 error + }{result1, result2} +} + +func (fake *FakeSpdxImplementation) GetDirectoryTreeReturnsOnCall(i int, result1 []string, result2 error) { + fake.getDirectoryTreeMutex.Lock() + defer fake.getDirectoryTreeMutex.Unlock() + fake.GetDirectoryTreeStub = nil + if fake.getDirectoryTreeReturnsOnCall == nil { + fake.getDirectoryTreeReturnsOnCall = make(map[int]struct { + result1 []string + result2 error + }) + } + fake.getDirectoryTreeReturnsOnCall[i] = struct { + result1 []string + result2 error + }{result1, result2} +} + +func (fake *FakeSpdxImplementation) GetGoDependencies(arg1 string, arg2 bool) ([]*spdx.Package, error) { + fake.getGoDependenciesMutex.Lock() + ret, specificReturn := fake.getGoDependenciesReturnsOnCall[len(fake.getGoDependenciesArgsForCall)] + fake.getGoDependenciesArgsForCall = append(fake.getGoDependenciesArgsForCall, struct { + arg1 string + arg2 bool + }{arg1, arg2}) + stub := fake.GetGoDependenciesStub + fakeReturns := fake.getGoDependenciesReturns + fake.recordInvocation("GetGoDependencies", []interface{}{arg1, arg2}) + fake.getGoDependenciesMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeSpdxImplementation) GetGoDependenciesCallCount() int { + fake.getGoDependenciesMutex.RLock() + defer fake.getGoDependenciesMutex.RUnlock() + return len(fake.getGoDependenciesArgsForCall) +} + +func (fake *FakeSpdxImplementation) GetGoDependenciesCalls(stub func(string, bool) ([]*spdx.Package, error)) { + fake.getGoDependenciesMutex.Lock() + defer fake.getGoDependenciesMutex.Unlock() + fake.GetGoDependenciesStub = stub +} + +func (fake *FakeSpdxImplementation) GetGoDependenciesArgsForCall(i int) (string, bool) { + fake.getGoDependenciesMutex.RLock() + defer fake.getGoDependenciesMutex.RUnlock() + argsForCall := fake.getGoDependenciesArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeSpdxImplementation) GetGoDependenciesReturns(result1 []*spdx.Package, result2 error) { + fake.getGoDependenciesMutex.Lock() + defer fake.getGoDependenciesMutex.Unlock() + fake.GetGoDependenciesStub = nil + fake.getGoDependenciesReturns = struct { + result1 []*spdx.Package + result2 error + }{result1, result2} +} + +func (fake *FakeSpdxImplementation) GetGoDependenciesReturnsOnCall(i int, result1 []*spdx.Package, result2 error) { + fake.getGoDependenciesMutex.Lock() + defer fake.getGoDependenciesMutex.Unlock() + fake.GetGoDependenciesStub = nil + if fake.getGoDependenciesReturnsOnCall == nil { + fake.getGoDependenciesReturnsOnCall = make(map[int]struct { + result1 []*spdx.Package + result2 error + }) + } + fake.getGoDependenciesReturnsOnCall[i] = struct { + result1 []*spdx.Package + result2 error + }{result1, result2} +} + +func (fake *FakeSpdxImplementation) IgnorePatterns(arg1 string, arg2 []string, arg3 bool) ([]gitignore.Pattern, error) { + var arg2Copy []string + if arg2 != nil { + arg2Copy = make([]string, len(arg2)) + copy(arg2Copy, arg2) + } + fake.ignorePatternsMutex.Lock() + ret, specificReturn := fake.ignorePatternsReturnsOnCall[len(fake.ignorePatternsArgsForCall)] + fake.ignorePatternsArgsForCall = append(fake.ignorePatternsArgsForCall, struct { + arg1 string + arg2 []string + arg3 bool + }{arg1, arg2Copy, arg3}) + stub := fake.IgnorePatternsStub + fakeReturns := fake.ignorePatternsReturns + fake.recordInvocation("IgnorePatterns", []interface{}{arg1, arg2Copy, arg3}) + fake.ignorePatternsMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeSpdxImplementation) IgnorePatternsCallCount() int { + fake.ignorePatternsMutex.RLock() + defer fake.ignorePatternsMutex.RUnlock() + return len(fake.ignorePatternsArgsForCall) +} + +func (fake *FakeSpdxImplementation) IgnorePatternsCalls(stub func(string, []string, bool) ([]gitignore.Pattern, error)) { + fake.ignorePatternsMutex.Lock() + defer fake.ignorePatternsMutex.Unlock() + fake.IgnorePatternsStub = stub +} + +func (fake *FakeSpdxImplementation) IgnorePatternsArgsForCall(i int) (string, []string, bool) { + fake.ignorePatternsMutex.RLock() + defer fake.ignorePatternsMutex.RUnlock() + argsForCall := fake.ignorePatternsArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeSpdxImplementation) IgnorePatternsReturns(result1 []gitignore.Pattern, result2 error) { + fake.ignorePatternsMutex.Lock() + defer fake.ignorePatternsMutex.Unlock() + fake.IgnorePatternsStub = nil + fake.ignorePatternsReturns = struct { + result1 []gitignore.Pattern + result2 error + }{result1, result2} +} + +func (fake *FakeSpdxImplementation) IgnorePatternsReturnsOnCall(i int, result1 []gitignore.Pattern, result2 error) { + fake.ignorePatternsMutex.Lock() + defer fake.ignorePatternsMutex.Unlock() + fake.IgnorePatternsStub = nil + if fake.ignorePatternsReturnsOnCall == nil { + fake.ignorePatternsReturnsOnCall = make(map[int]struct { + result1 []gitignore.Pattern + result2 error + }) + } + fake.ignorePatternsReturnsOnCall[i] = struct { + result1 []gitignore.Pattern + result2 error + }{result1, result2} +} + +func (fake *FakeSpdxImplementation) LicenseReader(arg1 *spdx.Options) (*license.Reader, error) { + fake.licenseReaderMutex.Lock() + ret, specificReturn := fake.licenseReaderReturnsOnCall[len(fake.licenseReaderArgsForCall)] + fake.licenseReaderArgsForCall = append(fake.licenseReaderArgsForCall, struct { + arg1 *spdx.Options + }{arg1}) + stub := fake.LicenseReaderStub + fakeReturns := fake.licenseReaderReturns + fake.recordInvocation("LicenseReader", []interface{}{arg1}) + fake.licenseReaderMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeSpdxImplementation) LicenseReaderCallCount() int { + fake.licenseReaderMutex.RLock() + defer fake.licenseReaderMutex.RUnlock() + return len(fake.licenseReaderArgsForCall) +} + +func (fake *FakeSpdxImplementation) LicenseReaderCalls(stub func(*spdx.Options) (*license.Reader, error)) { + fake.licenseReaderMutex.Lock() + defer fake.licenseReaderMutex.Unlock() + fake.LicenseReaderStub = stub +} + +func (fake *FakeSpdxImplementation) LicenseReaderArgsForCall(i int) *spdx.Options { + fake.licenseReaderMutex.RLock() + defer fake.licenseReaderMutex.RUnlock() + argsForCall := fake.licenseReaderArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeSpdxImplementation) LicenseReaderReturns(result1 *license.Reader, result2 error) { + fake.licenseReaderMutex.Lock() + defer fake.licenseReaderMutex.Unlock() + fake.LicenseReaderStub = nil + fake.licenseReaderReturns = struct { + result1 *license.Reader + result2 error + }{result1, result2} +} + +func (fake *FakeSpdxImplementation) LicenseReaderReturnsOnCall(i int, result1 *license.Reader, result2 error) { + fake.licenseReaderMutex.Lock() + defer fake.licenseReaderMutex.Unlock() + fake.LicenseReaderStub = nil + if fake.licenseReaderReturnsOnCall == nil { + fake.licenseReaderReturnsOnCall = make(map[int]struct { + result1 *license.Reader + result2 error + }) + } + fake.licenseReaderReturnsOnCall[i] = struct { + result1 *license.Reader + result2 error + }{result1, result2} +} + func (fake *FakeSpdxImplementation) PackageFromLayerTarBall(arg1 string, arg2 *spdx.TarballOptions) (*spdx.Package, error) { fake.packageFromLayerTarBallMutex.Lock() ret, specificReturn := fake.packageFromLayerTarBallReturnsOnCall[len(fake.packageFromLayerTarBallArgsForCall)] @@ -338,8 +808,20 @@ func (fake *FakeSpdxImplementation) ReadArchiveManifestReturnsOnCall(i int, resu func (fake *FakeSpdxImplementation) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() + fake.applyIgnorePatternsMutex.RLock() + defer fake.applyIgnorePatternsMutex.RUnlock() fake.extractTarballTmpMutex.RLock() defer fake.extractTarballTmpMutex.RUnlock() + fake.getDirectoryLicenseMutex.RLock() + defer fake.getDirectoryLicenseMutex.RUnlock() + fake.getDirectoryTreeMutex.RLock() + defer fake.getDirectoryTreeMutex.RUnlock() + fake.getGoDependenciesMutex.RLock() + defer fake.getGoDependenciesMutex.RUnlock() + fake.ignorePatternsMutex.RLock() + defer fake.ignorePatternsMutex.RUnlock() + fake.licenseReaderMutex.RLock() + defer fake.licenseReaderMutex.RUnlock() fake.packageFromLayerTarBallMutex.RLock() defer fake.packageFromLayerTarBallMutex.RUnlock() fake.pullImagesToArchiveMutex.RLock()