Skip to content

Commit

Permalink
add relationships for go binary packages (#2912)
Browse files Browse the repository at this point in the history
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
  • Loading branch information
wagoodman committed May 30, 2024
1 parent ac34808 commit f4a69e6
Show file tree
Hide file tree
Showing 18 changed files with 627 additions and 153 deletions.
110 changes: 9 additions & 101 deletions syft/pkg/cataloger/golang/cataloger.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,9 @@ Package golang provides a concrete Cataloger implementation relating to packages
package golang

import (
"context"
"fmt"
"regexp"
"strings"

"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/mimetype"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cpe"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
)
Expand All @@ -30,102 +23,17 @@ func NewGoModuleFileCataloger(opts CatalogerConfig) pkg.Cataloger {
c := goModCataloger{
licenses: newGoLicenses(modFileCatalogerName, opts),
}
return &progressingCataloger{
cataloger: generic.NewCataloger(modFileCatalogerName).
WithParserByGlobs(c.parseGoModFile, "**/go.mod"),
}

return generic.NewCataloger(modFileCatalogerName).
WithParserByGlobs(c.parseGoModFile, "**/go.mod")
}

// NewGoModuleBinaryCataloger returns a new cataloger object that searches within binaries built by the go compiler.
func NewGoModuleBinaryCataloger(opts CatalogerConfig) pkg.Cataloger {
return &progressingCataloger{
cataloger: generic.NewCataloger(binaryCatalogerName).
WithParserByMimeTypes(
newGoBinaryCataloger(opts).parseGoBinary,
mimetype.ExecutableMIMETypeSet.List()...,
),
}
}

type progressingCataloger struct {
cataloger *generic.Cataloger
}

func (p *progressingCataloger) Name() string {
return p.cataloger.Name()
}

func (p *progressingCataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
pkgs, relationships, err := p.cataloger.Catalog(ctx, resolver)
goCompilerPkgs := []pkg.Package{}
totalLocations := file.NewLocationSet()
for _, goPkg := range pkgs {
mValue, ok := goPkg.Metadata.(pkg.GolangBinaryBuildinfoEntry)
if !ok {
continue
}
// go binary packages should only contain a single location
for _, location := range goPkg.Locations.ToSlice() {
if !totalLocations.Contains(location) {
stdLibPkg := newGoStdLib(mValue.GoCompiledVersion, goPkg.Locations)
if stdLibPkg != nil {
goCompilerPkgs = append(goCompilerPkgs, *stdLibPkg)
totalLocations.Add(location)
}
}
}
}
pkgs = append(pkgs, goCompilerPkgs...)
return pkgs, relationships, err
}

func newGoStdLib(version string, location file.LocationSet) *pkg.Package {
stdlibCpe, err := generateStdlibCpe(version)
if err != nil {
return nil
}
goCompilerPkg := &pkg.Package{
Name: "stdlib",
Version: version,
PURL: packageURL("stdlib", strings.TrimPrefix(version, "go")),
CPEs: []cpe.CPE{stdlibCpe},
Locations: location,
Licenses: pkg.NewLicenseSet(pkg.NewLicense("BSD-3-Clause")),
Language: pkg.Go,
Type: pkg.GoModulePkg,
Metadata: pkg.GolangBinaryBuildinfoEntry{
GoCompiledVersion: version,
},
}
goCompilerPkg.SetID()

return goCompilerPkg
}

func generateStdlibCpe(version string) (stdlibCpe cpe.CPE, err error) {
// GoCompiledVersion when pulled from a binary is prefixed by go
version = strings.TrimPrefix(version, "go")

// we also need to trim starting from the first +<metadata> to
// correctly extract potential rc candidate information for cpe generation
// ex: 2.0.0-rc.1+build.123 -> 2.0.0-rc.1; if no + is found then + is returned
after, _, found := strings.Cut("+", version)
if found {
version = after
}

// extracting <version> and <candidate>
// https://regex101.com/r/985GsI/1
captureGroups := internal.MatchNamedCaptureGroups(versionCandidateGroups, version)
vr, ok := captureGroups["version"]
if !ok || vr == "" {
return stdlibCpe, fmt.Errorf("could not match candidate version for: %s", version)
}

cpeString := fmt.Sprintf("cpe:2.3:a:golang:go:%s:-:*:*:*:*:*:*", captureGroups["version"])
if candidate, ok := captureGroups["candidate"]; ok && candidate != "" {
cpeString = fmt.Sprintf("cpe:2.3:a:golang:go:%s:%s:*:*:*:*:*:*", vr, candidate)
}

return cpe.New(cpeString, cpe.GeneratedSource)
return generic.NewCataloger(binaryCatalogerName).
WithParserByMimeTypes(
newGoBinaryCataloger(opts).parseGoBinary,
mimetype.ExecutableMIMETypeSet.List()...,
).
WithProcessors(stdlibProcessor)
}
86 changes: 86 additions & 0 deletions syft/pkg/cataloger/golang/cataloger_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,92 @@ import (
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
)

func Test_PackageCataloger_Binary(t *testing.T) {

tests := []struct {
name string
fixture string
expectedPkgs []string
expectedRels []string
}{
{
name: "simple module with dependencies",
fixture: "image-small",
expectedPkgs: []string{
"anchore.io/not/real @ (devel) (/run-me)",
"github.com/andybalholm/brotli @ v1.0.1 (/run-me)",
"github.com/dsnet/compress @ v0.0.2-0.20210315054119-f66993602bf5 (/run-me)",
"github.com/golang/snappy @ v0.0.2 (/run-me)",
"github.com/klauspost/compress @ v1.11.4 (/run-me)",
"github.com/klauspost/pgzip @ v1.2.5 (/run-me)",
"github.com/mholt/archiver/v3 @ v3.5.1 (/run-me)",
"github.com/nwaples/rardecode @ v1.1.0 (/run-me)",
"github.com/pierrec/lz4/v4 @ v4.1.2 (/run-me)",
"github.com/ulikunitz/xz @ v0.5.9 (/run-me)",
"github.com/xi2/xz @ v0.0.0-20171230120015-48954b6210f8 (/run-me)",
"stdlib @ go1.22.3 (/run-me)",
},
expectedRels: []string{
"github.com/andybalholm/brotli @ v1.0.1 (/run-me) [dependency-of] anchore.io/not/real @ (devel) (/run-me)",
"github.com/dsnet/compress @ v0.0.2-0.20210315054119-f66993602bf5 (/run-me) [dependency-of] anchore.io/not/real @ (devel) (/run-me)",
"github.com/golang/snappy @ v0.0.2 (/run-me) [dependency-of] anchore.io/not/real @ (devel) (/run-me)",
"github.com/klauspost/compress @ v1.11.4 (/run-me) [dependency-of] anchore.io/not/real @ (devel) (/run-me)",
"github.com/klauspost/pgzip @ v1.2.5 (/run-me) [dependency-of] anchore.io/not/real @ (devel) (/run-me)",
"github.com/mholt/archiver/v3 @ v3.5.1 (/run-me) [dependency-of] anchore.io/not/real @ (devel) (/run-me)",
"github.com/nwaples/rardecode @ v1.1.0 (/run-me) [dependency-of] anchore.io/not/real @ (devel) (/run-me)",
"github.com/pierrec/lz4/v4 @ v4.1.2 (/run-me) [dependency-of] anchore.io/not/real @ (devel) (/run-me)",
"github.com/ulikunitz/xz @ v0.5.9 (/run-me) [dependency-of] anchore.io/not/real @ (devel) (/run-me)",
"github.com/xi2/xz @ v0.0.0-20171230120015-48954b6210f8 (/run-me) [dependency-of] anchore.io/not/real @ (devel) (/run-me)",
"stdlib @ go1.22.3 (/run-me) [dependency-of] anchore.io/not/real @ (devel) (/run-me)",
},
},
{
name: "partially built binary",
// the difference is the build flags used to build the binary... they will not reference the module directly
// see the dockerfile for details
fixture: "image-not-a-module",
expectedPkgs: []string{
"command-line-arguments @ (devel) (/run-me)", // this is the difference!
"github.com/andybalholm/brotli @ v1.0.1 (/run-me)",
"github.com/dsnet/compress @ v0.0.2-0.20210315054119-f66993602bf5 (/run-me)",
"github.com/golang/snappy @ v0.0.2 (/run-me)",
"github.com/klauspost/compress @ v1.11.4 (/run-me)",
"github.com/klauspost/pgzip @ v1.2.5 (/run-me)",
"github.com/mholt/archiver/v3 @ v3.5.1 (/run-me)",
"github.com/nwaples/rardecode @ v1.1.0 (/run-me)",
"github.com/pierrec/lz4/v4 @ v4.1.2 (/run-me)",
"github.com/ulikunitz/xz @ v0.5.9 (/run-me)",
"github.com/xi2/xz @ v0.0.0-20171230120015-48954b6210f8 (/run-me)",
"stdlib @ go1.22.3 (/run-me)",
},
expectedRels: []string{
"github.com/andybalholm/brotli @ v1.0.1 (/run-me) [dependency-of] command-line-arguments @ (devel) (/run-me)",
"github.com/dsnet/compress @ v0.0.2-0.20210315054119-f66993602bf5 (/run-me) [dependency-of] command-line-arguments @ (devel) (/run-me)",
"github.com/golang/snappy @ v0.0.2 (/run-me) [dependency-of] command-line-arguments @ (devel) (/run-me)",
"github.com/klauspost/compress @ v1.11.4 (/run-me) [dependency-of] command-line-arguments @ (devel) (/run-me)",
"github.com/klauspost/pgzip @ v1.2.5 (/run-me) [dependency-of] command-line-arguments @ (devel) (/run-me)",
"github.com/mholt/archiver/v3 @ v3.5.1 (/run-me) [dependency-of] command-line-arguments @ (devel) (/run-me)",
"github.com/nwaples/rardecode @ v1.1.0 (/run-me) [dependency-of] command-line-arguments @ (devel) (/run-me)",
"github.com/pierrec/lz4/v4 @ v4.1.2 (/run-me) [dependency-of] command-line-arguments @ (devel) (/run-me)",
"github.com/ulikunitz/xz @ v0.5.9 (/run-me) [dependency-of] command-line-arguments @ (devel) (/run-me)",
"github.com/xi2/xz @ v0.0.0-20171230120015-48954b6210f8 (/run-me) [dependency-of] command-line-arguments @ (devel) (/run-me)",
"stdlib @ go1.22.3 (/run-me) [dependency-of] command-line-arguments @ (devel) (/run-me)",
},
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
pkgtest.NewCatalogTester().
WithImageResolver(t, test.fixture).
ExpectsPackageStrings(test.expectedPkgs).
ExpectsRelationshipStrings(test.expectedRels).
TestCataloger(t, NewGoModuleBinaryCataloger(DefaultCatalogerConfig()))
})
}

}

func Test_Mod_Cataloger_Globs(t *testing.T) {
tests := []struct {
name string
Expand Down
34 changes: 27 additions & 7 deletions syft/pkg/cataloger/golang/parse_go_binary.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,17 +69,38 @@ func (c *goBinaryCataloger) parseGoBinary(_ context.Context, resolver file.Resol
mods := scanFile(unionReader, reader.RealPath)
internal.CloseAndLogError(reader.ReadCloser, reader.RealPath)

var rels []artifact.Relationship
for _, mod := range mods {
pkgs = append(pkgs, c.buildGoPkgInfo(resolver, reader.Location, mod, mod.arch, unionReader)...)
var depPkgs []pkg.Package
mainPkg, depPkgs := c.buildGoPkgInfo(resolver, reader.Location, mod, mod.arch, unionReader)
if mainPkg != nil {
rels = createModuleRelationships(*mainPkg, depPkgs)
pkgs = append(pkgs, *mainPkg)
}
pkgs = append(pkgs, depPkgs...)
}

return pkgs, rels, nil
}

func createModuleRelationships(main pkg.Package, deps []pkg.Package) []artifact.Relationship {
var relationships []artifact.Relationship

for _, dep := range deps {
relationships = append(relationships, artifact.Relationship{
From: dep,
To: main,
Type: artifact.DependencyOfRelationship,
})
}

return pkgs, nil, nil
return relationships
}

func (c *goBinaryCataloger) buildGoPkgInfo(resolver file.Resolver, location file.Location, mod *extendedBuildInfo, arch string, reader io.ReadSeekCloser) []pkg.Package {
func (c *goBinaryCataloger) buildGoPkgInfo(resolver file.Resolver, location file.Location, mod *extendedBuildInfo, arch string, reader io.ReadSeekCloser) (*pkg.Package, []pkg.Package) {
var pkgs []pkg.Package
if mod == nil {
return pkgs
return nil, pkgs
}

var empty debug.Module
Expand Down Expand Up @@ -110,13 +131,12 @@ func (c *goBinaryCataloger) buildGoPkgInfo(resolver file.Resolver, location file
}

if mod.Main == empty {
return pkgs
return nil, pkgs
}

main := c.makeGoMainPackage(resolver, mod, arch, location, reader)
pkgs = append(pkgs, main)

return pkgs
return &main, pkgs
}

func (c *goBinaryCataloger) makeGoMainPackage(resolver file.Resolver, mod *extendedBuildInfo, arch string, location file.Location, reader io.ReadSeekCloser) pkg.Package {
Expand Down
5 changes: 4 additions & 1 deletion syft/pkg/cataloger/golang/parse_go_binary_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -964,7 +964,10 @@ func TestBuildGoPkgInfo(t *testing.T) {
c := newGoBinaryCataloger(DefaultCatalogerConfig())
reader, err := unionreader.GetUnionReader(io.NopCloser(strings.NewReader(test.binaryContent)))
require.NoError(t, err)
pkgs := c.buildGoPkgInfo(fileresolver.Empty{}, location, test.mod, test.mod.arch, reader)
mainPkg, pkgs := c.buildGoPkgInfo(fileresolver.Empty{}, location, test.mod, test.mod.arch, reader)
if mainPkg != nil {
pkgs = append(pkgs, *mainPkg)
}
require.Len(t, pkgs, len(test.expected))
for i, p := range pkgs {
pkgtest.AssertPackagesEqual(t, test.expected[i], p)
Expand Down
100 changes: 100 additions & 0 deletions syft/pkg/cataloger/golang/stdlib_package.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
package golang

import (
"fmt"
"strings"

"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cpe"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
)

func stdlibProcessor(pkgs []pkg.Package, relationships []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) {
compilerPkgs, newRelationships := stdlibPackageAndRelationships(pkgs)
return append(pkgs, compilerPkgs...), append(relationships, newRelationships...), err
}

func stdlibPackageAndRelationships(pkgs []pkg.Package) ([]pkg.Package, []artifact.Relationship) {
var goCompilerPkgs []pkg.Package
var relationships []artifact.Relationship
totalLocations := file.NewLocationSet()
for _, goPkg := range pkgs {
mValue, ok := goPkg.Metadata.(pkg.GolangBinaryBuildinfoEntry)
if !ok {
continue
}

// go binary packages should only contain a single location
for _, location := range goPkg.Locations.ToSlice() {
if totalLocations.Contains(location) {
continue
}

stdLibPkg := newGoStdLib(mValue.GoCompiledVersion, goPkg.Locations)
if stdLibPkg != nil {
goCompilerPkgs = append(goCompilerPkgs, *stdLibPkg)
totalLocations.Add(location)
}

relationships = append(relationships, artifact.Relationship{
From: *stdLibPkg,
To: goPkg,
Type: artifact.DependencyOfRelationship,
})
}
}
return goCompilerPkgs, relationships
}

func newGoStdLib(version string, location file.LocationSet) *pkg.Package {
stdlibCpe, err := generateStdlibCpe(version)
if err != nil {
return nil
}
goCompilerPkg := &pkg.Package{
Name: "stdlib",
Version: version,
PURL: packageURL("stdlib", strings.TrimPrefix(version, "go")),
CPEs: []cpe.CPE{stdlibCpe},
Locations: location,
Licenses: pkg.NewLicenseSet(pkg.NewLicense("BSD-3-Clause")),
Language: pkg.Go,
Type: pkg.GoModulePkg,
Metadata: pkg.GolangBinaryBuildinfoEntry{
GoCompiledVersion: version,
},
}
goCompilerPkg.SetID()

return goCompilerPkg
}

func generateStdlibCpe(version string) (stdlibCpe cpe.CPE, err error) {
// GoCompiledVersion when pulled from a binary is prefixed by go
version = strings.TrimPrefix(version, "go")

// we also need to trim starting from the first +<metadata> to
// correctly extract potential rc candidate information for cpe generation
// ex: 2.0.0-rc.1+build.123 -> 2.0.0-rc.1; if no + is found then + is returned
after, _, found := strings.Cut("+", version)
if found {
version = after
}

// extracting <version> and <candidate>
// https://regex101.com/r/985GsI/1
captureGroups := internal.MatchNamedCaptureGroups(versionCandidateGroups, version)
vr, ok := captureGroups["version"]
if !ok || vr == "" {
return stdlibCpe, fmt.Errorf("could not match candidate version for: %s", version)
}

cpeString := fmt.Sprintf("cpe:2.3:a:golang:go:%s:-:*:*:*:*:*:*", captureGroups["version"])
if candidate, ok := captureGroups["candidate"]; ok && candidate != "" {
cpeString = fmt.Sprintf("cpe:2.3:a:golang:go:%s:%s:*:*:*:*:*:*", vr, candidate)
}

return cpe.New(cpeString, cpe.GeneratedSource)
}
Loading

0 comments on commit f4a69e6

Please sign in to comment.