From ffb7fc94967d4ba0ebcf08a181e7e7f500728fc8 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 7 May 2024 13:30:21 -0400 Subject: [PATCH 01/11] add internal dependency resolver Signed-off-by: Alex Goodman --- syft/pkg/cataloger/debian/parse_dpkg_db.go | 79 +------------------ syft/pkg/cataloger/debian/prosumer.go | 75 ++++++++++++++++++ .../internal/dependency/catalog_decorator.go | 31 ++++++++ .../cataloger/internal/dependency/resolver.go | 50 ++++++++++++ 4 files changed, 158 insertions(+), 77 deletions(-) create mode 100644 syft/pkg/cataloger/debian/prosumer.go create mode 100644 syft/pkg/cataloger/internal/dependency/catalog_decorator.go create mode 100644 syft/pkg/cataloger/internal/dependency/resolver.go diff --git a/syft/pkg/cataloger/debian/parse_dpkg_db.go b/syft/pkg/cataloger/debian/parse_dpkg_db.go index 4fb41323c4f..e56e34d3580 100644 --- a/syft/pkg/cataloger/debian/parse_dpkg_db.go +++ b/syft/pkg/cataloger/debian/parse_dpkg_db.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" "io" "regexp" "strings" @@ -37,7 +38,7 @@ func parseDpkgDB(_ context.Context, resolver file.Resolver, env *generic.Environ pkgs = append(pkgs, newDpkgPackage(m, reader.Location, resolver, env.LinuxRelease)) } - return pkgs, associateRelationships(pkgs), nil + return pkgs, dependency.Resolve(pkgs, newProsumer()), nil } // parseDpkgStatus is a parser function for Debian DB status contents, returning all Debian packages listed. @@ -238,79 +239,3 @@ func handleNewKeyValue(line string) (key string, val interface{}, err error) { return "", nil, fmt.Errorf("cannot parse field from line: '%s'", line) } - -// associateRelationships will create relationships between packages based on the "Depends", "Pre-Depends", and "Provides" -// fields for installed packages. if there is an installed package that has a dependency that is (somehow) not installed, -// then that relationship (between the installed and uninstalled package) will NOT be created. -func associateRelationships(pkgs []pkg.Package) (relationships []artifact.Relationship) { - // map["provides" + "package"] -> packages that provide that package - lookup := make(map[string][]pkg.Package) - - // read provided and add as keys for lookup keys as well as package names - for _, p := range pkgs { - meta, ok := p.Metadata.(pkg.DpkgDBEntry) - if !ok { - log.Warnf("cataloger failed to extract dpkg 'provides' metadata for package %+v", p.Name) - continue - } - lookup[p.Name] = append(lookup[p.Name], p) - for _, provides := range meta.Provides { - k := stripVersionSpecifier(provides) - lookup[k] = append(lookup[k], p) - } - } - - // read "Depends" and "Pre-Depends" and match with keys - for _, p := range pkgs { - meta, ok := p.Metadata.(pkg.DpkgDBEntry) - if !ok { - log.Warnf("cataloger failed to extract dpkg 'dependency' metadata for package %+v", p.Name) - continue - } - - var allDeps []string - allDeps = append(allDeps, meta.Depends...) - allDeps = append(allDeps, meta.PreDepends...) - - for _, depSpecifier := range allDeps { - deps := splitPackageChoice(depSpecifier) - for _, dep := range deps { - for _, depPkg := range lookup[dep] { - relationships = append(relationships, artifact.Relationship{ - From: depPkg, - To: p, - Type: artifact.DependencyOfRelationship, - }) - } - } - } - } - return relationships -} - -func stripVersionSpecifier(s string) string { - // examples: - // libgmp10 (>= 2:6.2.1+dfsg1) --> libgmp10 - // libgmp10 --> libgmp10 - // foo [i386] --> foo - // default-mta | mail-transport-agent --> default-mta | mail-transport-agent - // kernel-headers-2.2.10 [!hurd-i386] --> kernel-headers-2.2.10 - - items := internal.SplitAny(s, "[(<>=") - if len(items) == 0 { - return s - } - - return strings.TrimSpace(items[0]) -} - -func splitPackageChoice(s string) (ret []string) { - fields := strings.Split(s, "|") - for _, field := range fields { - field = strings.TrimSpace(field) - if field != "" { - ret = append(ret, stripVersionSpecifier(field)) - } - } - return ret -} diff --git a/syft/pkg/cataloger/debian/prosumer.go b/syft/pkg/cataloger/debian/prosumer.go new file mode 100644 index 00000000000..6291f9a0979 --- /dev/null +++ b/syft/pkg/cataloger/debian/prosumer.go @@ -0,0 +1,75 @@ +package debian + +import ( + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" + "strings" +) + +var _ dependency.Prosumer = (*dpkgDBEntryProsumer)(nil) + +type dpkgDBEntryProsumer struct{} + +func newProsumer() dpkgDBEntryProsumer { + return dpkgDBEntryProsumer{} +} + +func (ps dpkgDBEntryProsumer) Provides(p pkg.Package) []string { + meta, ok := p.Metadata.(pkg.DpkgDBEntry) + if !ok { + log.Warnf("cataloger failed to extract dpkg 'provides' metadata for package %+v", p.Name) + return nil + } + keys := []string{p.Name} + for _, provides := range meta.Provides { + keys = append(keys, stripVersionSpecifier(provides)) + } + return keys +} + +func (ps dpkgDBEntryProsumer) Requires(p pkg.Package) []string { + meta, ok := p.Metadata.(pkg.DpkgDBEntry) + if !ok { + log.Warnf("cataloger failed to extract dpkg 'requires' metadata for package %+v", p.Name) + return nil + } + + var allDeps []string + allDeps = append(allDeps, meta.Depends...) + allDeps = append(allDeps, meta.PreDepends...) + + var keys []string + for _, depSpecifier := range allDeps { + keys = append(keys, splitPackageChoice(depSpecifier)...) + } + return keys +} + +func stripVersionSpecifier(s string) string { + // examples: + // libgmp10 (>= 2:6.2.1+dfsg1) --> libgmp10 + // libgmp10 --> libgmp10 + // foo [i386] --> foo + // default-mta | mail-transport-agent --> default-mta | mail-transport-agent + // kernel-headers-2.2.10 [!hurd-i386] --> kernel-headers-2.2.10 + + items := internal.SplitAny(s, "[(<>=") + if len(items) == 0 { + return s + } + + return strings.TrimSpace(items[0]) +} + +func splitPackageChoice(s string) (ret []string) { + fields := strings.Split(s, "|") + for _, field := range fields { + field = strings.TrimSpace(field) + if field != "" { + ret = append(ret, stripVersionSpecifier(field)) + } + } + return ret +} diff --git a/syft/pkg/cataloger/internal/dependency/catalog_decorator.go b/syft/pkg/cataloger/internal/dependency/catalog_decorator.go new file mode 100644 index 00000000000..f758aa436d5 --- /dev/null +++ b/syft/pkg/cataloger/internal/dependency/catalog_decorator.go @@ -0,0 +1,31 @@ +package dependency + +import ( + "context" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" +) + +type catalogerDecorator struct { + pkg.Cataloger + resolver RelationshipResolver +} + +func DecorateCatalogerWithRelationships(cataloger pkg.Cataloger, prosumer Prosumer) pkg.Cataloger { + return &catalogerDecorator{ + Cataloger: cataloger, + resolver: NewRelationshipResolver(prosumer), + } +} + +func (c catalogerDecorator) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { + pkgs, rels, err := c.Cataloger.Catalog(ctx, resolver) + if err != nil { + return nil, nil, err + } + + rels = append(rels, c.resolver.Resolve(pkgs)...) + + return pkgs, rels, nil +} diff --git a/syft/pkg/cataloger/internal/dependency/resolver.go b/syft/pkg/cataloger/internal/dependency/resolver.go new file mode 100644 index 00000000000..4204082589d --- /dev/null +++ b/syft/pkg/cataloger/internal/dependency/resolver.go @@ -0,0 +1,50 @@ +package dependency + +import ( + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/pkg" +) + +// Prosumer is a producer and consumer, in this context, for packages that provide resources and require resources. +type Prosumer interface { + Provides(pkg.Package) []string + Requires(pkg.Package) []string +} + +type RelationshipResolver struct { + prosumer Prosumer +} + +func NewRelationshipResolver(p Prosumer) RelationshipResolver { + return RelationshipResolver{ + prosumer: p, + } +} + +// Resolve will create relationships between packages based on the "Depends" and "Provides" specifications from the given packages. +func (r RelationshipResolver) Resolve(pkgs []pkg.Package) (relationships []artifact.Relationship) { + lookup := make(map[string][]pkg.Package) + + for _, p := range pkgs { + for _, key := range r.prosumer.Provides(p) { + lookup[key] = append(lookup[key], p) + } + } + + for _, p := range pkgs { + for _, requirement := range r.prosumer.Requires(p) { + for _, depPkg := range lookup[requirement] { + relationships = append(relationships, artifact.Relationship{ + From: depPkg, + To: p, + Type: artifact.DependencyOfRelationship, + }) + } + } + } + return relationships +} + +func Resolve(pkgs []pkg.Package, prosumer Prosumer) []artifact.Relationship { + return NewRelationshipResolver(prosumer).Resolve(pkgs) +} From eb7f4d486d3f45429436efbf60d2d9a45ba771fa Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 7 May 2024 15:19:09 -0400 Subject: [PATCH 02/11] refactor dependency relationship resolution to common object Signed-off-by: Alex Goodman --- syft/pkg/cataloger/arch/cataloger.go | 90 +----------- syft/pkg/cataloger/arch/prosumer.go | 53 ++++++++ syft/pkg/cataloger/debian/cataloger.go | 12 +- syft/pkg/cataloger/debian/parse_dpkg_db.go | 3 +- .../cataloger/debian/parse_dpkg_db_test.go | 30 ++-- syft/pkg/cataloger/debian/prosumer.go | 5 +- .../dpkg/status.d}/coreutils-relationships | 0 .../lib/dpkg/status.d}/doc-examples | 0 .../{status => var/lib/dpkg/status.d}/empty | 0 .../lib/dpkg/status.d}/installed-size-4KB | 0 .../lib/dpkg/status.d}/libpam-runtime | 0 .../lib/dpkg/status.d}/multiple | 0 .../{status => var/lib/dpkg/status.d}/single | 0 .../internal/dependency/catalog_decorator.go | 1 + .../dependency/catalog_decorator_test.go | 106 +++++++++++++++ .../cataloger/internal/dependency/resolver.go | 37 ++++- .../internal/dependency/resolver_test.go | 128 ++++++++++++++++++ 17 files changed, 347 insertions(+), 118 deletions(-) create mode 100644 syft/pkg/cataloger/arch/prosumer.go rename syft/pkg/cataloger/debian/test-fixtures/{status => var/lib/dpkg/status.d}/coreutils-relationships (100%) rename syft/pkg/cataloger/debian/test-fixtures/{status => var/lib/dpkg/status.d}/doc-examples (100%) rename syft/pkg/cataloger/debian/test-fixtures/{status => var/lib/dpkg/status.d}/empty (100%) rename syft/pkg/cataloger/debian/test-fixtures/{status => var/lib/dpkg/status.d}/installed-size-4KB (100%) rename syft/pkg/cataloger/debian/test-fixtures/{status => var/lib/dpkg/status.d}/libpam-runtime (100%) rename syft/pkg/cataloger/debian/test-fixtures/{status => var/lib/dpkg/status.d}/multiple (100%) rename syft/pkg/cataloger/debian/test-fixtures/{status => var/lib/dpkg/status.d}/single (100%) create mode 100644 syft/pkg/cataloger/internal/dependency/catalog_decorator_test.go create mode 100644 syft/pkg/cataloger/internal/dependency/resolver_test.go diff --git a/syft/pkg/cataloger/arch/cataloger.go b/syft/pkg/cataloger/arch/cataloger.go index 920dda60a8f..b7fcc587d18 100644 --- a/syft/pkg/cataloger/arch/cataloger.go +++ b/syft/pkg/cataloger/arch/cataloger.go @@ -4,96 +4,16 @@ Package arch provides a concrete Cataloger implementations for packages relating package arch import ( - "context" - "strings" - - "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/artifact" - "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" ) -type cataloger struct { - *generic.Cataloger -} - // NewDBCataloger returns a new cataloger object initialized for arch linux pacman database flat-file stores. func NewDBCataloger() pkg.Cataloger { - return cataloger{ - Cataloger: generic.NewCataloger("alpm-db-cataloger"). + return dependency.DecorateCatalogerWithRelationships( + generic.NewCataloger("alpm-db-cataloger"). WithParserByGlobs(parseAlpmDB, pkg.AlpmDBGlob), - } -} - -func (c cataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { - pkgs, rels, err := c.Cataloger.Catalog(ctx, resolver) - if err != nil { - return nil, nil, err - } - - rels = append(rels, associateRelationships(pkgs)...) - - return pkgs, rels, nil -} - -// associateRelationships will create relationships between packages based on the "Depends" and "Provides" -// fields for installed packages. If there is an installed package that has a dependency that is (somehow) not installed, -// then that relationship (between the installed and uninstalled package) will NOT be created. -func associateRelationships(pkgs []pkg.Package) (relationships []artifact.Relationship) { - // map["provides" + "package"] -> packages that provide that package - lookup := make(map[string][]pkg.Package) - - // read providers and add lookup keys as needed - for _, p := range pkgs { - meta, ok := p.Metadata.(pkg.AlpmDBEntry) - if !ok { - log.Warnf("cataloger failed to extract alpm 'provides' metadata for package %+v", p.Name) - continue - } - // allow for lookup by package name - lookup[p.Name] = append(lookup[p.Name], p) - - for _, provides := range meta.Provides { - // allow for lookup by exact specification - lookup[provides] = append(lookup[provides], p) - - // allow for lookup by library name only - k := stripVersionSpecifier(provides) - lookup[k] = append(lookup[k], p) - } - } - - // read "Depends" and match with provider keys - for _, p := range pkgs { - meta, ok := p.Metadata.(pkg.AlpmDBEntry) - if !ok { - log.Warnf("cataloger failed to extract alpm 'dependency' metadata for package %+v", p.Name) - continue - } - - for _, dep := range meta.Depends { - for _, depPkg := range lookup[dep] { - relationships = append(relationships, artifact.Relationship{ - From: depPkg, - To: p, - Type: artifact.DependencyOfRelationship, - }) - } - } - } - return relationships -} - -func stripVersionSpecifier(s string) string { - // examples: - // gcc-libs --> gcc-libs - // libtree-sitter.so=0-64 --> libtree-sitter.so - - items := strings.Split(s, "=") - if len(items) == 0 { - return s - } - - return strings.TrimSpace(items[0]) + newDBProsumer(), + ) } diff --git a/syft/pkg/cataloger/arch/prosumer.go b/syft/pkg/cataloger/arch/prosumer.go new file mode 100644 index 00000000000..b59f56f2956 --- /dev/null +++ b/syft/pkg/cataloger/arch/prosumer.go @@ -0,0 +1,53 @@ +package arch + +import ( + "strings" + + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" +) + +var _ dependency.Prosumer = (*alpmDBEntryProsumer)(nil) + +type alpmDBEntryProsumer struct{} + +func newDBProsumer() alpmDBEntryProsumer { + return alpmDBEntryProsumer{} +} + +func (ps alpmDBEntryProsumer) Provides(p pkg.Package) []string { + meta, ok := p.Metadata.(pkg.AlpmDBEntry) + if !ok { + log.Warnf("cataloger failed to extract alpm 'provides' metadata for package %+v", p.Name) + return nil + } + keys := []string{p.Name} + for _, provides := range meta.Provides { + keys = append(keys, provides, stripVersionSpecifier(provides)) + } + return keys +} + +func (ps alpmDBEntryProsumer) Requires(p pkg.Package) []string { + meta, ok := p.Metadata.(pkg.AlpmDBEntry) + if !ok { + log.Warnf("cataloger failed to extract alpm 'requires' metadata for package %+v", p.Name) + return nil + } + + return meta.Depends +} + +func stripVersionSpecifier(s string) string { + // examples: + // gcc-libs --> gcc-libs + // libtree-sitter.so=0-64 --> libtree-sitter.so + + items := strings.Split(s, "=") + if len(items) == 0 { + return s + } + + return strings.TrimSpace(items[0]) +} diff --git a/syft/pkg/cataloger/debian/cataloger.go b/syft/pkg/cataloger/debian/cataloger.go index 71369984344..41efe70dc83 100644 --- a/syft/pkg/cataloger/debian/cataloger.go +++ b/syft/pkg/cataloger/debian/cataloger.go @@ -6,12 +6,16 @@ package debian import ( "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" ) // NewDBCataloger returns a new Deb package cataloger capable of parsing DPKG status DB flat-file stores. func NewDBCataloger() pkg.Cataloger { - return generic.NewCataloger("dpkg-db-cataloger"). - // note: these globs have been intentionally split up in order to improve search performance, - // please do NOT combine into: "**/var/lib/dpkg/{status,status.d/*}" - WithParserByGlobs(parseDpkgDB, "**/var/lib/dpkg/status", "**/var/lib/dpkg/status.d/*", "**/lib/opkg/info/*.control", "**/lib/opkg/status") + return dependency.DecorateCatalogerWithRelationships( + generic.NewCataloger("dpkg-db-cataloger"). + // note: these globs have been intentionally split up in order to improve search performance, + // please do NOT combine into: "**/var/lib/dpkg/{status,status.d/*}" + WithParserByGlobs(parseDpkgDB, "**/var/lib/dpkg/status", "**/var/lib/dpkg/status.d/*", "**/lib/opkg/info/*.control", "**/lib/opkg/status"), + newDBProsumer(), + ) } diff --git a/syft/pkg/cataloger/debian/parse_dpkg_db.go b/syft/pkg/cataloger/debian/parse_dpkg_db.go index e56e34d3580..077e5bef906 100644 --- a/syft/pkg/cataloger/debian/parse_dpkg_db.go +++ b/syft/pkg/cataloger/debian/parse_dpkg_db.go @@ -5,7 +5,6 @@ import ( "context" "errors" "fmt" - "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" "io" "regexp" "strings" @@ -38,7 +37,7 @@ func parseDpkgDB(_ context.Context, resolver file.Resolver, env *generic.Environ pkgs = append(pkgs, newDpkgPackage(m, reader.Location, resolver, env.LinuxRelease)) } - return pkgs, dependency.Resolve(pkgs, newProsumer()), nil + return pkgs, nil, nil } // parseDpkgStatus is a parser function for Debian DB status contents, returning all Debian packages listed. diff --git a/syft/pkg/cataloger/debian/parse_dpkg_db_test.go b/syft/pkg/cataloger/debian/parse_dpkg_db_test.go index 9a2d4dcf1c6..c411cf52f80 100644 --- a/syft/pkg/cataloger/debian/parse_dpkg_db_test.go +++ b/syft/pkg/cataloger/debian/parse_dpkg_db_test.go @@ -16,7 +16,6 @@ import ( "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/linux" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/pkg/cataloger/generic" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" ) @@ -28,7 +27,7 @@ func Test_parseDpkgStatus(t *testing.T) { }{ { name: "single package", - fixturePath: "test-fixtures/status/single", + fixturePath: "test-fixtures/var/lib/dpkg/status.d/single", expected: []pkg.DpkgDBEntry{ { Package: "apt", @@ -102,7 +101,7 @@ func Test_parseDpkgStatus(t *testing.T) { }, { name: "single package with installed size", - fixturePath: "test-fixtures/status/installed-size-4KB", + fixturePath: "test-fixtures/var/lib/dpkg/status.d/installed-size-4KB", expected: []pkg.DpkgDBEntry{ { Package: "apt", @@ -143,7 +142,7 @@ func Test_parseDpkgStatus(t *testing.T) { }, { name: "multiple entries", - fixturePath: "test-fixtures/status/multiple", + fixturePath: "test-fixtures/var/lib/dpkg/status.d/multiple", expected: []pkg.DpkgDBEntry{ { Package: "no-version", @@ -477,7 +476,7 @@ func Test_associateRelationships(t *testing.T) { }{ { name: "relationships for coreutils", - fixture: "test-fixtures/status/coreutils-relationships", + fixture: "test-fixtures/var/lib/dpkg/status.d/coreutils-relationships", wantRelationships: map[string][]string{ "coreutils": {"libacl1", "libattr1", "libc6", "libgmp10", "libselinux1"}, "libacl1": {"libc6"}, @@ -491,36 +490,31 @@ func Test_associateRelationships(t *testing.T) { }, { name: "relationships from dpkg example docs", - fixture: "test-fixtures/status/doc-examples", + fixture: "test-fixtures/var/lib/dpkg/status.d/doc-examples", wantRelationships: map[string][]string{ - "made-up-package-1": {"kernel-headers-2.2.10", "hurd-dev", "gnumach-dev"}, - "made-up-package-2": {"libluajit5.1-dev", "liblua5.1-dev"}, - "made-up-package-3": {"foo", "bar"}, + "made-up-package-1": {"gnumach-dev", "hurd-dev", "kernel-headers-2.2.10"}, + "made-up-package-2": {"liblua5.1-dev", "libluajit5.1-dev"}, + "made-up-package-3": {"bar", "foo"}, // note that the "made-up-package-4" depends on "made-up-package-5" but not via the direct // package name, but through the "provides" virtual package name "virtual-package-5". "made-up-package-4": {"made-up-package-5"}, // note that though there is a "default-mta | mail-transport-agent | not-installed" // dependency choice we raise up the packages that are installed for every choice. // In this case that means that "default-mta" and "mail-transport-agent". - "mutt": {"libc6", "default-mta", "mail-transport-agent"}, + "mutt": {"default-mta", "libc6", "mail-transport-agent"}, }, }, { name: "relationships for libpam-runtime", - fixture: "test-fixtures/status/libpam-runtime", + fixture: "test-fixtures/var/lib/dpkg/status.d/libpam-runtime", wantRelationships: map[string][]string{ - "libpam-runtime": {"debconf1", "debconf-2.0", "debconf2", "cdebconf", "libpam-modules"}, + "libpam-runtime": {"cdebconf", "debconf-2.0", "debconf1", "debconf2", "libpam-modules"}, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - f, err := os.Open(tt.fixture) - require.NoError(t, err) - - reader := file.NewLocationReadCloser(file.NewLocation(tt.fixture), f) - - pkgs, relationships, err := parseDpkgDB(context.Background(), nil, &generic.Environment{}, reader) + pkgs, relationships, err := NewDBCataloger().Catalog(context.Background(), file.NewMockResolverForPaths(tt.fixture)) require.NotEmpty(t, pkgs) require.NotEmpty(t, relationships) require.NoError(t, err) diff --git a/syft/pkg/cataloger/debian/prosumer.go b/syft/pkg/cataloger/debian/prosumer.go index 6291f9a0979..3d377e3656e 100644 --- a/syft/pkg/cataloger/debian/prosumer.go +++ b/syft/pkg/cataloger/debian/prosumer.go @@ -1,18 +1,19 @@ package debian import ( + "strings" + "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" - "strings" ) var _ dependency.Prosumer = (*dpkgDBEntryProsumer)(nil) type dpkgDBEntryProsumer struct{} -func newProsumer() dpkgDBEntryProsumer { +func newDBProsumer() dpkgDBEntryProsumer { return dpkgDBEntryProsumer{} } diff --git a/syft/pkg/cataloger/debian/test-fixtures/status/coreutils-relationships b/syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/coreutils-relationships similarity index 100% rename from syft/pkg/cataloger/debian/test-fixtures/status/coreutils-relationships rename to syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/coreutils-relationships diff --git a/syft/pkg/cataloger/debian/test-fixtures/status/doc-examples b/syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/doc-examples similarity index 100% rename from syft/pkg/cataloger/debian/test-fixtures/status/doc-examples rename to syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/doc-examples diff --git a/syft/pkg/cataloger/debian/test-fixtures/status/empty b/syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/empty similarity index 100% rename from syft/pkg/cataloger/debian/test-fixtures/status/empty rename to syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/empty diff --git a/syft/pkg/cataloger/debian/test-fixtures/status/installed-size-4KB b/syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/installed-size-4KB similarity index 100% rename from syft/pkg/cataloger/debian/test-fixtures/status/installed-size-4KB rename to syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/installed-size-4KB diff --git a/syft/pkg/cataloger/debian/test-fixtures/status/libpam-runtime b/syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/libpam-runtime similarity index 100% rename from syft/pkg/cataloger/debian/test-fixtures/status/libpam-runtime rename to syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/libpam-runtime diff --git a/syft/pkg/cataloger/debian/test-fixtures/status/multiple b/syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/multiple similarity index 100% rename from syft/pkg/cataloger/debian/test-fixtures/status/multiple rename to syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/multiple diff --git a/syft/pkg/cataloger/debian/test-fixtures/status/single b/syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/single similarity index 100% rename from syft/pkg/cataloger/debian/test-fixtures/status/single rename to syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/single diff --git a/syft/pkg/cataloger/internal/dependency/catalog_decorator.go b/syft/pkg/cataloger/internal/dependency/catalog_decorator.go index f758aa436d5..917197a3760 100644 --- a/syft/pkg/cataloger/internal/dependency/catalog_decorator.go +++ b/syft/pkg/cataloger/internal/dependency/catalog_decorator.go @@ -2,6 +2,7 @@ package dependency import ( "context" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" diff --git a/syft/pkg/cataloger/internal/dependency/catalog_decorator_test.go b/syft/pkg/cataloger/internal/dependency/catalog_decorator_test.go new file mode 100644 index 00000000000..a4a9af54c95 --- /dev/null +++ b/syft/pkg/cataloger/internal/dependency/catalog_decorator_test.go @@ -0,0 +1,106 @@ +package dependency + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" +) + +var _ pkg.Cataloger = (*catalogerDecorator)(nil) + +type mockCataloger struct { + pkgs []pkg.Package + rels []artifact.Relationship + err error +} + +func (m mockCataloger) Name() string { + return "mock" +} + +func (m mockCataloger) Catalog(_ context.Context, _ file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { + return m.pkgs, m.rels, m.err +} + +func Test_catalogerDecorator_Catalog(t *testing.T) { + a := pkg.Package{ + Name: "a", + } + + b := pkg.Package{ + Name: "b", + } + + c := pkg.Package{ + Name: "c", + } + + subjects := []pkg.Package{a, b, c} + + for _, p := range subjects { + p.SetID() + } + + tests := []struct { + name string + prosumer Prosumer + cataloger pkg.Cataloger + wantPkgCount int + wantRelCount int + wantErr assert.ErrorAssertionFunc + }{ + { + name: "happy path preserves decorated values", + prosumer: newMockProsumer().WithProvides(b, "b-resource").WithRequires(c, "b-resource"), + cataloger: mockCataloger{ + pkgs: []pkg.Package{a, b, c}, + rels: []artifact.Relationship{ + { + From: a, + To: b, + Type: artifact.DependencyOfRelationship, + }, + }, + }, + wantPkgCount: 3, + wantRelCount: 2, // original + new + }, + { + name: "error from cataloger is propagated", + prosumer: newMockProsumer().WithProvides(b, "b-resource").WithRequires(c, "b-resource"), + cataloger: mockCataloger{ + err: errors.New("surprise!"), + pkgs: []pkg.Package{a, b, c}, + rels: []artifact.Relationship{ + { + From: a, + To: b, + Type: artifact.DependencyOfRelationship, + }, + }, + }, + wantPkgCount: 0, + wantRelCount: 0, + wantErr: assert.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == nil { + tt.wantErr = assert.NoError + } + + gotPkgs, gotRels, err := DecorateCatalogerWithRelationships(tt.cataloger, tt.prosumer).Catalog(context.Background(), nil) + + tt.wantErr(t, err) + assert.Len(t, gotPkgs, tt.wantPkgCount) + assert.Len(t, gotRels, tt.wantRelCount) + }) + } +} diff --git a/syft/pkg/cataloger/internal/dependency/resolver.go b/syft/pkg/cataloger/internal/dependency/resolver.go index 4204082589d..808037b1d67 100644 --- a/syft/pkg/cataloger/internal/dependency/resolver.go +++ b/syft/pkg/cataloger/internal/dependency/resolver.go @@ -1,6 +1,10 @@ package dependency import ( + "sort" + + "github.com/scylladb/go-set/strset" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" ) @@ -26,25 +30,44 @@ func (r RelationshipResolver) Resolve(pkgs []pkg.Package) (relationships []artif lookup := make(map[string][]pkg.Package) for _, p := range pkgs { - for _, key := range r.prosumer.Provides(p) { + for _, key := range deduplicate(r.prosumer.Provides(p)) { lookup[key] = append(lookup[key], p) } } + seen := strset.New() for _, p := range pkgs { - for _, requirement := range r.prosumer.Requires(p) { + for _, requirement := range deduplicate(r.prosumer.Requires(p)) { for _, depPkg := range lookup[requirement] { - relationships = append(relationships, artifact.Relationship{ - From: depPkg, - To: p, - Type: artifact.DependencyOfRelationship, - }) + // prevent creating duplicate relationships + pairKey := string(depPkg.ID()) + "-" + string(p.ID()) + if seen.Has(pairKey) { + continue + } + + relationships = append(relationships, + artifact.Relationship{ + From: depPkg, + To: p, + Type: artifact.DependencyOfRelationship, + }, + ) + + seen.Add(pairKey) } } } return relationships } +func deduplicate(ss []string) []string { + set := strset.New(ss...) + // note: this must be a stable function + list := set.List() + sort.Strings(list) + return list +} + func Resolve(pkgs []pkg.Package, prosumer Prosumer) []artifact.Relationship { return NewRelationshipResolver(prosumer).Resolve(pkgs) } diff --git a/syft/pkg/cataloger/internal/dependency/resolver_test.go b/syft/pkg/cataloger/internal/dependency/resolver_test.go new file mode 100644 index 00000000000..b19eddd5d1c --- /dev/null +++ b/syft/pkg/cataloger/internal/dependency/resolver_test.go @@ -0,0 +1,128 @@ +package dependency + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/pkg" +) + +func TestRelationshipResolver_Resolve(t *testing.T) { + a := pkg.Package{ + Name: "a", + } + + b := pkg.Package{ + Name: "b", + } + + c := pkg.Package{ + Name: "c", + } + + subjects := []pkg.Package{a, b, c} + + for _, p := range subjects { + p.SetID() + } + + tests := []struct { + name string + prosumer Prosumer + want map[string][]string + }{ + { + name: "find relationships between packages", + prosumer: newMockProsumer(). + WithProvides(a /* provides */, "a-resource"). + WithRequires(b /* requires */, "a-resource"), + want: map[string][]string{ + "b": /* depends on */ {"a"}, + }, + }, + { + name: "deduplicates provider keys", + prosumer: newMockProsumer(). + WithProvides(a /* provides */, "a-resource", "a-resource", "a-resource"). + WithRequires(b /* requires */, "a-resource", "a-resource", "a-resource"), + want: map[string][]string{ + "b": /* depends on */ {"a"}, + // note: we're NOT seeing: + // "b": /* depends on */ {"a", "a", "a"}, + }, + }, + { + name: "deduplicates crafted relationships", + prosumer: newMockProsumer(). + WithProvides(a /* provides */, "a1-resource", "a2-resource", "a3-resource"). + WithRequires(b /* requires */, "a1-resource", "a2-resource"), + want: map[string][]string{ + "b": /* depends on */ {"a"}, + // note: we're NOT seeing: + // "b": /* depends on */ {"a", "a"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + relationships := NewRelationshipResolver(tt.prosumer).Resolve(subjects) + if d := cmp.Diff(tt.want, abstractRelationships(t, relationships)); d != "" { + t.Errorf("unexpected relationships (-want +got):\n%s", d) + } + }) + } +} + +type mockProsumer struct { + provides map[string][]string + requires map[string][]string +} + +func newMockProsumer() *mockProsumer { + return &mockProsumer{ + provides: make(map[string][]string), + requires: make(map[string][]string), + } +} + +func (m *mockProsumer) WithProvides(p pkg.Package, provides ...string) *mockProsumer { + m.provides[p.Name] = append(m.provides[p.Name], provides...) + return m +} + +func (m *mockProsumer) WithRequires(p pkg.Package, requires ...string) *mockProsumer { + m.requires[p.Name] = append(m.requires[p.Name], requires...) + return m +} + +func (m mockProsumer) Provides(p pkg.Package) []string { + return m.provides[p.Name] +} + +func (m mockProsumer) Requires(p pkg.Package) []string { + return m.requires[p.Name] + +} + +func abstractRelationships(t testing.TB, relationships []artifact.Relationship) map[string][]string { + t.Helper() + + abstracted := make(map[string][]string) + for _, relationship := range relationships { + fromPkg, ok := relationship.From.(pkg.Package) + if !ok { + continue + } + toPkg, ok := relationship.To.(pkg.Package) + if !ok { + continue + } + + // we build this backwards since we use DependencyOfRelationship instead of DependsOn + abstracted[toPkg.Name] = append(abstracted[toPkg.Name], fromPkg.Name) + } + + return abstracted +} From 8b0875ec16a95999e58105eae0444dc598efd19a Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Wed, 8 May 2024 09:35:21 -0400 Subject: [PATCH 03/11] replace cataloger decorator with generic processor Signed-off-by: Alex Goodman --- syft/pkg/cataloger/arch/cataloger.go | 8 +- syft/pkg/cataloger/debian/cataloger.go | 12 +- syft/pkg/cataloger/generic/cataloger.go | 29 +++-- syft/pkg/cataloger/generic/cataloger_test.go | 4 +- .../internal/dependency/catalog_decorator.go | 32 ------ .../dependency/catalog_decorator_test.go | 106 ------------------ .../cataloger/internal/dependency/resolver.go | 8 ++ .../internal/dependency/resolver_test.go | 78 +++++++++++++ 8 files changed, 118 insertions(+), 159 deletions(-) delete mode 100644 syft/pkg/cataloger/internal/dependency/catalog_decorator.go delete mode 100644 syft/pkg/cataloger/internal/dependency/catalog_decorator_test.go diff --git a/syft/pkg/cataloger/arch/cataloger.go b/syft/pkg/cataloger/arch/cataloger.go index b7fcc587d18..04ed5c41bbb 100644 --- a/syft/pkg/cataloger/arch/cataloger.go +++ b/syft/pkg/cataloger/arch/cataloger.go @@ -11,9 +11,7 @@ import ( // NewDBCataloger returns a new cataloger object initialized for arch linux pacman database flat-file stores. func NewDBCataloger() pkg.Cataloger { - return dependency.DecorateCatalogerWithRelationships( - generic.NewCataloger("alpm-db-cataloger"). - WithParserByGlobs(parseAlpmDB, pkg.AlpmDBGlob), - newDBProsumer(), - ) + return generic.NewCataloger("alpm-db-cataloger"). + WithParserByGlobs(parseAlpmDB, pkg.AlpmDBGlob). + WithProcessors(dependency.Processor(newDBProsumer())) } diff --git a/syft/pkg/cataloger/debian/cataloger.go b/syft/pkg/cataloger/debian/cataloger.go index 41efe70dc83..2644b5fbda2 100644 --- a/syft/pkg/cataloger/debian/cataloger.go +++ b/syft/pkg/cataloger/debian/cataloger.go @@ -11,11 +11,9 @@ import ( // NewDBCataloger returns a new Deb package cataloger capable of parsing DPKG status DB flat-file stores. func NewDBCataloger() pkg.Cataloger { - return dependency.DecorateCatalogerWithRelationships( - generic.NewCataloger("dpkg-db-cataloger"). - // note: these globs have been intentionally split up in order to improve search performance, - // please do NOT combine into: "**/var/lib/dpkg/{status,status.d/*}" - WithParserByGlobs(parseDpkgDB, "**/var/lib/dpkg/status", "**/var/lib/dpkg/status.d/*", "**/lib/opkg/info/*.control", "**/lib/opkg/status"), - newDBProsumer(), - ) + return generic.NewCataloger("dpkg-db-cataloger"). + // note: these globs have been intentionally split up in order to improve search performance, + // please do NOT combine into: "**/var/lib/dpkg/{status,status.d/*}" + WithParserByGlobs(parseDpkgDB, "**/var/lib/dpkg/status", "**/var/lib/dpkg/status.d/*", "**/lib/opkg/info/*.control", "**/lib/opkg/status"). + WithProcessors(dependency.Processor(newDBProsumer())) } diff --git a/syft/pkg/cataloger/generic/cataloger.go b/syft/pkg/cataloger/generic/cataloger.go index 49052ec88e0..8009594ddca 100644 --- a/syft/pkg/cataloger/generic/cataloger.go +++ b/syft/pkg/cataloger/generic/cataloger.go @@ -12,7 +12,9 @@ import ( "github.com/anchore/syft/syft/pkg" ) -type processor func(resolver file.Resolver, env Environment) []request +type Processor func([]pkg.Package, []artifact.Relationship, error) ([]pkg.Package, []artifact.Relationship, error) + +type requester func(resolver file.Resolver, env Environment) []request type request struct { file.Location @@ -22,12 +24,13 @@ type request struct { // Cataloger implements the Catalog interface and is responsible for dispatching the proper parser function for // a given path or glob pattern. This is intended to be reusable across many package cataloger types. type Cataloger struct { - processor []processor + processors []Processor + requesters []requester upstreamCataloger string } func (c *Cataloger) WithParserByGlobs(parser Parser, globs ...string) *Cataloger { - c.processor = append(c.processor, + c.requesters = append(c.requesters, func(resolver file.Resolver, _ Environment) []request { var requests []request for _, g := range globs { @@ -47,7 +50,7 @@ func (c *Cataloger) WithParserByGlobs(parser Parser, globs ...string) *Cataloger } func (c *Cataloger) WithParserByMimeTypes(parser Parser, types ...string) *Cataloger { - c.processor = append(c.processor, + c.requesters = append(c.requesters, func(resolver file.Resolver, _ Environment) []request { var requests []request log.WithFields("mimetypes", types).Trace("searching for paths matching mimetype") @@ -64,7 +67,7 @@ func (c *Cataloger) WithParserByMimeTypes(parser Parser, types ...string) *Catal } func (c *Cataloger) WithParserByPath(parser Parser, paths ...string) *Cataloger { - c.processor = append(c.processor, + c.requesters = append(c.requesters, func(resolver file.Resolver, _ Environment) []request { var requests []request for _, p := range paths { @@ -83,6 +86,11 @@ func (c *Cataloger) WithParserByPath(parser Parser, paths ...string) *Cataloger return c } +func (c *Cataloger) WithProcessors(processors ...Processor) *Cataloger { + c.processors = append(c.processors, processors...) + return c +} + func makeRequests(parser Parser, locations []file.Location) []request { var requests []request for _, l := range locations { @@ -135,7 +143,14 @@ func (c *Cataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg. relationships = append(relationships, discoveredRelationships...) } - return packages, relationships, nil + return c.process(packages, relationships, nil) +} + +func (c *Cataloger) process(pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) { + for _, proc := range c.processors { + pkgs, rels, err = proc(pkgs, rels, err) + } + return pkgs, rels, err } func invokeParser(ctx context.Context, resolver file.Resolver, location file.Location, logger logger.Logger, parser Parser, env *Environment) ([]pkg.Package, []artifact.Relationship, error) { @@ -158,7 +173,7 @@ func invokeParser(ctx context.Context, resolver file.Resolver, location file.Loc // selectFiles takes a set of file trees and resolves and file references of interest for future cataloging func (c *Cataloger) selectFiles(resolver file.Resolver) []request { var requests []request - for _, proc := range c.processor { + for _, proc := range c.requesters { requests = append(requests, proc(resolver, Environment{})...) } return requests diff --git a/syft/pkg/cataloger/generic/cataloger_test.go b/syft/pkg/cataloger/generic/cataloger_test.go index c22113432c3..e971b4459f4 100644 --- a/syft/pkg/cataloger/generic/cataloger_test.go +++ b/syft/pkg/cataloger/generic/cataloger_test.go @@ -159,7 +159,7 @@ func TestClosesFileOnParserPanic(t *testing.T) { resolver := newSpyReturningFileResolver(&spy, "test-fixtures/another-path.txt") ctx := context.TODO() - processors := []processor{ + processors := []requester{ func(resolver file.Resolver, env Environment) []request { return []request{ { @@ -178,7 +178,7 @@ func TestClosesFileOnParserPanic(t *testing.T) { } c := Cataloger{ - processor: processors, + requesters: processors, upstreamCataloger: "unit-test-cataloger", } diff --git a/syft/pkg/cataloger/internal/dependency/catalog_decorator.go b/syft/pkg/cataloger/internal/dependency/catalog_decorator.go deleted file mode 100644 index 917197a3760..00000000000 --- a/syft/pkg/cataloger/internal/dependency/catalog_decorator.go +++ /dev/null @@ -1,32 +0,0 @@ -package dependency - -import ( - "context" - - "github.com/anchore/syft/syft/artifact" - "github.com/anchore/syft/syft/file" - "github.com/anchore/syft/syft/pkg" -) - -type catalogerDecorator struct { - pkg.Cataloger - resolver RelationshipResolver -} - -func DecorateCatalogerWithRelationships(cataloger pkg.Cataloger, prosumer Prosumer) pkg.Cataloger { - return &catalogerDecorator{ - Cataloger: cataloger, - resolver: NewRelationshipResolver(prosumer), - } -} - -func (c catalogerDecorator) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { - pkgs, rels, err := c.Cataloger.Catalog(ctx, resolver) - if err != nil { - return nil, nil, err - } - - rels = append(rels, c.resolver.Resolve(pkgs)...) - - return pkgs, rels, nil -} diff --git a/syft/pkg/cataloger/internal/dependency/catalog_decorator_test.go b/syft/pkg/cataloger/internal/dependency/catalog_decorator_test.go deleted file mode 100644 index a4a9af54c95..00000000000 --- a/syft/pkg/cataloger/internal/dependency/catalog_decorator_test.go +++ /dev/null @@ -1,106 +0,0 @@ -package dependency - -import ( - "context" - "errors" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/anchore/syft/syft/artifact" - "github.com/anchore/syft/syft/file" - "github.com/anchore/syft/syft/pkg" -) - -var _ pkg.Cataloger = (*catalogerDecorator)(nil) - -type mockCataloger struct { - pkgs []pkg.Package - rels []artifact.Relationship - err error -} - -func (m mockCataloger) Name() string { - return "mock" -} - -func (m mockCataloger) Catalog(_ context.Context, _ file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { - return m.pkgs, m.rels, m.err -} - -func Test_catalogerDecorator_Catalog(t *testing.T) { - a := pkg.Package{ - Name: "a", - } - - b := pkg.Package{ - Name: "b", - } - - c := pkg.Package{ - Name: "c", - } - - subjects := []pkg.Package{a, b, c} - - for _, p := range subjects { - p.SetID() - } - - tests := []struct { - name string - prosumer Prosumer - cataloger pkg.Cataloger - wantPkgCount int - wantRelCount int - wantErr assert.ErrorAssertionFunc - }{ - { - name: "happy path preserves decorated values", - prosumer: newMockProsumer().WithProvides(b, "b-resource").WithRequires(c, "b-resource"), - cataloger: mockCataloger{ - pkgs: []pkg.Package{a, b, c}, - rels: []artifact.Relationship{ - { - From: a, - To: b, - Type: artifact.DependencyOfRelationship, - }, - }, - }, - wantPkgCount: 3, - wantRelCount: 2, // original + new - }, - { - name: "error from cataloger is propagated", - prosumer: newMockProsumer().WithProvides(b, "b-resource").WithRequires(c, "b-resource"), - cataloger: mockCataloger{ - err: errors.New("surprise!"), - pkgs: []pkg.Package{a, b, c}, - rels: []artifact.Relationship{ - { - From: a, - To: b, - Type: artifact.DependencyOfRelationship, - }, - }, - }, - wantPkgCount: 0, - wantRelCount: 0, - wantErr: assert.Error, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.wantErr == nil { - tt.wantErr = assert.NoError - } - - gotPkgs, gotRels, err := DecorateCatalogerWithRelationships(tt.cataloger, tt.prosumer).Catalog(context.Background(), nil) - - tt.wantErr(t, err) - assert.Len(t, gotPkgs, tt.wantPkgCount) - assert.Len(t, gotRels, tt.wantRelCount) - }) - } -} diff --git a/syft/pkg/cataloger/internal/dependency/resolver.go b/syft/pkg/cataloger/internal/dependency/resolver.go index 808037b1d67..a965a9f738b 100644 --- a/syft/pkg/cataloger/internal/dependency/resolver.go +++ b/syft/pkg/cataloger/internal/dependency/resolver.go @@ -7,6 +7,7 @@ import ( "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" ) // Prosumer is a producer and consumer, in this context, for packages that provide resources and require resources. @@ -25,6 +26,13 @@ func NewRelationshipResolver(p Prosumer) RelationshipResolver { } } +func Processor(p Prosumer) generic.Processor { + return func(pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) { + rels = append(rels, NewRelationshipResolver(p).Resolve(pkgs)...) + return pkgs, rels, err + } +} + // Resolve will create relationships between packages based on the "Depends" and "Provides" specifications from the given packages. func (r RelationshipResolver) Resolve(pkgs []pkg.Package) (relationships []artifact.Relationship) { lookup := make(map[string][]pkg.Package) diff --git a/syft/pkg/cataloger/internal/dependency/resolver_test.go b/syft/pkg/cataloger/internal/dependency/resolver_test.go index b19eddd5d1c..ee40151642e 100644 --- a/syft/pkg/cataloger/internal/dependency/resolver_test.go +++ b/syft/pkg/cataloger/internal/dependency/resolver_test.go @@ -1,9 +1,11 @@ package dependency import ( + "errors" "testing" "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" @@ -126,3 +128,79 @@ func abstractRelationships(t testing.TB, relationships []artifact.Relationship) return abstracted } + +func Test_Processor(t *testing.T) { + a := pkg.Package{ + Name: "a", + } + + b := pkg.Package{ + Name: "b", + } + + c := pkg.Package{ + Name: "c", + } + + subjects := []pkg.Package{a, b, c} + + for _, p := range subjects { + p.SetID() + } + + tests := []struct { + name string + prosumer Prosumer + pkgs []pkg.Package + rels []artifact.Relationship + err error + wantPkgCount int + wantRelCount int + wantErr assert.ErrorAssertionFunc + }{ + { + name: "happy path preserves decorated values", + prosumer: newMockProsumer().WithProvides(b, "b-resource").WithRequires(c, "b-resource"), + pkgs: []pkg.Package{a, b, c}, + rels: []artifact.Relationship{ + { + From: a, + To: b, + Type: artifact.DependencyOfRelationship, + }, + }, + + wantPkgCount: 3, + wantRelCount: 2, // original + new + }, + { + name: "error from cataloger is propagated", + prosumer: newMockProsumer().WithProvides(b, "b-resource").WithRequires(c, "b-resource"), + err: errors.New("surprise!"), + pkgs: []pkg.Package{a, b, c}, + rels: []artifact.Relationship{ + { + From: a, + To: b, + Type: artifact.DependencyOfRelationship, + }, + }, + wantPkgCount: 3, + wantRelCount: 2, // original + new + wantErr: assert.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == nil { + tt.wantErr = assert.NoError + } + + gotPkgs, gotRels, err := Processor(tt.prosumer)(tt.pkgs, tt.rels, tt.err) + + tt.wantErr(t, err) + assert.Len(t, gotPkgs, tt.wantPkgCount) + assert.Len(t, gotRels, tt.wantRelCount) + }) + } +} From ff20b73d1ab43eaaa4f0f4423669ce9542d27c81 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 13 May 2024 17:59:21 -0400 Subject: [PATCH 04/11] refactor resolver to be a single function Signed-off-by: Alex Goodman --- .../cataloger/internal/dependency/resolver.go | 72 ++++++++++----- .../internal/dependency/resolver_test.go | 87 ++++++++++--------- 2 files changed, 95 insertions(+), 64 deletions(-) diff --git a/syft/pkg/cataloger/internal/dependency/resolver.go b/syft/pkg/cataloger/internal/dependency/resolver.go index a965a9f738b..66ebc034eae 100644 --- a/syft/pkg/cataloger/internal/dependency/resolver.go +++ b/syft/pkg/cataloger/internal/dependency/resolver.go @@ -10,53 +10,83 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/generic" ) -// Prosumer is a producer and consumer, in this context, for packages that provide resources and require resources. -type Prosumer interface { - Provides(pkg.Package) []string - Requires(pkg.Package) []string +// Specification holds strings that indicate abstract resources that a package provides for other packages and +// requires for itself. These strings can represent anything from file paths, package names, or any other concept +// that is useful for dependency resolution within that packing ecosystem. +type Specification struct { + // Provides holds a list of abstract resources that this package provides for other packages. + Provides []string + + // Requires holds a list of abstract resources that this package requires from other packages. + Requires []string } +// Specifier is a function that takes a package and extracts a Specification, describing resources +// the package provides and needs. +type Specifier func(pkg.Package) Specification + +// RelationshipResolver uses a Specifier to resolve relationships between packages based on generic dependency claims +// and provider claims from any given package. type RelationshipResolver struct { - prosumer Prosumer + specifier Specifier } -func NewRelationshipResolver(p Prosumer) RelationshipResolver { +func NewRelationshipResolver(s Specifier) RelationshipResolver { return RelationshipResolver{ - prosumer: p, + specifier: s, } } -func Processor(p Prosumer) generic.Processor { +// Processor returns a generic processor that will resolve relationships between packages based on the dependency claims. +func Processor(s Specifier) generic.Processor { return func(pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) { - rels = append(rels, NewRelationshipResolver(p).Resolve(pkgs)...) + // we can't move forward unless all package IDs have been set + for idx, p := range pkgs { + id := p.ID() + if id == "" { + p.SetID() + pkgs[idx] = p + } + } + + rels = append(rels, NewRelationshipResolver(s).Resolve(pkgs)...) return pkgs, rels, err } } -// Resolve will create relationships between packages based on the "Depends" and "Provides" specifications from the given packages. +// Resolve will create relationships between packages based on the dependency claims of each package. func (r RelationshipResolver) Resolve(pkgs []pkg.Package) (relationships []artifact.Relationship) { - lookup := make(map[string][]pkg.Package) + pkgsProvidingResource := make(map[string][]artifact.ID) + + pkgsByID := make(map[artifact.ID]pkg.Package) + specsByPkg := make(map[artifact.ID]Specification) for _, p := range pkgs { - for _, key := range deduplicate(r.prosumer.Provides(p)) { - lookup[key] = append(lookup[key], p) + id := p.ID() + pkgsByID[id] = p + specsByPkg[id] = r.specifier(p) + for _, resource := range deduplicate(r.specifier(p).Provides) { + pkgsProvidingResource[resource] = append(pkgsProvidingResource[resource], id) } } seen := strset.New() - for _, p := range pkgs { - for _, requirement := range deduplicate(r.prosumer.Requires(p)) { - for _, depPkg := range lookup[requirement] { + for _, dependantPkg := range pkgs { + spec := specsByPkg[dependantPkg.ID()] + for _, resource := range deduplicate(spec.Requires) { + for _, providingPkgID := range pkgsProvidingResource[resource] { // prevent creating duplicate relationships - pairKey := string(depPkg.ID()) + "-" + string(p.ID()) + pairKey := string(providingPkgID) + "-" + string(dependantPkg.ID()) if seen.Has(pairKey) { continue } + providingPkg := pkgsByID[providingPkgID] + relationships = append(relationships, artifact.Relationship{ - From: depPkg, - To: p, + From: providingPkg, + To: dependantPkg, Type: artifact.DependencyOfRelationship, }, ) @@ -75,7 +105,3 @@ func deduplicate(ss []string) []string { sort.Strings(list) return list } - -func Resolve(pkgs []pkg.Package, prosumer Prosumer) []artifact.Relationship { - return NewRelationshipResolver(prosumer).Resolve(pkgs) -} diff --git a/syft/pkg/cataloger/internal/dependency/resolver_test.go b/syft/pkg/cataloger/internal/dependency/resolver_test.go index ee40151642e..2549b996670 100644 --- a/syft/pkg/cataloger/internal/dependency/resolver_test.go +++ b/syft/pkg/cataloger/internal/dependency/resolver_test.go @@ -15,40 +15,41 @@ func TestRelationshipResolver_Resolve(t *testing.T) { a := pkg.Package{ Name: "a", } + a.SetID() b := pkg.Package{ Name: "b", } + b.SetID() c := pkg.Package{ Name: "c", } + c.SetID() subjects := []pkg.Package{a, b, c} - for _, p := range subjects { - p.SetID() - } - tests := []struct { - name string - prosumer Prosumer - want map[string][]string + name string + s Specifier + want map[string][]string }{ { name: "find relationships between packages", - prosumer: newMockProsumer(). + s: newSpecifierBuilder(). WithProvides(a /* provides */, "a-resource"). - WithRequires(b /* requires */, "a-resource"), + WithRequires(b /* requires */, "a-resource"). + Specifier(), want: map[string][]string{ "b": /* depends on */ {"a"}, }, }, { name: "deduplicates provider keys", - prosumer: newMockProsumer(). + s: newSpecifierBuilder(). WithProvides(a /* provides */, "a-resource", "a-resource", "a-resource"). - WithRequires(b /* requires */, "a-resource", "a-resource", "a-resource"), + WithRequires(b /* requires */, "a-resource", "a-resource", "a-resource"). + Specifier(), want: map[string][]string{ "b": /* depends on */ {"a"}, // note: we're NOT seeing: @@ -57,9 +58,10 @@ func TestRelationshipResolver_Resolve(t *testing.T) { }, { name: "deduplicates crafted relationships", - prosumer: newMockProsumer(). + s: newSpecifierBuilder(). WithProvides(a /* provides */, "a1-resource", "a2-resource", "a3-resource"). - WithRequires(b /* requires */, "a1-resource", "a2-resource"), + WithRequires(b /* requires */, "a1-resource", "a2-resource"). + Specifier(), want: map[string][]string{ "b": /* depends on */ {"a"}, // note: we're NOT seeing: @@ -69,7 +71,7 @@ func TestRelationshipResolver_Resolve(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - relationships := NewRelationshipResolver(tt.prosumer).Resolve(subjects) + relationships := NewRelationshipResolver(tt.s).Resolve(subjects) if d := cmp.Diff(tt.want, abstractRelationships(t, relationships)); d != "" { t.Errorf("unexpected relationships (-want +got):\n%s", d) } @@ -77,35 +79,35 @@ func TestRelationshipResolver_Resolve(t *testing.T) { } } -type mockProsumer struct { +type specifierBuilder struct { provides map[string][]string requires map[string][]string } -func newMockProsumer() *mockProsumer { - return &mockProsumer{ +func newSpecifierBuilder() *specifierBuilder { + return &specifierBuilder{ provides: make(map[string][]string), requires: make(map[string][]string), } } -func (m *mockProsumer) WithProvides(p pkg.Package, provides ...string) *mockProsumer { +func (m *specifierBuilder) WithProvides(p pkg.Package, provides ...string) *specifierBuilder { m.provides[p.Name] = append(m.provides[p.Name], provides...) return m } -func (m *mockProsumer) WithRequires(p pkg.Package, requires ...string) *mockProsumer { +func (m *specifierBuilder) WithRequires(p pkg.Package, requires ...string) *specifierBuilder { m.requires[p.Name] = append(m.requires[p.Name], requires...) return m } -func (m mockProsumer) Provides(p pkg.Package) []string { - return m.provides[p.Name] -} - -func (m mockProsumer) Requires(p pkg.Package) []string { - return m.requires[p.Name] - +func (m specifierBuilder) Specifier() Specifier { + return func(p pkg.Package) Specification { + return Specification{ + Provides: m.provides[p.Name], + Requires: m.requires[p.Name], + } + } } func abstractRelationships(t testing.TB, relationships []artifact.Relationship) map[string][]string { @@ -133,24 +135,21 @@ func Test_Processor(t *testing.T) { a := pkg.Package{ Name: "a", } + a.SetID() b := pkg.Package{ Name: "b", } + b.SetID() c := pkg.Package{ Name: "c", } - - subjects := []pkg.Package{a, b, c} - - for _, p := range subjects { - p.SetID() - } + c.SetID() tests := []struct { name string - prosumer Prosumer + sp Specifier pkgs []pkg.Package rels []artifact.Relationship err error @@ -159,9 +158,12 @@ func Test_Processor(t *testing.T) { wantErr assert.ErrorAssertionFunc }{ { - name: "happy path preserves decorated values", - prosumer: newMockProsumer().WithProvides(b, "b-resource").WithRequires(c, "b-resource"), - pkgs: []pkg.Package{a, b, c}, + name: "happy path preserves decorated values", + sp: newSpecifierBuilder(). + WithProvides(b, "b-resource"). + WithRequires(c, "b-resource"). + Specifier(), + pkgs: []pkg.Package{a, b, c}, rels: []artifact.Relationship{ { From: a, @@ -174,10 +176,13 @@ func Test_Processor(t *testing.T) { wantRelCount: 2, // original + new }, { - name: "error from cataloger is propagated", - prosumer: newMockProsumer().WithProvides(b, "b-resource").WithRequires(c, "b-resource"), - err: errors.New("surprise!"), - pkgs: []pkg.Package{a, b, c}, + name: "error from cataloger is propagated", + sp: newSpecifierBuilder(). + WithProvides(b, "b-resource"). + WithRequires(c, "b-resource"). + Specifier(), + err: errors.New("surprise!"), + pkgs: []pkg.Package{a, b, c}, rels: []artifact.Relationship{ { From: a, @@ -196,7 +201,7 @@ func Test_Processor(t *testing.T) { tt.wantErr = assert.NoError } - gotPkgs, gotRels, err := Processor(tt.prosumer)(tt.pkgs, tt.rels, tt.err) + gotPkgs, gotRels, err := Processor(tt.sp)(tt.pkgs, tt.rels, tt.err) tt.wantErr(t, err) assert.Len(t, gotPkgs, tt.wantPkgCount) From 6097b2043d351a414d77512863a5a89490fa2888 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 13 May 2024 18:00:08 -0400 Subject: [PATCH 05/11] use common dependency specifier for debian Signed-off-by: Alex Goodman --- syft/pkg/cataloger/debian/cataloger.go | 2 +- syft/pkg/cataloger/debian/cataloger_test.go | 62 +++++++++++ .../debian/{prosumer.go => dependency.go} | 52 ++++----- syft/pkg/cataloger/debian/dependency_test.go | 101 ++++++++++++++++++ .../cataloger/debian/parse_dpkg_db_test.go | 94 ---------------- 5 files changed, 185 insertions(+), 126 deletions(-) rename syft/pkg/cataloger/debian/{prosumer.go => dependency.go} (53%) create mode 100644 syft/pkg/cataloger/debian/dependency_test.go diff --git a/syft/pkg/cataloger/debian/cataloger.go b/syft/pkg/cataloger/debian/cataloger.go index 2644b5fbda2..d6876981c45 100644 --- a/syft/pkg/cataloger/debian/cataloger.go +++ b/syft/pkg/cataloger/debian/cataloger.go @@ -15,5 +15,5 @@ func NewDBCataloger() pkg.Cataloger { // note: these globs have been intentionally split up in order to improve search performance, // please do NOT combine into: "**/var/lib/dpkg/{status,status.d/*}" WithParserByGlobs(parseDpkgDB, "**/var/lib/dpkg/status", "**/var/lib/dpkg/status.d/*", "**/lib/opkg/info/*.control", "**/lib/opkg/status"). - WithProcessors(dependency.Processor(newDBProsumer())) + WithProcessors(dependency.Processor(dbEntryDependencySpecifier)) } diff --git a/syft/pkg/cataloger/debian/cataloger_test.go b/syft/pkg/cataloger/debian/cataloger_test.go index 5d3addcbf0d..0f8a70a9af4 100644 --- a/syft/pkg/cataloger/debian/cataloger_test.go +++ b/syft/pkg/cataloger/debian/cataloger_test.go @@ -1,8 +1,12 @@ package debian import ( + "context" "testing" + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" @@ -160,6 +164,64 @@ func TestDpkgCataloger(t *testing.T) { } } +func Test_CatalogerRelationships(t *testing.T) { + tests := []struct { + name string + fixture string + wantRelationships map[string][]string + }{ + { + name: "relationships for coreutils", + fixture: "test-fixtures/var/lib/dpkg/status.d/coreutils-relationships", + wantRelationships: map[string][]string{ + "coreutils": {"libacl1", "libattr1", "libc6", "libgmp10", "libselinux1"}, + "libacl1": {"libc6"}, + "libattr1": {"libc6"}, + "libc6": {"libgcc-s1"}, + "libgcc-s1": {"gcc-12-base", "libc6"}, + "libgmp10": {"libc6"}, + "libpcre2-8-0": {"libc6"}, + "libselinux1": {"libc6", "libpcre2-8-0"}, + }, + }, + { + name: "relationships from dpkg example docs", + fixture: "test-fixtures/var/lib/dpkg/status.d/doc-examples", + wantRelationships: map[string][]string{ + "made-up-package-1": {"gnumach-dev", "hurd-dev", "kernel-headers-2.2.10"}, + "made-up-package-2": {"liblua5.1-dev", "libluajit5.1-dev"}, + "made-up-package-3": {"bar", "foo"}, + // note that the "made-up-package-4" depends on "made-up-package-5" but not via the direct + // package name, but through the "provides" virtual package name "virtual-package-5". + "made-up-package-4": {"made-up-package-5"}, + // note that though there is a "default-mta | mail-transport-agent | not-installed" + // dependency choice we raise up the packages that are installed for every choice. + // In this case that means that "default-mta" and "mail-transport-agent". + "mutt": {"default-mta", "libc6", "mail-transport-agent"}, + }, + }, + { + name: "relationships for libpam-runtime", + fixture: "test-fixtures/var/lib/dpkg/status.d/libpam-runtime", + wantRelationships: map[string][]string{ + "libpam-runtime": {"cdebconf", "debconf-2.0", "debconf1", "debconf2", "libpam-modules"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pkgs, relationships, err := NewDBCataloger().Catalog(context.Background(), file.NewMockResolverForPaths(tt.fixture)) + require.NotEmpty(t, pkgs) + require.NotEmpty(t, relationships) + require.NoError(t, err) + + if d := cmp.Diff(tt.wantRelationships, abstractRelationships(t, relationships)); d != "" { + t.Errorf("unexpected relationships (-want +got):\n%s", d) + } + }) + } +} + func TestCataloger_Globs(t *testing.T) { tests := []struct { name string diff --git a/syft/pkg/cataloger/debian/prosumer.go b/syft/pkg/cataloger/debian/dependency.go similarity index 53% rename from syft/pkg/cataloger/debian/prosumer.go rename to syft/pkg/cataloger/debian/dependency.go index 3d377e3656e..9fe39899eca 100644 --- a/syft/pkg/cataloger/debian/prosumer.go +++ b/syft/pkg/cataloger/debian/dependency.go @@ -9,43 +9,38 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" ) -var _ dependency.Prosumer = (*dpkgDBEntryProsumer)(nil) +var _ dependency.Specifier = dbEntryDependencySpecifier -type dpkgDBEntryProsumer struct{} - -func newDBProsumer() dpkgDBEntryProsumer { - return dpkgDBEntryProsumer{} -} - -func (ps dpkgDBEntryProsumer) Provides(p pkg.Package) []string { +func dbEntryDependencySpecifier(p pkg.Package) dependency.Specification { meta, ok := p.Metadata.(pkg.DpkgDBEntry) if !ok { - log.Warnf("cataloger failed to extract dpkg 'provides' metadata for package %+v", p.Name) - return nil - } - keys := []string{p.Name} - for _, provides := range meta.Provides { - keys = append(keys, stripVersionSpecifier(provides)) + log.Tracef("cataloger failed to extract dpkg metadata for package %+v", p.Name) + return dependency.Specification{} } - return keys -} - -func (ps dpkgDBEntryProsumer) Requires(p pkg.Package) []string { - meta, ok := p.Metadata.(pkg.DpkgDBEntry) - if !ok { - log.Warnf("cataloger failed to extract dpkg 'requires' metadata for package %+v", p.Name) - return nil + provides := []string{p.Name} + for _, key := range meta.Provides { + if key == "" { + continue + } + provides = append(provides, stripVersionSpecifier(key)) } var allDeps []string allDeps = append(allDeps, meta.Depends...) allDeps = append(allDeps, meta.PreDepends...) - var keys []string + var requires []string for _, depSpecifier := range allDeps { - keys = append(keys, splitPackageChoice(depSpecifier)...) + if depSpecifier == "" { + continue + } + requires = append(requires, splitPackageChoice(depSpecifier)...) + } + + return dependency.Specification{ + Provides: provides, + Requires: requires, } - return keys } func stripVersionSpecifier(s string) string { @@ -56,12 +51,7 @@ func stripVersionSpecifier(s string) string { // default-mta | mail-transport-agent --> default-mta | mail-transport-agent // kernel-headers-2.2.10 [!hurd-i386] --> kernel-headers-2.2.10 - items := internal.SplitAny(s, "[(<>=") - if len(items) == 0 { - return s - } - - return strings.TrimSpace(items[0]) + return strings.TrimSpace(internal.SplitAny(s, "[(<>=")[0]) } func splitPackageChoice(s string) (ret []string) { diff --git a/syft/pkg/cataloger/debian/dependency_test.go b/syft/pkg/cataloger/debian/dependency_test.go new file mode 100644 index 00000000000..5e222574862 --- /dev/null +++ b/syft/pkg/cataloger/debian/dependency_test.go @@ -0,0 +1,101 @@ +package debian + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" +) + +func Test_dbEntryDependencySpecifier(t *testing.T) { + tests := []struct { + name string + p pkg.Package + want dependency.Specification + }{ + { + name: "keeps given values + package name", + p: pkg.Package{ + Name: "package-c", + Metadata: pkg.DpkgDBEntry{ + Provides: []string{"a-thing"}, + Depends: []string{"b-thing"}, + }, + }, + want: dependency.Specification{ + Provides: []string{"package-c", "a-thing"}, + Requires: []string{"b-thing"}, + }, + }, + { + name: "strip version specifiers + split package deps", + p: pkg.Package{ + Name: "package-a", + Metadata: pkg.DpkgDBEntry{ + Provides: []string{"foo [i386]"}, + Depends: []string{"libgmp10 (>= 2:6.2.1+dfsg1)", "default-mta | mail-transport-agent"}, + }, + }, + want: dependency.Specification{ + Provides: []string{"package-a", "foo"}, + Requires: []string{"libgmp10", "default-mta", "mail-transport-agent"}, + }, + }, + { + name: "empty dependency data entries", + p: pkg.Package{ + Name: "package-a", + Metadata: pkg.DpkgDBEntry{ + Provides: []string{""}, + Depends: []string{""}, + }, + }, + want: dependency.Specification{ + Provides: []string{"package-a"}, + Requires: nil, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, dbEntryDependencySpecifier(tt.p)) + }) + } +} + +func Test_stripVersionSpecifier(t *testing.T) { + + tests := []struct { + name string + input string + want string + }{ + { + name: "package name only", + input: "test", + want: "test", + }, + { + name: "with version", + input: "test (1.2.3)", + want: "test", + }, + { + name: "multiple packages", + input: "test | other", + want: "test | other", + }, + { + name: "with architecture specifiers", + input: "test [amd64 i386]", + want: "test", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, stripVersionSpecifier(tt.input)) + }) + } +} diff --git a/syft/pkg/cataloger/debian/parse_dpkg_db_test.go b/syft/pkg/cataloger/debian/parse_dpkg_db_test.go index c411cf52f80..603de91e3a6 100644 --- a/syft/pkg/cataloger/debian/parse_dpkg_db_test.go +++ b/syft/pkg/cataloger/debian/parse_dpkg_db_test.go @@ -2,7 +2,6 @@ package debian import ( "bufio" - "context" "errors" "fmt" "os" @@ -433,99 +432,6 @@ func Test_handleNewKeyValue(t *testing.T) { } } -func Test_stripVersionSpecifier(t *testing.T) { - - tests := []struct { - name string - input string - want string - }{ - { - name: "package name only", - input: "test", - want: "test", - }, - { - name: "with version", - input: "test (1.2.3)", - want: "test", - }, - { - name: "multiple packages", - input: "test | other", - want: "test | other", - }, - { - name: "with architecture specifiers", - input: "test [amd64 i386]", - want: "test", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, stripVersionSpecifier(tt.input)) - }) - } -} - -func Test_associateRelationships(t *testing.T) { - tests := []struct { - name string - fixture string - wantRelationships map[string][]string - }{ - { - name: "relationships for coreutils", - fixture: "test-fixtures/var/lib/dpkg/status.d/coreutils-relationships", - wantRelationships: map[string][]string{ - "coreutils": {"libacl1", "libattr1", "libc6", "libgmp10", "libselinux1"}, - "libacl1": {"libc6"}, - "libattr1": {"libc6"}, - "libc6": {"libgcc-s1"}, - "libgcc-s1": {"gcc-12-base", "libc6"}, - "libgmp10": {"libc6"}, - "libpcre2-8-0": {"libc6"}, - "libselinux1": {"libc6", "libpcre2-8-0"}, - }, - }, - { - name: "relationships from dpkg example docs", - fixture: "test-fixtures/var/lib/dpkg/status.d/doc-examples", - wantRelationships: map[string][]string{ - "made-up-package-1": {"gnumach-dev", "hurd-dev", "kernel-headers-2.2.10"}, - "made-up-package-2": {"liblua5.1-dev", "libluajit5.1-dev"}, - "made-up-package-3": {"bar", "foo"}, - // note that the "made-up-package-4" depends on "made-up-package-5" but not via the direct - // package name, but through the "provides" virtual package name "virtual-package-5". - "made-up-package-4": {"made-up-package-5"}, - // note that though there is a "default-mta | mail-transport-agent | not-installed" - // dependency choice we raise up the packages that are installed for every choice. - // In this case that means that "default-mta" and "mail-transport-agent". - "mutt": {"default-mta", "libc6", "mail-transport-agent"}, - }, - }, - { - name: "relationships for libpam-runtime", - fixture: "test-fixtures/var/lib/dpkg/status.d/libpam-runtime", - wantRelationships: map[string][]string{ - "libpam-runtime": {"cdebconf", "debconf-2.0", "debconf1", "debconf2", "libpam-modules"}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - pkgs, relationships, err := NewDBCataloger().Catalog(context.Background(), file.NewMockResolverForPaths(tt.fixture)) - require.NotEmpty(t, pkgs) - require.NotEmpty(t, relationships) - require.NoError(t, err) - - if d := cmp.Diff(tt.wantRelationships, abstractRelationships(t, relationships)); d != "" { - t.Errorf("unexpected relationships (-want +got):\n%s", d) - } - }) - } -} - func abstractRelationships(t testing.TB, relationships []artifact.Relationship) map[string][]string { t.Helper() From 1cea07790ed610541a182969f7be6f6d871bd019 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 13 May 2024 18:00:23 -0400 Subject: [PATCH 06/11] use common dependency specifier for arch Signed-off-by: Alex Goodman --- syft/pkg/cataloger/arch/cataloger.go | 2 +- syft/pkg/cataloger/arch/dependency.go | 48 ++++++++++ syft/pkg/cataloger/arch/dependency_test.go | 100 +++++++++++++++++++++ syft/pkg/cataloger/arch/prosumer.go | 53 ----------- 4 files changed, 149 insertions(+), 54 deletions(-) create mode 100644 syft/pkg/cataloger/arch/dependency.go create mode 100644 syft/pkg/cataloger/arch/dependency_test.go delete mode 100644 syft/pkg/cataloger/arch/prosumer.go diff --git a/syft/pkg/cataloger/arch/cataloger.go b/syft/pkg/cataloger/arch/cataloger.go index 04ed5c41bbb..75153bdd498 100644 --- a/syft/pkg/cataloger/arch/cataloger.go +++ b/syft/pkg/cataloger/arch/cataloger.go @@ -13,5 +13,5 @@ import ( func NewDBCataloger() pkg.Cataloger { return generic.NewCataloger("alpm-db-cataloger"). WithParserByGlobs(parseAlpmDB, pkg.AlpmDBGlob). - WithProcessors(dependency.Processor(newDBProsumer())) + WithProcessors(dependency.Processor(dbEntryDependencySpecifier)) } diff --git a/syft/pkg/cataloger/arch/dependency.go b/syft/pkg/cataloger/arch/dependency.go new file mode 100644 index 00000000000..796a61e1bb9 --- /dev/null +++ b/syft/pkg/cataloger/arch/dependency.go @@ -0,0 +1,48 @@ +package arch + +import ( + "strings" + + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" +) + +var _ dependency.Specifier = dbEntryDependencySpecifier + +func dbEntryDependencySpecifier(p pkg.Package) dependency.Specification { + meta, ok := p.Metadata.(pkg.AlpmDBEntry) + if !ok { + log.Tracef("cataloger failed to extract alpm metadata for package %+v", p.Name) + return dependency.Specification{} + } + + provides := []string{p.Name} + for _, key := range meta.Provides { + if key == "" { + continue + } + provides = append(provides, key, stripVersionSpecifier(key)) + } + + var requires []string + for _, depSpecifier := range meta.Depends { + if depSpecifier == "" { + continue + } + requires = append(requires, depSpecifier) + } + + return dependency.Specification{ + Provides: provides, + Requires: requires, + } +} + +func stripVersionSpecifier(s string) string { + // examples: + // gcc-libs --> gcc-libs + // libtree-sitter.so=0-64 --> libtree-sitter.so + + return strings.TrimSpace(strings.Split(s, "=")[0]) +} diff --git a/syft/pkg/cataloger/arch/dependency_test.go b/syft/pkg/cataloger/arch/dependency_test.go new file mode 100644 index 00000000000..e4eafae81af --- /dev/null +++ b/syft/pkg/cataloger/arch/dependency_test.go @@ -0,0 +1,100 @@ +package arch + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" +) + +func Test_dbEntryDependencySpecifier(t *testing.T) { + tests := []struct { + name string + p pkg.Package + want dependency.Specification + }{ + { + name: "keeps given values + package name", + p: pkg.Package{ + Name: "package-c", + Metadata: pkg.AlpmDBEntry{ + Provides: []string{"a-thing"}, + Depends: []string{"b-thing"}, + }, + }, + want: dependency.Specification{ + Provides: []string{"package-c", "a-thing", "a-thing"}, // note: gets deduplicated downstream + Requires: []string{"b-thing"}, + }, + }, + { + name: "strip version specifiers", + p: pkg.Package{ + Name: "package-a", + Metadata: pkg.AlpmDBEntry{ + Provides: []string{"libtree-sitter.so.me=1-64"}, + Depends: []string{"libtree-sitter.so.thing=2-64"}, + }, + }, + want: dependency.Specification{ + Provides: []string{"package-a", "libtree-sitter.so.me=1-64", "libtree-sitter.so.me"}, + Requires: []string{"libtree-sitter.so.thing=2-64"}, + }, + }, + { + name: "empty dependency data entries", + p: pkg.Package{ + Name: "package-a", + Metadata: pkg.AlpmDBEntry{ + Provides: []string{""}, + Depends: []string{""}, + }, + }, + want: dependency.Specification{ + Provides: []string{"package-a"}, + Requires: nil, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, dbEntryDependencySpecifier(tt.p)) + }) + } +} + +func Test_stripVersionSpecifier(t *testing.T) { + tests := []struct { + name string + version string + want string + }{ + { + name: "empty expression", + version: "", + want: "", + }, + { + name: "no expression", + version: "gcc-libs", + want: "gcc-libs", + }, + { + name: "=", + version: "libtree-sitter.so=0-64", + want: "libtree-sitter.so", + }, + { + name: "ignores file paths", + version: "/bin/sh", + want: "/bin/sh", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, stripVersionSpecifier(tt.version)) + }) + } +} diff --git a/syft/pkg/cataloger/arch/prosumer.go b/syft/pkg/cataloger/arch/prosumer.go deleted file mode 100644 index b59f56f2956..00000000000 --- a/syft/pkg/cataloger/arch/prosumer.go +++ /dev/null @@ -1,53 +0,0 @@ -package arch - -import ( - "strings" - - "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" -) - -var _ dependency.Prosumer = (*alpmDBEntryProsumer)(nil) - -type alpmDBEntryProsumer struct{} - -func newDBProsumer() alpmDBEntryProsumer { - return alpmDBEntryProsumer{} -} - -func (ps alpmDBEntryProsumer) Provides(p pkg.Package) []string { - meta, ok := p.Metadata.(pkg.AlpmDBEntry) - if !ok { - log.Warnf("cataloger failed to extract alpm 'provides' metadata for package %+v", p.Name) - return nil - } - keys := []string{p.Name} - for _, provides := range meta.Provides { - keys = append(keys, provides, stripVersionSpecifier(provides)) - } - return keys -} - -func (ps alpmDBEntryProsumer) Requires(p pkg.Package) []string { - meta, ok := p.Metadata.(pkg.AlpmDBEntry) - if !ok { - log.Warnf("cataloger failed to extract alpm 'requires' metadata for package %+v", p.Name) - return nil - } - - return meta.Depends -} - -func stripVersionSpecifier(s string) string { - // examples: - // gcc-libs --> gcc-libs - // libtree-sitter.so=0-64 --> libtree-sitter.so - - items := strings.Split(s, "=") - if len(items) == 0 { - return s - } - - return strings.TrimSpace(items[0]) -} From b874a3f7acaadef1bed70de82728a1c2987e3bc1 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 13 May 2024 18:00:40 -0400 Subject: [PATCH 07/11] use common dependency specifier for alpine Signed-off-by: Alex Goodman --- syft/pkg/cataloger/alpine/cataloger.go | 4 +- syft/pkg/cataloger/alpine/cataloger_test.go | 230 ++++++++++ syft/pkg/cataloger/alpine/dependency.go | 48 ++ syft/pkg/cataloger/alpine/dependency_test.go | 110 +++++ syft/pkg/cataloger/alpine/parse_apk_db.go | 56 +-- .../pkg/cataloger/alpine/parse_apk_db_test.go | 410 ------------------ .../cataloger/alpine/test-fixtures/multiple | 56 --- .../multiple-1/lib/apk/db/installed | 78 ++++ .../{ => multiple-2/lib/apk/db}/installed | 0 9 files changed, 470 insertions(+), 522 deletions(-) create mode 100644 syft/pkg/cataloger/alpine/dependency.go create mode 100644 syft/pkg/cataloger/alpine/dependency_test.go delete mode 100644 syft/pkg/cataloger/alpine/test-fixtures/multiple create mode 100644 syft/pkg/cataloger/alpine/test-fixtures/multiple-1/lib/apk/db/installed rename syft/pkg/cataloger/alpine/test-fixtures/{ => multiple-2/lib/apk/db}/installed (100%) diff --git a/syft/pkg/cataloger/alpine/cataloger.go b/syft/pkg/cataloger/alpine/cataloger.go index 5df825741cc..75075108a65 100644 --- a/syft/pkg/cataloger/alpine/cataloger.go +++ b/syft/pkg/cataloger/alpine/cataloger.go @@ -6,10 +6,12 @@ package alpine import ( "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" ) // NewDBCataloger returns a new cataloger object initialized for Alpine package DB flat-file stores. func NewDBCataloger() pkg.Cataloger { return generic.NewCataloger("apk-db-cataloger"). - WithParserByGlobs(parseApkDB, pkg.ApkDBGlob) + WithParserByGlobs(parseApkDB, pkg.ApkDBGlob). + WithProcessors(dependency.Processor(dbEntryDependencySpecifier)) } diff --git a/syft/pkg/cataloger/alpine/cataloger_test.go b/syft/pkg/cataloger/alpine/cataloger_test.go index d9ed0e30c14..8ac6eaa8ed3 100644 --- a/syft/pkg/cataloger/alpine/cataloger_test.go +++ b/syft/pkg/cataloger/alpine/cataloger_test.go @@ -3,9 +3,239 @@ package alpine import ( "testing" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" ) +func TestApkDBCataloger(t *testing.T) { + dbLocation := file.NewLocation("lib/apk/db/installed") + + bashPkg := pkg.Package{ + Name: "bash", + Version: "5.2.21-r0", + Type: pkg.ApkPkg, + FoundBy: "apk-db-cataloger", + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("GPL-3.0-or-later", dbLocation), + ), + Locations: file.NewLocationSet(dbLocation), + Metadata: pkg.ApkDBEntry{ + Package: "bash", + OriginPackage: "bash", + Maintainer: "Natanael Copa ", + Version: "5.2.21-r0", + Architecture: "x86_64", + URL: "https://www.gnu.org/software/bash/bash.html", + Description: "The GNU Bourne Again shell", + Size: 448728, + InstalledSize: 1396736, + Dependencies: []string{ + "/bin/sh", "so:libc.musl-x86_64.so.1", "so:libreadline.so.8", + }, + Provides: []string{ + "cmd:bash=5.2.21-r0", + }, + // note: files not provided and not under test + }, + } + + busyboxBinshPkg := pkg.Package{ + Name: "busybox-binsh", + Version: "1.36.1-r15", + Type: pkg.ApkPkg, + FoundBy: "apk-db-cataloger", + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("GPL-2.0-only", dbLocation), + ), + Locations: file.NewLocationSet(dbLocation), + Metadata: pkg.ApkDBEntry{ + Package: "busybox-binsh", + OriginPackage: "busybox", + Maintainer: "Sören Tempel ", + Version: "1.36.1-r15", + Architecture: "x86_64", + URL: "https://busybox.net/", + Description: "busybox ash /bin/sh", + Size: 1543, + InstalledSize: 8192, + Dependencies: []string{ + "busybox=1.36.1-r15", + }, + Provides: []string{ + "/bin/sh", "cmd:sh=1.36.1-r15", + }, + // note: files not provided and not under test + }, + } + + muslPkg := pkg.Package{ + Name: "musl", + Version: "1.2.4_git20230717-r4", + Type: pkg.ApkPkg, + FoundBy: "apk-db-cataloger", + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("MIT", dbLocation), + ), + Locations: file.NewLocationSet(dbLocation), + Metadata: pkg.ApkDBEntry{ + Package: "musl", + OriginPackage: "musl", + Maintainer: "Timo Teräs ", + Version: "1.2.4_git20230717-r4", + Architecture: "x86_64", + URL: "https://musl.libc.org/", + Description: "the musl c library (libc) implementation", + Size: 407278, + InstalledSize: 667648, + Dependencies: []string{}, + Provides: []string{ + "so:libc.musl-x86_64.so.1=1", + }, + // note: files not provided and not under test + }, + } + + readlinePkg := pkg.Package{ + Name: "readline", + Version: "8.2.1-r2", + Type: pkg.ApkPkg, + FoundBy: "apk-db-cataloger", + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("GPL-2.0-or-later", dbLocation), + ), + Locations: file.NewLocationSet(dbLocation), + Metadata: pkg.ApkDBEntry{ + Package: "readline", + OriginPackage: "readline", + Maintainer: "Natanael Copa ", + Version: "8.2.1-r2", + Architecture: "x86_64", + URL: "https://tiswww.cwru.edu/php/chet/readline/rltop.html", + Description: "GNU readline library", + Size: 119878, + InstalledSize: 303104, + Dependencies: []string{ + "so:libc.musl-x86_64.so.1", "so:libncursesw.so.6", + }, + Provides: []string{ + "so:libreadline.so.8=8.2", + }, + // note: files not provided and not under test + }, + } + + expectedPkgs := []pkg.Package{ + bashPkg, + busyboxBinshPkg, + muslPkg, + readlinePkg, + } + + // # apk info --depends bash + // bash-5.2.21-r0 depends on: + // /bin/sh + // so:libc.musl-x86_64.so.1 + // so:libreadline.so.8 + // + // # apk info --who-owns /bin/sh + // /bin/sh is owned by busybox-binsh-1.36.1-r15 + // + // # find / | grep musl + // /lib/ld-musl-x86_64.so.1 + // /lib/libc.musl-x86_64.so.1 + // + // # apk info --who-owns '/lib/libc.musl-x86_64.so.1' + // /lib/libc.musl-x86_64.so.1 is owned by musl-1.2.4_git20230717-r4 + // + // # find / | grep libreadline + // /usr/lib/libreadline.so.8.2 + // /usr/lib/libreadline.so.8 + // + // # apk info --who-owns '/usr/lib/libreadline.so.8' + // /usr/lib/libreadline.so.8 is owned by readline-8.2.1-r2 + + expectedRelationships := []artifact.Relationship{ + { + From: busyboxBinshPkg, + To: bashPkg, + Type: artifact.DependencyOfRelationship, + }, + { + From: readlinePkg, + To: bashPkg, + Type: artifact.DependencyOfRelationship, + }, + { + From: muslPkg, + To: readlinePkg, + Type: artifact.DependencyOfRelationship, + }, + { + From: muslPkg, + To: bashPkg, + Type: artifact.DependencyOfRelationship, + }, + } + + pkgtest.NewCatalogTester(). + FromDirectory(t, "test-fixtures/multiple-1"). + WithCompareOptions(cmpopts.IgnoreFields(pkg.ApkDBEntry{}, "Files", "GitCommit", "Checksum")). + Expects(expectedPkgs, expectedRelationships). + TestCataloger(t, NewDBCataloger()) + +} + +func TestCatalogerDependencyTree(t *testing.T) { + assertion := func(t *testing.T, pkgs []pkg.Package, relationships []artifact.Relationship) { + expected := map[string][]string{ + "alpine-baselayout": {"busybox", "alpine-baselayout-data", "musl"}, + "apk-tools": {"ca-certificates-bundle", "musl", "libcrypto1.1", "libssl1.1", "zlib"}, + "busybox": {"musl"}, + "libc-utils": {"musl-utils"}, + "libcrypto1.1": {"musl"}, + "libssl1.1": {"musl", "libcrypto1.1"}, + "musl-utils": {"scanelf", "musl"}, + "scanelf": {"musl"}, + "ssl_client": {"musl", "libcrypto1.1", "libssl1.1"}, + "zlib": {"musl"}, + } + pkgsByID := make(map[artifact.ID]pkg.Package) + for _, p := range pkgs { + p.SetID() + pkgsByID[p.ID()] = p + } + + actualDependencies := make(map[string][]string) + + for _, r := range relationships { + switch r.Type { + case artifact.DependencyOfRelationship: + to := pkgsByID[r.To.ID()] + from := pkgsByID[r.From.ID()] + actualDependencies[to.Name] = append(actualDependencies[to.Name], from.Name) + default: + t.Fatalf("unexpected relationship type: %+v", r.Type) + } + } + + if d := cmp.Diff(expected, actualDependencies); d != "" { + t.Fail() + t.Log(d) + } + } + + pkgtest.NewCatalogTester(). + FromDirectory(t, "test-fixtures/multiple-2"). + ExpectsAssertion(assertion). + TestCataloger(t, NewDBCataloger()) + +} + func TestCataloger_Globs(t *testing.T) { tests := []struct { name string diff --git a/syft/pkg/cataloger/alpine/dependency.go b/syft/pkg/cataloger/alpine/dependency.go new file mode 100644 index 00000000000..ea63f36bbba --- /dev/null +++ b/syft/pkg/cataloger/alpine/dependency.go @@ -0,0 +1,48 @@ +package alpine + +import ( + "strings" + + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" +) + +var _ dependency.Specifier = dbEntryDependencySpecifier + +func dbEntryDependencySpecifier(p pkg.Package) dependency.Specification { + meta, ok := p.Metadata.(pkg.ApkDBEntry) + if !ok { + log.Tracef("cataloger failed to extract apk metadata for package %+v", p.Name) + return dependency.Specification{} + } + + provides := []string{p.Name} + provides = append(provides, stripVersionSpecifiers(meta.Provides)...) + + return dependency.Specification{ + Provides: provides, + Requires: stripVersionSpecifiers(meta.Dependencies), + } +} + +func stripVersionSpecifiers(given []string) []string { + var keys []string + for _, key := range given { + key = stripVersionSpecifier(key) + if key == "" { + continue + } + keys = append(keys, key) + } + return keys +} + +func stripVersionSpecifier(s string) string { + // examples: + // musl>=1 --> musl + // cmd:scanelf=1.3.4-r0 --> cmd:scanelf + + return strings.TrimSpace(internal.SplitAny(s, "<>=")[0]) +} diff --git a/syft/pkg/cataloger/alpine/dependency_test.go b/syft/pkg/cataloger/alpine/dependency_test.go new file mode 100644 index 00000000000..a55cba4cb2a --- /dev/null +++ b/syft/pkg/cataloger/alpine/dependency_test.go @@ -0,0 +1,110 @@ +package alpine + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" +) + +func Test_dbEntryDependencySpecifier(t *testing.T) { + tests := []struct { + name string + p pkg.Package + want dependency.Specification + }{ + { + name: "keeps given values + package name", + p: pkg.Package{ + Name: "package-c", + Metadata: pkg.ApkDBEntry{ + Provides: []string{"a-thing"}, + Dependencies: []string{"b-thing"}, + }, + }, + want: dependency.Specification{ + Provides: []string{"package-c", "a-thing"}, + Requires: []string{"b-thing"}, + }, + }, + { + name: "strip version specifiers", + p: pkg.Package{ + Name: "package-a", + Metadata: pkg.ApkDBEntry{ + Provides: []string{"so:libc.musl-x86_64.so.1=1"}, + Dependencies: []string{"so:libc.musl-x86_64.so.2=2"}, + }, + }, + want: dependency.Specification{ + Provides: []string{"package-a", "so:libc.musl-x86_64.so.1"}, + Requires: []string{"so:libc.musl-x86_64.so.2"}, + }, + }, + { + name: "empty dependency data entries", + p: pkg.Package{ + Name: "package-a", + Metadata: pkg.ApkDBEntry{ + Provides: []string{""}, + Dependencies: []string{""}, + }, + }, + want: dependency.Specification{ + Provides: []string{"package-a"}, + Requires: nil, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, dbEntryDependencySpecifier(tt.p)) + }) + } +} + +func Test_stripVersionSpecifier(t *testing.T) { + tests := []struct { + name string + version string + want string + }{ + { + name: "empty expression", + version: "", + want: "", + }, + { + name: "no expression", + version: "cmd:foo", + want: "cmd:foo", + }, + { + name: "=", + version: "cmd:scanelf=1.3.4-r0", + want: "cmd:scanelf", + }, + { + name: ">=", + version: "cmd:scanelf>=1.3.4-r0", + want: "cmd:scanelf", + }, + { + name: "<", + version: "cmd:scanelf<1.3.4-r0", + want: "cmd:scanelf", + }, + { + name: "ignores file paths", + version: "/bin/sh", + want: "/bin/sh", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, stripVersionSpecifier(tt.version)) + }) + } +} diff --git a/syft/pkg/cataloger/alpine/parse_apk_db.go b/syft/pkg/cataloger/alpine/parse_apk_db.go index 5952ec16c9b..5948e303554 100644 --- a/syft/pkg/cataloger/alpine/parse_apk_db.go +++ b/syft/pkg/cataloger/alpine/parse_apk_db.go @@ -131,7 +131,7 @@ func parseApkDB(_ context.Context, resolver file.Resolver, env *generic.Environm pkgs = append(pkgs, newPackage(apk, r, reader.Location)) } - return pkgs, discoverPackageDependencies(pkgs), nil + return pkgs, nil, nil } func findReleases(resolver file.Resolver, dbPath string) []linux.Release { @@ -386,57 +386,3 @@ func processChecksum(value string) *file.Digest { Value: value, } } - -func discoverPackageDependencies(pkgs []pkg.Package) (relationships []artifact.Relationship) { - // map["provides" string] -> packages that provide the "p" key - lookup := make(map[string][]pkg.Package) - // read "Provides" (p) and add as keys for lookup keys as well as package names - for _, p := range pkgs { - apkg, ok := p.Metadata.(pkg.ApkDBEntry) - if !ok { - log.Warnf("cataloger failed to extract apk 'provides' metadata for package %+v", p.Name) - continue - } - lookup[p.Name] = append(lookup[p.Name], p) - for _, provides := range apkg.Provides { - k := stripVersionSpecifier(provides) - lookup[k] = append(lookup[k], p) - } - } - - // read "Pull Dependencies" (D) and match with keys - for _, p := range pkgs { - apkg, ok := p.Metadata.(pkg.ApkDBEntry) - if !ok { - log.Warnf("cataloger failed to extract apk dependency metadata for package %+v", p.Name) - continue - } - - for _, depSpecifier := range apkg.Dependencies { - // use the lookup to find what pkg we depend on - dep := stripVersionSpecifier(depSpecifier) - for _, depPkg := range lookup[dep] { - // this is a pkg that package "p" depends on... make a relationship - relationships = append(relationships, artifact.Relationship{ - From: depPkg, - To: p, - Type: artifact.DependencyOfRelationship, - }) - } - } - } - return relationships -} - -func stripVersionSpecifier(s string) string { - // examples: - // musl>=1 --> musl - // cmd:scanelf=1.3.4-r0 --> cmd:scanelf - - items := internal.SplitAny(s, "<>=") - if len(items) == 0 { - return s - } - - return items[0] -} diff --git a/syft/pkg/cataloger/alpine/parse_apk_db_test.go b/syft/pkg/cataloger/alpine/parse_apk_db_test.go index fef3275ab6b..7918f38f652 100644 --- a/syft/pkg/cataloger/alpine/parse_apk_db_test.go +++ b/syft/pkg/cataloger/alpine/parse_apk_db_test.go @@ -9,11 +9,9 @@ import ( "testing" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/linux" "github.com/anchore/syft/syft/pkg" @@ -689,144 +687,6 @@ func TestSinglePackageDetails(t *testing.T) { } } -func TestMultiplePackages(t *testing.T) { - fixture := "test-fixtures/multiple" - location := file.NewLocation(fixture) - fixtureLocationSet := file.NewLocationSet(location) - expectedPkgs := []pkg.Package{ - { - Name: "libc-utils", - Version: "0.7.2-r0", - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("MPL-2.0 AND MIT", location), - ), - Type: pkg.ApkPkg, - PURL: "pkg:apk/alpine/libc-utils@0.7.2-r0?arch=x86_64&upstream=libc-dev&distro=alpine-3.12", - Locations: fixtureLocationSet, - Metadata: pkg.ApkDBEntry{ - Package: "libc-utils", - OriginPackage: "libc-dev", - Maintainer: "Natanael Copa ", - Version: "0.7.2-r0", - Architecture: "x86_64", - URL: "http://alpinelinux.org", - Description: "Meta package to pull in correct libc", - Size: 1175, - InstalledSize: 4096, - Checksum: "Q1p78yvTLG094tHE1+dToJGbmYzQE=", - GitCommit: "97b1c2842faa3bfa30f5811ffbf16d5ff9f1a479", - Dependencies: []string{"musl-utils"}, - Provides: []string{}, - Files: []pkg.ApkFileRecord{}, - }, - }, - { - Name: "musl-utils", - Version: "1.1.24-r2", - Type: pkg.ApkPkg, - PURL: "pkg:apk/alpine/musl-utils@1.1.24-r2?arch=x86_64&upstream=musl&distro=alpine-3.12", - Locations: fixtureLocationSet, - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("MIT", location), - pkg.NewLicenseFromLocations("BSD", location), - pkg.NewLicenseFromLocations("GPL2+", location), - ), - Metadata: pkg.ApkDBEntry{ - Package: "musl-utils", - OriginPackage: "musl", - Version: "1.1.24-r2", - Description: "the musl c library (libc) implementation", - Maintainer: "Timo Teräs ", - Architecture: "x86_64", - URL: "https://musl.libc.org/", - Size: 37944, - InstalledSize: 151552, - GitCommit: "4024cc3b29ad4c65544ad068b8f59172b5494306", - Dependencies: []string{"scanelf", "so:libc.musl-x86_64.so.1"}, - Provides: []string{"cmd:getconf", "cmd:getent", "cmd:iconv", "cmd:ldconfig", "cmd:ldd"}, - Checksum: "Q1bTtF5526tETKfL+lnigzIDvm+2o=", - Files: []pkg.ApkFileRecord{ - { - Path: "/sbin", - }, - { - Path: "/sbin/ldconfig", - OwnerUID: "0", - OwnerGID: "0", - Permissions: "755", - Digest: &file.Digest{ - Algorithm: "'Q1'+base64(sha1)", - Value: "Q1Kja2+POZKxEkUOZqwSjC6kmaED4=", - }, - }, - { - Path: "/usr", - }, - { - Path: "/usr/bin", - }, - { - Path: "/usr/bin/iconv", - OwnerUID: "0", - OwnerGID: "0", - Permissions: "755", - Digest: &file.Digest{ - Algorithm: "'Q1'+base64(sha1)", - Value: "Q1CVmFbdY+Hv6/jAHl1gec2Kbx1EY=", - }, - }, - { - Path: "/usr/bin/ldd", - OwnerUID: "0", - OwnerGID: "0", - Permissions: "755", - Digest: &file.Digest{ - Algorithm: "'Q1'+base64(sha1)", - Value: "Q1yFAhGggmL7ERgbIA7KQxyTzf3ks=", - }, - }, - { - Path: "/usr/bin/getconf", - OwnerUID: "0", - OwnerGID: "0", - Permissions: "755", - Digest: &file.Digest{ - Algorithm: "'Q1'+base64(sha1)", - Value: "Q1dAdYK8M/INibRQF5B3Rw7cmNDDA=", - }, - }, - { - Path: "/usr/bin/getent", - OwnerUID: "0", - OwnerGID: "0", - Permissions: "755", - Digest: &file.Digest{ - Algorithm: "'Q1'+base64(sha1)", - Value: "Q1eR2Dz/WylabgbWMTkd2+hGmEya4=", - }, - }, - }, - }, - }, - } - - expectedRelationships := []artifact.Relationship{ - { - From: expectedPkgs[1], // musl-utils - To: expectedPkgs[0], // libc-utils - Type: artifact.DependencyOfRelationship, - Data: nil, - }, - } - - env := generic.Environment{LinuxRelease: &linux.Release{ - ID: "alpine", - VersionID: "3.12", - }} - - pkgtest.TestFileParserWithEnv(t, fixture, parseApkDB, &env, expectedPkgs, expectedRelationships) -} - func Test_processChecksum(t *testing.T) { tests := []struct { name string @@ -858,237 +718,6 @@ func Test_processChecksum(t *testing.T) { } } -func Test_discoverPackageDependencies(t *testing.T) { - tests := []struct { - name string - genFn func() ([]pkg.Package, []artifact.Relationship) - }{ - { - name: "has no dependency", - genFn: func() ([]pkg.Package, []artifact.Relationship) { - a := pkg.Package{ - Name: "package-a", - Metadata: pkg.ApkDBEntry{ - Provides: []string{"a-thing"}, - }, - } - a.SetID() - b := pkg.Package{ - Name: "package-b", - Metadata: pkg.ApkDBEntry{ - Provides: []string{"b-thing"}, - }, - } - b.SetID() - - return []pkg.Package{a, b}, nil - }, - }, - { - name: "has 1 dependency", - genFn: func() ([]pkg.Package, []artifact.Relationship) { - a := pkg.Package{ - Name: "package-a", - Metadata: pkg.ApkDBEntry{ - Dependencies: []string{"b-thing"}, - }, - } - a.SetID() - b := pkg.Package{ - Name: "package-b", - Metadata: pkg.ApkDBEntry{ - Provides: []string{"b-thing"}, - }, - } - b.SetID() - - return []pkg.Package{a, b}, []artifact.Relationship{ - { - From: b, - To: a, - Type: artifact.DependencyOfRelationship, - }, - } - }, - }, - { - name: "strip version specifiers", - genFn: func() ([]pkg.Package, []artifact.Relationship) { - a := pkg.Package{ - Name: "package-a", - Metadata: pkg.ApkDBEntry{ - Dependencies: []string{"so:libc.musl-x86_64.so.1"}, - }, - } - a.SetID() - b := pkg.Package{ - Name: "package-b", - Metadata: pkg.ApkDBEntry{ - Provides: []string{"so:libc.musl-x86_64.so.1=1"}, - }, - } - b.SetID() - - return []pkg.Package{a, b}, []artifact.Relationship{ - { - From: b, - To: a, - Type: artifact.DependencyOfRelationship, - }, - } - }, - }, - { - name: "strip version specifiers with empty provides value", - genFn: func() ([]pkg.Package, []artifact.Relationship) { - a := pkg.Package{ - Name: "package-a", - Metadata: pkg.ApkDBEntry{ - Dependencies: []string{"so:libc.musl-x86_64.so.1"}, - }, - } - a.SetID() - b := pkg.Package{ - Name: "package-b", - Metadata: pkg.ApkDBEntry{ - Provides: []string{""}, - }, - } - b.SetID() - - return []pkg.Package{a, b}, nil - }, - }, - { - name: "depends on package name", - genFn: func() ([]pkg.Package, []artifact.Relationship) { - a := pkg.Package{ - Name: "package-a", - Metadata: pkg.ApkDBEntry{ - Dependencies: []string{"musl>=1.2"}, - }, - } - a.SetID() - b := pkg.Package{ - Name: "musl", - Metadata: pkg.ApkDBEntry{ - Provides: []string{"so:libc.musl-x86_64.so.1=1"}, - }, - } - b.SetID() - - return []pkg.Package{a, b}, []artifact.Relationship{ - { - From: b, - To: a, - Type: artifact.DependencyOfRelationship, - }, - } - }, - }, - { - name: "depends on package file", - genFn: func() ([]pkg.Package, []artifact.Relationship) { - a := pkg.Package{ - Name: "alpine-baselayout", - Metadata: pkg.ApkDBEntry{ - Dependencies: []string{"/bin/sh"}, - }, - } - a.SetID() - b := pkg.Package{ - Name: "busybox", - Metadata: pkg.ApkDBEntry{ - Provides: []string{"/bin/sh"}, - }, - } - b.SetID() - - return []pkg.Package{a, b}, []artifact.Relationship{ - { - From: b, - To: a, - Type: artifact.DependencyOfRelationship, - }, - } - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - pkgs, wantRelationships := test.genFn() - gotRelationships := discoverPackageDependencies(pkgs) - d := cmp.Diff(wantRelationships, gotRelationships, cmpopts.IgnoreUnexported(pkg.Package{}, file.LocationSet{}, pkg.LicenseSet{})) - if d != "" { - t.Fail() - t.Log(d) - } - }) - } -} - -func TestPackageDbDependenciesByParse(t *testing.T) { - tests := []struct { - fixture string - expected map[string][]string - }{ - { - fixture: "test-fixtures/installed", - expected: map[string][]string{ - "alpine-baselayout": {"alpine-baselayout-data", "busybox", "musl"}, - "apk-tools": {"musl", "ca-certificates-bundle", "musl", "libcrypto1.1", "libssl1.1", "zlib"}, - "busybox": {"musl"}, - "libc-utils": {"musl-utils"}, - "libcrypto1.1": {"musl"}, - "libssl1.1": {"musl", "libcrypto1.1"}, - "musl-utils": {"scanelf", "musl"}, - "scanelf": {"musl"}, - "ssl_client": {"musl", "libcrypto1.1", "libssl1.1"}, - "zlib": {"musl"}, - }, - }, - } - - for _, test := range tests { - t.Run(test.fixture, func(t *testing.T) { - f, err := os.Open(test.fixture) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, f.Close()) }) - - pkgs, relationships, err := parseApkDB(context.Background(), nil, nil, file.LocationReadCloser{ - Location: file.NewLocation(test.fixture), - ReadCloser: f, - }) - require.NoError(t, err) - - pkgsByID := make(map[artifact.ID]pkg.Package) - for _, p := range pkgs { - p.SetID() - pkgsByID[p.ID()] = p - } - - actualDependencies := make(map[string][]string) - - for _, r := range relationships { - switch r.Type { - case artifact.DependencyOfRelationship: - to := pkgsByID[r.To.ID()] - from := pkgsByID[r.From.ID()] - actualDependencies[to.Name] = append(actualDependencies[to.Name], from.Name) - default: - t.Fatalf("unexpected relationship type: %+v", r.Type) - } - } - - if d := cmp.Diff(test.expected, actualDependencies); d != "" { - t.Fail() - t.Log(d) - } - }) - } -} - func Test_parseApkDB_expectedPkgNames(t *testing.T) { tests := []struct { fixture string @@ -1175,45 +804,6 @@ func newLocationReadCloser(t *testing.T, path string) file.LocationReadCloser { return file.NewLocationReadCloser(file.NewLocation(path), f) } -func Test_stripVersionSpecifier(t *testing.T) { - tests := []struct { - name string - version string - want string - }{ - { - name: "empty expression", - version: "", - want: "", - }, - { - name: "no expression", - version: "cmd:foo", - want: "cmd:foo", - }, - { - name: "=", - version: "cmd:scanelf=1.3.4-r0", - want: "cmd:scanelf", - }, - { - name: ">=", - version: "cmd:scanelf>=1.3.4-r0", - want: "cmd:scanelf", - }, - { - name: "<", - version: "cmd:scanelf<1.3.4-r0", - want: "cmd:scanelf", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, stripVersionSpecifier(tt.version)) - }) - } -} - func TestParseReleasesFromAPKRepository(t *testing.T) { tests := []struct { repos string diff --git a/syft/pkg/cataloger/alpine/test-fixtures/multiple b/syft/pkg/cataloger/alpine/test-fixtures/multiple deleted file mode 100644 index 7bf964cf8ec..00000000000 --- a/syft/pkg/cataloger/alpine/test-fixtures/multiple +++ /dev/null @@ -1,56 +0,0 @@ -C:Q1p78yvTLG094tHE1+dToJGbmYzQE= -P:libc-utils -V:0.7.2-r0 -A:x86_64 -S:1175 -I:4096 -T:Meta package to pull in correct libc -U:http://alpinelinux.org -L:MPL-2.0 AND MIT -o:libc-dev -m:Natanael Copa -t:1575749004 -c:97b1c2842faa3bfa30f5811ffbf16d5ff9f1a479 -D:musl-utils - - - - - - - - -C:Q1bTtF5526tETKfL+lnigzIDvm+2o= -P:musl-utils -V:1.1.24-r2 -A:x86_64 -S:37944 -I:151552 -T:the musl c library (libc) implementation -U:https://musl.libc.org/ -L:MIT BSD GPL2+ -o:musl -m:Timo Teräs -t:1584790550 -c:4024cc3b29ad4c65544ad068b8f59172b5494306 -D:scanelf so:libc.musl-x86_64.so.1 -p:cmd:getconf cmd:getent cmd:iconv cmd:ldconfig cmd:ldd -r:libiconv -F:sbin -R:ldconfig -a:0:0:755 -Z:Q1Kja2+POZKxEkUOZqwSjC6kmaED4= -F:usr -F:usr/bin -R:iconv -a:0:0:755 -Z:Q1CVmFbdY+Hv6/jAHl1gec2Kbx1EY= -R:ldd -a:0:0:755 -Z:Q1yFAhGggmL7ERgbIA7KQxyTzf3ks= -R:getconf -a:0:0:755 -Z:Q1dAdYK8M/INibRQF5B3Rw7cmNDDA= -R:getent -a:0:0:755 -Z:Q1eR2Dz/WylabgbWMTkd2+hGmEya4= \ No newline at end of file diff --git a/syft/pkg/cataloger/alpine/test-fixtures/multiple-1/lib/apk/db/installed b/syft/pkg/cataloger/alpine/test-fixtures/multiple-1/lib/apk/db/installed new file mode 100644 index 00000000000..1a93d9e3092 --- /dev/null +++ b/syft/pkg/cataloger/alpine/test-fixtures/multiple-1/lib/apk/db/installed @@ -0,0 +1,78 @@ +P:bash +V:5.2.21-r0 +A:x86_64 +S:448728 +I:1396736 +T:The GNU Bourne Again shell +U:https://www.gnu.org/software/bash/bash.html +L:GPL-3.0-or-later +o:bash +m:Natanael Copa +t:1701073495 +c:6a9559d98850225ba80771901ef1abda91cb29aa +D:/bin/sh so:libc.musl-x86_64.so.1 so:libreadline.so.8 +p:cmd:bash=5.2.21-r0 + +P:busybox-binsh +V:1.36.1-r15 +A:x86_64 +S:1543 +I:8192 +T:busybox ash /bin/sh +U:https://busybox.net/ +L:GPL-2.0-only +o:busybox +m:Sören Tempel +t:1699383189 +c:d1b6f274f29076967826e0ecf6ebcaa5d360272f +k:100 +D:busybox=1.36.1-r15 +p:/bin/sh cmd:sh=1.36.1-r15 +r:busybox-initscripts +F:bin +R:sh +a:0:0:777 +Z:Q1pcfTfDNEbNKQc2s1tia7da05M8Q= + +P:musl +V:1.2.4_git20230717-r4 +A:x86_64 +S:407278 +I:667648 +T:the musl c library (libc) implementation +U:https://musl.libc.org/ +L:MIT +o:musl +m:Timo Teräs +t:1699271358 +c:ca7f2ab5e88794e4e654b40776f8a92256f50639 +p:so:libc.musl-x86_64.so.1=1 +F:lib +R:ld-musl-x86_64.so.1 +a:0:0:755 +Z:Q1+zEJiG53Cxy7DkV5oZQqeWnzybY= +R:libc.musl-x86_64.so.1 +a:0:0:777 +Z:Q17yJ3JFNypA4mxhJJr0ou6CzsJVI= + +P:readline +V:8.2.1-r2 +A:x86_64 +S:119878 +I:303104 +T:GNU readline library +U:https://tiswww.cwru.edu/php/chet/readline/rltop.html +L:GPL-2.0-or-later +o:readline +m:Natanael Copa +t:1684120357 +c:33283848034c9885d984c8e8697c645c57324938 +D:so:libc.musl-x86_64.so.1 so:libncursesw.so.6 +p:so:libreadline.so.8=8.2 +F:etc +R:inputrc +Z:Q1ilcgkuEseXEH6iMo9UNjLn1pPfg= +F:usr +F:usr/lib +R:libreadline.so.8 +a:0:0:777 diff --git a/syft/pkg/cataloger/alpine/test-fixtures/installed b/syft/pkg/cataloger/alpine/test-fixtures/multiple-2/lib/apk/db/installed similarity index 100% rename from syft/pkg/cataloger/alpine/test-fixtures/installed rename to syft/pkg/cataloger/alpine/test-fixtures/multiple-2/lib/apk/db/installed From c8beddd1c7012b6d564baef067b2032de2dce5d9 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 13 May 2024 18:01:27 -0400 Subject: [PATCH 08/11] allow for generic pkg and rel assertions in testpkg helper Signed-off-by: Alex Goodman --- .../internal/pkgtest/test_generic_parser.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go index 11efb263bff..227620aa59d 100644 --- a/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go +++ b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go @@ -44,6 +44,7 @@ type CatalogTester struct { compareOptions []cmp.Option locationComparer locationComparer licenseComparer licenseComparer + customAssertions []func(t *testing.T, pkgs []pkg.Package, relationships []artifact.Relationship) } func NewCatalogTester() *CatalogTester { @@ -164,6 +165,11 @@ func (p *CatalogTester) WithImageResolver(t *testing.T, fixtureName string) *Cat return p } +func (p *CatalogTester) ExpectsAssertion(a func(t *testing.T, pkgs []pkg.Package, relationships []artifact.Relationship)) *CatalogTester { + p.customAssertions = append(p.customAssertions, a) + return p +} + func (p *CatalogTester) IgnoreLocationLayer() *CatalogTester { p.locationComparer = func(x, y file.Location) bool { return cmp.Equal(x.Coordinates.RealPath, y.Coordinates.RealPath) && cmp.Equal(x.AccessPath, y.AccessPath) @@ -250,7 +256,13 @@ func (p *CatalogTester) TestCataloger(t *testing.T, cataloger pkg.Cataloger) { if p.assertResultExpectations { p.wantErr(t, err) p.assertPkgs(t, pkgs, relationships) - } else { + } + + for _, a := range p.customAssertions { + a(t, pkgs, relationships) + } + + if !p.assertResultExpectations && len(p.customAssertions) == 0 { resolver.PruneUnfulfilledPathResponses(p.ignoreUnfulfilledPathResponses, p.ignoreAnyUnfulfilledPaths...) // if we aren't testing the results, we should focus on what was searched for (for glob-centric tests) From 74dddd11b16f89d86794b20b928b44f0722c752b Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 13 May 2024 18:01:44 -0400 Subject: [PATCH 09/11] do not allow for empty results Signed-off-by: Alex Goodman --- internal/string_helpers.go | 6 +++++- internal/string_helpers_test.go | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/string_helpers.go b/internal/string_helpers.go index 4f5c65a4ad5..f43d5918d30 100644 --- a/internal/string_helpers.go +++ b/internal/string_helpers.go @@ -33,5 +33,9 @@ func SplitAny(s string, seps string) []string { splitter := func(r rune) bool { return strings.ContainsRune(seps, r) } - return strings.FieldsFunc(s, splitter) + result := strings.FieldsFunc(s, splitter) + if len(result) == 0 { + return []string{s} + } + return result } diff --git a/internal/string_helpers_test.go b/internal/string_helpers_test.go index 45b90195aaa..401b28dc30f 100644 --- a/internal/string_helpers_test.go +++ b/internal/string_helpers_test.go @@ -123,7 +123,7 @@ func TestSplitAny(t *testing.T) { name: "empty", input: "", fields: ",", - want: []string{}, + want: []string{""}, }, { name: "multiple separators", From 6005cbf61c385ce81a7b6273780c6439b9e8be14 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 13 May 2024 18:06:02 -0400 Subject: [PATCH 10/11] move stable deduplicate comment Signed-off-by: Alex Goodman --- syft/pkg/cataloger/internal/dependency/resolver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syft/pkg/cataloger/internal/dependency/resolver.go b/syft/pkg/cataloger/internal/dependency/resolver.go index 66ebc034eae..0501096afd0 100644 --- a/syft/pkg/cataloger/internal/dependency/resolver.go +++ b/syft/pkg/cataloger/internal/dependency/resolver.go @@ -99,8 +99,8 @@ func (r RelationshipResolver) Resolve(pkgs []pkg.Package) (relationships []artif } func deduplicate(ss []string) []string { + // note: we sort the set such that multiple invocations of this function will be deterministic set := strset.New(ss...) - // note: this must be a stable function list := set.List() sort.Strings(list) return list From d3884e7ba2268d815b33abf0b724333c95bd9ad8 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 14 May 2024 09:17:49 -0400 Subject: [PATCH 11/11] remove relationship resolver type Signed-off-by: Alex Goodman --- .../cataloger/internal/dependency/resolver.go | 22 +++++-------------- .../internal/dependency/resolver_test.go | 4 ++-- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/syft/pkg/cataloger/internal/dependency/resolver.go b/syft/pkg/cataloger/internal/dependency/resolver.go index 0501096afd0..d60e3bb3c15 100644 --- a/syft/pkg/cataloger/internal/dependency/resolver.go +++ b/syft/pkg/cataloger/internal/dependency/resolver.go @@ -25,18 +25,6 @@ type Specification struct { // the package provides and needs. type Specifier func(pkg.Package) Specification -// RelationshipResolver uses a Specifier to resolve relationships between packages based on generic dependency claims -// and provider claims from any given package. -type RelationshipResolver struct { - specifier Specifier -} - -func NewRelationshipResolver(s Specifier) RelationshipResolver { - return RelationshipResolver{ - specifier: s, - } -} - // Processor returns a generic processor that will resolve relationships between packages based on the dependency claims. func Processor(s Specifier) generic.Processor { return func(pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) { @@ -49,13 +37,13 @@ func Processor(s Specifier) generic.Processor { } } - rels = append(rels, NewRelationshipResolver(s).Resolve(pkgs)...) + rels = append(rels, resolve(s, pkgs)...) return pkgs, rels, err } } -// Resolve will create relationships between packages based on the dependency claims of each package. -func (r RelationshipResolver) Resolve(pkgs []pkg.Package) (relationships []artifact.Relationship) { +// resolve will create relationships between packages based on the dependency claims of each package. +func resolve(specifier Specifier, pkgs []pkg.Package) (relationships []artifact.Relationship) { pkgsProvidingResource := make(map[string][]artifact.ID) pkgsByID := make(map[artifact.ID]pkg.Package) @@ -64,8 +52,8 @@ func (r RelationshipResolver) Resolve(pkgs []pkg.Package) (relationships []artif for _, p := range pkgs { id := p.ID() pkgsByID[id] = p - specsByPkg[id] = r.specifier(p) - for _, resource := range deduplicate(r.specifier(p).Provides) { + specsByPkg[id] = specifier(p) + for _, resource := range deduplicate(specifier(p).Provides) { pkgsProvidingResource[resource] = append(pkgsProvidingResource[resource], id) } } diff --git a/syft/pkg/cataloger/internal/dependency/resolver_test.go b/syft/pkg/cataloger/internal/dependency/resolver_test.go index 2549b996670..c6690d9589b 100644 --- a/syft/pkg/cataloger/internal/dependency/resolver_test.go +++ b/syft/pkg/cataloger/internal/dependency/resolver_test.go @@ -11,7 +11,7 @@ import ( "github.com/anchore/syft/syft/pkg" ) -func TestRelationshipResolver_Resolve(t *testing.T) { +func Test_resolve(t *testing.T) { a := pkg.Package{ Name: "a", } @@ -71,7 +71,7 @@ func TestRelationshipResolver_Resolve(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - relationships := NewRelationshipResolver(tt.s).Resolve(subjects) + relationships := resolve(tt.s, subjects) if d := cmp.Diff(tt.want, abstractRelationships(t, relationships)); d != "" { t.Errorf("unexpected relationships (-want +got):\n%s", d) }