From 0c5f03235ee1a8297a7ee6419949d2e2b26dc5af Mon Sep 17 00:00:00 2001 From: Weston Steimel Date: Fri, 24 Feb 2023 15:59:19 +0000 Subject: [PATCH] refactor: move apk upstream logic to apk metadata (#1619) * refactor: move apk upstream logic to apk metadata Export the logic for parsing upstream APK package names so it can be accessed from apk metadata objects directly. This also tightens the upstream regex pattern as several edge cases were being missed. Signed-off-by: Weston Steimel * fix: ensure correct handling for apk packages beginning with digits Signed-off-by: Weston Steimel * fix: upstream generation for ruby Signed-off-by: Weston Steimel --------- Signed-off-by: Weston Steimel --- syft/pkg/apk_metadata.go | 29 +++- syft/pkg/apk_metadata_test.go | 167 +++++++++++++++++++++++ syft/pkg/cataloger/apkdb/package.go | 33 +---- syft/pkg/cataloger/apkdb/package_test.go | 28 ++++ 4 files changed, 226 insertions(+), 31 deletions(-) diff --git a/syft/pkg/apk_metadata.go b/syft/pkg/apk_metadata.go index 21abd50df50..327de0ac8f1 100644 --- a/syft/pkg/apk_metadata.go +++ b/syft/pkg/apk_metadata.go @@ -4,18 +4,24 @@ import ( "encoding/json" "fmt" "reflect" + "regexp" "sort" "strings" "github.com/mitchellh/mapstructure" "github.com/scylladb/go-set/strset" + "github.com/anchore/syft/internal" "github.com/anchore/syft/syft/file" ) const ApkDBGlob = "**/lib/apk/db/installed" -var _ FileOwner = (*ApkMetadata)(nil) +var ( + _ FileOwner = (*ApkMetadata)(nil) + prefixes = []string{"py-", "py2-", "py3-", "ruby-"} + upstreamPattern = regexp.MustCompile(`^(?P[a-zA-Z][\w-]*?)\-?\d[\d\.]*$`) +) // ApkMetadata represents all captured data for a Alpine DB package entry. // See the following sources for more information: @@ -114,3 +120,24 @@ func (m ApkMetadata) OwnedFiles() (result []string) { sort.Strings(result) return result } + +func (m ApkMetadata) Upstream() string { + if m.OriginPackage != "" && m.OriginPackage != m.Package { + return m.OriginPackage + } + + groups := internal.MatchNamedCaptureGroups(upstreamPattern, m.Package) + + upstream, ok := groups["upstream"] + if !ok { + upstream = m.Package + } + + for _, p := range prefixes { + if strings.HasPrefix(upstream, p) { + return strings.TrimPrefix(upstream, p) + } + } + + return upstream +} diff --git a/syft/pkg/apk_metadata_test.go b/syft/pkg/apk_metadata_test.go index a5e5f4121dd..610d8321a49 100644 --- a/syft/pkg/apk_metadata_test.go +++ b/syft/pkg/apk_metadata_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "testing" + "github.com/sergi/go-diff/diffmatchpatch" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -163,3 +164,169 @@ func TestSpaceDelimitedStringSlice_UnmarshalJSON(t *testing.T) { }) } } + +func TestApkMetadata_Upstream(t *testing.T) { + tests := []struct { + name string + metadata ApkMetadata + expected string + }{ + { + name: "gocase", + metadata: ApkMetadata{ + Package: "p", + }, + expected: "p", + }, + { + name: "same package and origin", + metadata: ApkMetadata{ + Package: "p", + OriginPackage: "p", + }, + expected: "p", + }, + { + name: "different package and origin", + metadata: ApkMetadata{ + Package: "p", + OriginPackage: "origin", + }, + expected: "origin", + }, + { + name: "upstream python package information as qualifier", + metadata: ApkMetadata{ + Package: "py3-potatoes", + OriginPackage: "py3-potatoes", + }, + expected: "potatoes", + }, + { + name: "python package with distinct origin package", + metadata: ApkMetadata{ + Package: "py3-non-existant", + OriginPackage: "abcdefg", + }, + expected: "abcdefg", + }, + { + name: "upstream ruby package information as qualifier", + metadata: ApkMetadata{ + Package: "ruby-something", + OriginPackage: "ruby-something", + }, + expected: "something", + }, + { + name: "python package with distinct origin package", + metadata: ApkMetadata{ + Package: "ruby-something", + OriginPackage: "1234567", + }, + expected: "1234567", + }, + { + name: "postgesql-15 upstream postgresql", + metadata: ApkMetadata{ + Package: "postgresql-15", + }, + expected: "postgresql", + }, + { + name: "postgesql15 upstream postgresql", + metadata: ApkMetadata{ + Package: "postgresql15", + }, + expected: "postgresql", + }, + { + name: "go-1.19 upstream go", + metadata: ApkMetadata{ + Package: "go-1.19", + }, + expected: "go", + }, + { + name: "go1.143 upstream go", + metadata: ApkMetadata{ + Package: "go1.143", + }, + expected: "go", + }, + { + name: "abc-101.191.23456 upstream abc", + metadata: ApkMetadata{ + Package: "abc-101.191.23456", + }, + expected: "abc", + }, + { + name: "abc101.191.23456 upstream abc", + metadata: ApkMetadata{ + Package: "abc101.191.23456", + }, + expected: "abc", + }, + { + name: "abc101-12345-1045 upstream abc101-12345", + metadata: ApkMetadata{ + Package: "abc101-12345-1045", + }, + expected: "abc101-12345", + }, + { + name: "abc101-a12345-1045 upstream abc101-a12345", + metadata: ApkMetadata{ + Package: "abc101-a12345-1045", + }, + expected: "abc101-a12345", + }, + { + name: "package starting with single digit", + metadata: ApkMetadata{ + Package: "3proxy", + }, + expected: "3proxy", + }, + { + name: "package starting with multiple digits", + metadata: ApkMetadata{ + Package: "356proxy", + }, + expected: "356proxy", + }, + { + name: "package composed of only digits", + metadata: ApkMetadata{ + Package: "123456", + }, + expected: "123456", + }, + { + name: "ruby-3.6 upstream ruby", + metadata: ApkMetadata{ + Package: "ruby-3.6", + }, + expected: "ruby", + }, + { + name: "ruby3.6 upstream ruby", + metadata: ApkMetadata{ + Package: "ruby3.6", + }, + expected: "ruby", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + actual := test.metadata.Upstream() + if actual != test.expected { + dmp := diffmatchpatch.New() + diffs := dmp.DiffMain(test.expected, actual, true) + t.Errorf("diff: %s", dmp.DiffPrettyText(diffs)) + } + }) + } +} diff --git a/syft/pkg/cataloger/apkdb/package.go b/syft/pkg/cataloger/apkdb/package.go index 3ea189d551d..85f58da64aa 100644 --- a/syft/pkg/cataloger/apkdb/package.go +++ b/syft/pkg/cataloger/apkdb/package.go @@ -1,20 +1,14 @@ package apkdb import ( - "regexp" "strings" "github.com/anchore/packageurl-go" - "github.com/anchore/syft/internal" "github.com/anchore/syft/syft/linux" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" ) -var ( - prefixes = []string{"py-", "py2-", "py3-", "ruby-"} -) - func newPackage(d pkg.ApkMetadata, release *linux.Release, locations ...source.Location) pkg.Package { p := pkg.Package{ Name: d.Package, @@ -32,28 +26,6 @@ func newPackage(d pkg.ApkMetadata, release *linux.Release, locations ...source.L return p } -func generateUpstream(m pkg.ApkMetadata) string { - if m.OriginPackage != "" && m.OriginPackage != m.Package { - return m.OriginPackage - } - - for _, p := range prefixes { - if strings.HasPrefix(m.Package, p) { - return strings.TrimPrefix(m.Package, p) - } - } - - pattern := regexp.MustCompile(`(?P\w+?)\-?\d[\d\.]*`) - groups := internal.MatchNamedCaptureGroups(pattern, m.Package) - - upstream, ok := groups["upstream"] - if ok { - return upstream - } - - return m.Package -} - // packageURL returns the PURL for the specific Alpine package (see https://github.com/package-url/purl-spec) func packageURL(m pkg.ApkMetadata, distro *linux.Release) string { if distro == nil || distro.ID != "alpine" { @@ -65,8 +37,9 @@ func packageURL(m pkg.ApkMetadata, distro *linux.Release) string { pkg.PURLQualifierArch: m.Architecture, } - if m.OriginPackage != "" { - qualifiers[pkg.PURLQualifierUpstream] = generateUpstream(m) + upstream := m.Upstream() + if upstream != "" && upstream != m.Package { + qualifiers[pkg.PURLQualifierUpstream] = upstream } return packageurl.NewPackageURL( diff --git a/syft/pkg/cataloger/apkdb/package_test.go b/syft/pkg/cataloger/apkdb/package_test.go index 31dc1c7ca60..8cffeabaf65 100644 --- a/syft/pkg/cataloger/apkdb/package_test.go +++ b/syft/pkg/cataloger/apkdb/package_test.go @@ -208,6 +208,34 @@ func Test_PackageURL(t *testing.T) { }, expected: "pkg:apk/alpine/abc101.191.23456@101.191.23456?arch=a&upstream=abc&distro=alpine-3.4.6", }, + { + name: "abc101-12345-1045 upstream abc101-12345", + metadata: pkg.ApkMetadata{ + Package: "abc101-12345-1045", + Version: "101.191.23456", + Architecture: "a", + OriginPackage: "abc101-12345-1045", + }, + distro: linux.Release{ + ID: "alpine", + VersionID: "3.4.6", + }, + expected: "pkg:apk/alpine/abc101-12345-1045@101.191.23456?arch=a&upstream=abc101-12345&distro=alpine-3.4.6", + }, + { + name: "abc101-a12345-1045 upstream abc101-a12345", + metadata: pkg.ApkMetadata{ + Package: "abc101-a12345-1045", + Version: "101.191.23456", + Architecture: "a", + OriginPackage: "abc101-a12345-1045", + }, + distro: linux.Release{ + ID: "alpine", + VersionID: "3.4.6", + }, + expected: "pkg:apk/alpine/abc101-a12345-1045@101.191.23456?arch=a&upstream=abc101-a12345&distro=alpine-3.4.6", + }, } for _, test := range tests {