From aeebe4f5676c762237148c84063a0c3cc88617ea Mon Sep 17 00:00:00 2001 From: Weston Steimel Date: Fri, 24 Feb 2023 10:50:31 +0000 Subject: [PATCH 1/3] refactor: move apk upstream logic to apk metadata Export the logic for parsing upstream APK package names so it can be accessed from apk metadata objects directly. This also tightens the upstream regex pattern as several edge cases were being missed. Signed-off-by: Weston Steimel --- syft/pkg/apk_metadata.go | 29 ++++- syft/pkg/apk_metadata_test.go | 132 +++++++++++++++++++++++ syft/pkg/cataloger/apkdb/package.go | 33 +----- syft/pkg/cataloger/apkdb/package_test.go | 28 +++++ 4 files changed, 191 insertions(+), 31 deletions(-) diff --git a/syft/pkg/apk_metadata.go b/syft/pkg/apk_metadata.go index 21abd50df50..88c2229aef1 100644 --- a/syft/pkg/apk_metadata.go +++ b/syft/pkg/apk_metadata.go @@ -4,18 +4,24 @@ import ( "encoding/json" "fmt" "reflect" + "regexp" "sort" "strings" "github.com/mitchellh/mapstructure" "github.com/scylladb/go-set/strset" + "github.com/anchore/syft/internal" "github.com/anchore/syft/syft/file" ) const ApkDBGlob = "**/lib/apk/db/installed" -var _ FileOwner = (*ApkMetadata)(nil) +var ( + _ FileOwner = (*ApkMetadata)(nil) + prefixes = []string{"py-", "py2-", "py3-", "ruby-"} + upstreamPattern = regexp.MustCompile(`^(?P[\w-]+?)\-?\d[\d\.]*$`) +) // ApkMetadata represents all captured data for a Alpine DB package entry. // See the following sources for more information: @@ -114,3 +120,24 @@ func (m ApkMetadata) OwnedFiles() (result []string) { sort.Strings(result) return result } + +func (m ApkMetadata) Upstream() string { + if m.OriginPackage != "" && m.OriginPackage != m.Package { + return m.OriginPackage + } + + for _, p := range prefixes { + if strings.HasPrefix(m.Package, p) { + return strings.TrimPrefix(m.Package, p) + } + } + + groups := internal.MatchNamedCaptureGroups(upstreamPattern, m.Package) + + upstream, ok := groups["upstream"] + if ok { + return upstream + } + + return m.Package +} diff --git a/syft/pkg/apk_metadata_test.go b/syft/pkg/apk_metadata_test.go index a5e5f4121dd..d7a0cc14bc3 100644 --- a/syft/pkg/apk_metadata_test.go +++ b/syft/pkg/apk_metadata_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "testing" + "github.com/sergi/go-diff/diffmatchpatch" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -163,3 +164,134 @@ func TestSpaceDelimitedStringSlice_UnmarshalJSON(t *testing.T) { }) } } + +func TestApkMetadata_Upstream(t *testing.T) { + tests := []struct { + name string + metadata ApkMetadata + expected string + }{ + { + name: "gocase", + metadata: ApkMetadata{ + Package: "p", + }, + expected: "p", + }, + { + name: "same package and origin", + metadata: ApkMetadata{ + Package: "p", + OriginPackage: "p", + }, + expected: "p", + }, + { + name: "different package and origin", + metadata: ApkMetadata{ + Package: "p", + OriginPackage: "origin", + }, + expected: "origin", + }, + { + name: "upstream python package information as qualifier", + metadata: ApkMetadata{ + Package: "py3-potatoes", + OriginPackage: "py3-potatoes", + }, + expected: "potatoes", + }, + { + name: "python package with distinct origin package", + metadata: ApkMetadata{ + Package: "py3-non-existant", + OriginPackage: "abcdefg", + }, + expected: "abcdefg", + }, + { + name: "upstream ruby package information as qualifier", + metadata: ApkMetadata{ + Package: "ruby-something", + OriginPackage: "ruby-something", + }, + expected: "something", + }, + { + name: "python package with distinct origin package", + metadata: ApkMetadata{ + Package: "ruby-something", + OriginPackage: "1234567", + }, + expected: "1234567", + }, + { + name: "postgesql-15 upstream postgresql", + metadata: ApkMetadata{ + Package: "postgresql-15", + }, + expected: "postgresql", + }, + { + name: "postgesql15 upstream postgresql", + metadata: ApkMetadata{ + Package: "postgresql15", + }, + expected: "postgresql", + }, + { + name: "go-1.19 upstream go", + metadata: ApkMetadata{ + Package: "go-1.19", + }, + expected: "go", + }, + { + name: "go1.143 upstream go", + metadata: ApkMetadata{ + Package: "go1.143", + }, + expected: "go", + }, + { + name: "abc-101.191.23456 upstream abc", + metadata: ApkMetadata{ + Package: "abc-101.191.23456", + }, + expected: "abc", + }, + { + name: "abc101.191.23456 upstream abc", + metadata: ApkMetadata{ + Package: "abc101.191.23456", + }, + expected: "abc", + }, + { + name: "abc101-12345-1045 upstream abc101-12345", + metadata: ApkMetadata{ + Package: "abc101-12345-1045", + }, + expected: "abc101-12345", + }, + { + name: "abc101-a12345-1045 upstream abc101-a12345", + metadata: ApkMetadata{ + Package: "abc101-a12345-1045", + }, + expected: "abc101-a12345", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + actual := test.metadata.Upstream() + if actual != test.expected { + dmp := diffmatchpatch.New() + diffs := dmp.DiffMain(test.expected, actual, true) + t.Errorf("diff: %s", dmp.DiffPrettyText(diffs)) + } + }) + } +} diff --git a/syft/pkg/cataloger/apkdb/package.go b/syft/pkg/cataloger/apkdb/package.go index 3ea189d551d..85f58da64aa 100644 --- a/syft/pkg/cataloger/apkdb/package.go +++ b/syft/pkg/cataloger/apkdb/package.go @@ -1,20 +1,14 @@ package apkdb import ( - "regexp" "strings" "github.com/anchore/packageurl-go" - "github.com/anchore/syft/internal" "github.com/anchore/syft/syft/linux" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" ) -var ( - prefixes = []string{"py-", "py2-", "py3-", "ruby-"} -) - func newPackage(d pkg.ApkMetadata, release *linux.Release, locations ...source.Location) pkg.Package { p := pkg.Package{ Name: d.Package, @@ -32,28 +26,6 @@ func newPackage(d pkg.ApkMetadata, release *linux.Release, locations ...source.L return p } -func generateUpstream(m pkg.ApkMetadata) string { - if m.OriginPackage != "" && m.OriginPackage != m.Package { - return m.OriginPackage - } - - for _, p := range prefixes { - if strings.HasPrefix(m.Package, p) { - return strings.TrimPrefix(m.Package, p) - } - } - - pattern := regexp.MustCompile(`(?P\w+?)\-?\d[\d\.]*`) - groups := internal.MatchNamedCaptureGroups(pattern, m.Package) - - upstream, ok := groups["upstream"] - if ok { - return upstream - } - - return m.Package -} - // packageURL returns the PURL for the specific Alpine package (see https://github.com/package-url/purl-spec) func packageURL(m pkg.ApkMetadata, distro *linux.Release) string { if distro == nil || distro.ID != "alpine" { @@ -65,8 +37,9 @@ func packageURL(m pkg.ApkMetadata, distro *linux.Release) string { pkg.PURLQualifierArch: m.Architecture, } - if m.OriginPackage != "" { - qualifiers[pkg.PURLQualifierUpstream] = generateUpstream(m) + upstream := m.Upstream() + if upstream != "" && upstream != m.Package { + qualifiers[pkg.PURLQualifierUpstream] = upstream } return packageurl.NewPackageURL( diff --git a/syft/pkg/cataloger/apkdb/package_test.go b/syft/pkg/cataloger/apkdb/package_test.go index 31dc1c7ca60..8cffeabaf65 100644 --- a/syft/pkg/cataloger/apkdb/package_test.go +++ b/syft/pkg/cataloger/apkdb/package_test.go @@ -208,6 +208,34 @@ func Test_PackageURL(t *testing.T) { }, expected: "pkg:apk/alpine/abc101.191.23456@101.191.23456?arch=a&upstream=abc&distro=alpine-3.4.6", }, + { + name: "abc101-12345-1045 upstream abc101-12345", + metadata: pkg.ApkMetadata{ + Package: "abc101-12345-1045", + Version: "101.191.23456", + Architecture: "a", + OriginPackage: "abc101-12345-1045", + }, + distro: linux.Release{ + ID: "alpine", + VersionID: "3.4.6", + }, + expected: "pkg:apk/alpine/abc101-12345-1045@101.191.23456?arch=a&upstream=abc101-12345&distro=alpine-3.4.6", + }, + { + name: "abc101-a12345-1045 upstream abc101-a12345", + metadata: pkg.ApkMetadata{ + Package: "abc101-a12345-1045", + Version: "101.191.23456", + Architecture: "a", + OriginPackage: "abc101-a12345-1045", + }, + distro: linux.Release{ + ID: "alpine", + VersionID: "3.4.6", + }, + expected: "pkg:apk/alpine/abc101-a12345-1045@101.191.23456?arch=a&upstream=abc101-a12345&distro=alpine-3.4.6", + }, } for _, test := range tests { From 6da20f942d91b447be0bf30bc4ab6bab0a7bf82a Mon Sep 17 00:00:00 2001 From: Weston Steimel Date: Fri, 24 Feb 2023 11:20:35 +0000 Subject: [PATCH 2/3] fix: ensure correct handling for apk packages beginning with digits Signed-off-by: Weston Steimel --- syft/pkg/apk_metadata.go | 2 +- syft/pkg/apk_metadata_test.go | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/syft/pkg/apk_metadata.go b/syft/pkg/apk_metadata.go index 88c2229aef1..02ec69e7df2 100644 --- a/syft/pkg/apk_metadata.go +++ b/syft/pkg/apk_metadata.go @@ -20,7 +20,7 @@ const ApkDBGlob = "**/lib/apk/db/installed" var ( _ FileOwner = (*ApkMetadata)(nil) prefixes = []string{"py-", "py2-", "py3-", "ruby-"} - upstreamPattern = regexp.MustCompile(`^(?P[\w-]+?)\-?\d[\d\.]*$`) + upstreamPattern = regexp.MustCompile(`^(?P[a-zA-Z][\w-]*?)\-?\d[\d\.]*$`) ) // ApkMetadata represents all captured data for a Alpine DB package entry. diff --git a/syft/pkg/apk_metadata_test.go b/syft/pkg/apk_metadata_test.go index d7a0cc14bc3..f04e7a09eb2 100644 --- a/syft/pkg/apk_metadata_test.go +++ b/syft/pkg/apk_metadata_test.go @@ -282,6 +282,27 @@ func TestApkMetadata_Upstream(t *testing.T) { }, expected: "abc101-a12345", }, + { + name: "package starting with single digit", + metadata: ApkMetadata{ + Package: "3proxy", + }, + expected: "3proxy", + }, + { + name: "package starting with multiple digits", + metadata: ApkMetadata{ + Package: "356proxy", + }, + expected: "356proxy", + }, + { + name: "package composed of only digits", + metadata: ApkMetadata{ + Package: "123456", + }, + expected: "123456", + }, } for _, test := range tests { From 3a78371fa6608776411bc1be511c4d476faa31ee Mon Sep 17 00:00:00 2001 From: Weston Steimel Date: Fri, 24 Feb 2023 12:04:57 +0000 Subject: [PATCH 3/3] fix: upstream generation for ruby Signed-off-by: Weston Steimel --- syft/pkg/apk_metadata.go | 18 +++++++++--------- syft/pkg/apk_metadata_test.go | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/syft/pkg/apk_metadata.go b/syft/pkg/apk_metadata.go index 02ec69e7df2..327de0ac8f1 100644 --- a/syft/pkg/apk_metadata.go +++ b/syft/pkg/apk_metadata.go @@ -126,18 +126,18 @@ func (m ApkMetadata) Upstream() string { return m.OriginPackage } - for _, p := range prefixes { - if strings.HasPrefix(m.Package, p) { - return strings.TrimPrefix(m.Package, p) - } - } - groups := internal.MatchNamedCaptureGroups(upstreamPattern, m.Package) upstream, ok := groups["upstream"] - if ok { - return upstream + if !ok { + upstream = m.Package + } + + for _, p := range prefixes { + if strings.HasPrefix(upstream, p) { + return strings.TrimPrefix(upstream, p) + } } - return m.Package + return upstream } diff --git a/syft/pkg/apk_metadata_test.go b/syft/pkg/apk_metadata_test.go index f04e7a09eb2..610d8321a49 100644 --- a/syft/pkg/apk_metadata_test.go +++ b/syft/pkg/apk_metadata_test.go @@ -303,6 +303,20 @@ func TestApkMetadata_Upstream(t *testing.T) { }, expected: "123456", }, + { + name: "ruby-3.6 upstream ruby", + metadata: ApkMetadata{ + Package: "ruby-3.6", + }, + expected: "ruby", + }, + { + name: "ruby3.6 upstream ruby", + metadata: ApkMetadata{ + Package: "ruby3.6", + }, + expected: "ruby", + }, } for _, test := range tests {