Skip to content

Commit

Permalink
Added tests for version matchers
Browse files Browse the repository at this point in the history 
Signed-off-by: Laurent Goderre <laurent.goderre@docker.com>
  • Loading branch information
@LaurentGoderre
LaurentGoderre committed May 21, 2024
1 parent 89d13a3 commit ded5922
Show file tree
Hide file tree
Showing 4 changed files with 86 additions and 30 deletions.
71 changes: 43 additions & 28 deletions syft/pkg/cataloger/binary/classifier.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,18 @@ func (cfg Classifier) MarshalJSON() ([]byte, error) {
}

// EvidenceMatcher is a function called to catalog Packages that match some sort of evidence
type EvidenceMatcher func(resolver file.Resolver, classifier Classifier, location file.Location) ([]pkg.Package, error)
type EvidenceMatcher func(classifier Classifier, context matcherContext) ([]pkg.Package, error)

type matcherContext struct {
resolver file.Resolver
location file.Location
getContents func(resolver matcherContext) ([]byte, error)
}

func evidenceMatchers(matchers ...EvidenceMatcher) EvidenceMatcher {
return func(resolver file.Resolver, classifier Classifier, location file.Location) ([]pkg.Package, error) {
return func(classifier Classifier, context matcherContext) ([]pkg.Package, error) {
for _, matcher := range matchers {
match, err := matcher(resolver, classifier, location)
match, err := matcher(classifier, context)
if err != nil {
return nil, err
}
Expand All @@ -90,12 +96,12 @@ func evidenceMatchers(matchers ...EvidenceMatcher) EvidenceMatcher {

func fileNameTemplateVersionMatcher(fileNamePattern string, contentTemplate string) EvidenceMatcher {
pat := regexp.MustCompile(fileNamePattern)
return func(resolver file.Resolver, classifier Classifier, location file.Location) ([]pkg.Package, error) {
if !pat.MatchString(location.RealPath) {
return func(classifier Classifier, context matcherContext) ([]pkg.Package, error) {
if !pat.MatchString(context.location.RealPath) {
return nil, nil
}

filepathNamedGroupValues := internal.MatchNamedCaptureGroups(pat, location.RealPath)
filepathNamedGroupValues := internal.MatchNamedCaptureGroups(pat, context.location.RealPath)

// versions like 3.5 should not match any character, but explicit dot
for k, v := range filepathNamedGroupValues {
Expand All @@ -118,14 +124,14 @@ func fileNameTemplateVersionMatcher(fileNamePattern string, contentTemplate stri
return nil, fmt.Errorf("unable to compile rendered regex=%q: %w", patternBuf.String(), err)
}

contents, err := getContents(resolver, location)
contents, err := getContents(context)
if err != nil {
return nil, fmt.Errorf("unable to get read contents for file: %w", err)
}

matchMetadata := internal.MatchNamedCaptureGroups(tmplPattern, string(contents))

p := newClassifierPackage(classifier, location, matchMetadata)
p := newClassifierPackage(classifier, context.location, matchMetadata)
if p == nil {
return nil, nil
}
Expand All @@ -136,8 +142,8 @@ func fileNameTemplateVersionMatcher(fileNamePattern string, contentTemplate stri

func FileContentsVersionMatcher(pattern string) EvidenceMatcher {
pat := regexp.MustCompile(pattern)
return func(resolver file.Resolver, classifier Classifier, location file.Location) ([]pkg.Package, error) {
contents, err := getContents(resolver, location)
return func(classifier Classifier, context matcherContext) ([]pkg.Package, error) {
contents, err := getContents(context)
if err != nil {
return nil, fmt.Errorf("unable to get read contents for file: %w", err)
}
Expand All @@ -160,7 +166,7 @@ func FileContentsVersionMatcher(pattern string) EvidenceMatcher {
}
}

p := newClassifierPackage(classifier, location, matchMetadata)
p := newClassifierPackage(classifier, context.location, matchMetadata)
if p == nil {
return nil, nil
}
Expand All @@ -176,8 +182,8 @@ func matchExcluding(matcher EvidenceMatcher, contentPatternsToExclude ...string)
for _, p := range contentPatternsToExclude {
nonMatchPatterns = append(nonMatchPatterns, regexp.MustCompile(p))
}
return func(resolver file.Resolver, classifier Classifier, location file.Location) ([]pkg.Package, error) {
contents, err := getContents(resolver, location)
return func(classifier Classifier, context matcherContext) ([]pkg.Package, error) {
contents, err := getContents(context)
if err != nil {
return nil, fmt.Errorf("unable to get read contents for file: %w", err)
}
Expand All @@ -186,15 +192,15 @@ func matchExcluding(matcher EvidenceMatcher, contentPatternsToExclude ...string)
return nil, nil
}
}
return matcher(resolver, classifier, location)
return matcher(classifier, context)
}
}

//nolint:gocognit
func sharedLibraryLookup(sharedLibraryPattern string, sharedLibraryMatcher EvidenceMatcher) EvidenceMatcher {
pat := regexp.MustCompile(sharedLibraryPattern)
return func(resolver file.Resolver, classifier Classifier, location file.Location) (packages []pkg.Package, _ error) {
libs, err := sharedLibraries(resolver, location)
return func(classifier Classifier, context matcherContext) (packages []pkg.Package, _ error) {
libs, err := sharedLibraries(context)
if err != nil {
return nil, err
}
Expand All @@ -203,26 +209,32 @@ func sharedLibraryLookup(sharedLibraryPattern string, sharedLibraryMatcher Evide
continue
}

locations, err := resolver.FilesByGlob("**/" + lib)
locations, err := context.resolver.FilesByGlob("**/" + lib)
if err != nil {
return nil, err
}
for _, libraryLocation := range locations {
pkgs, err := sharedLibraryMatcher(resolver, classifier, libraryLocation)
newResolver := matcherContext{
resolver: context.resolver,
location: libraryLocation,
getContents: context.getContents,
}
newResolver.location = libraryLocation
pkgs, err := sharedLibraryMatcher(classifier, newResolver)
if err != nil {
return nil, err
}
for _, p := range pkgs {
// set the source binary as the first location
locationSet := file.NewLocationSet(location)
locationSet := file.NewLocationSet(context.location)
locationSet.Add(p.Locations.ToSlice()...)
p.Locations = locationSet
meta, _ := p.Metadata.(pkg.BinarySignature)
p.Metadata = pkg.BinarySignature{
Matches: append([]pkg.ClassifierMatch{
{
Classifier: classifier.Class,
Location: location,
Location: context.location,
},
}, meta.Matches...),
}
Expand All @@ -242,12 +254,15 @@ func mustPURL(purl string) packageurl.PackageURL {
return p
}

func getContents(resolver file.Resolver, location file.Location) ([]byte, error) {
reader, err := resolver.FileContentsByLocation(location)
func getContents(context matcherContext) ([]byte, error) {
if context.getContents != nil {
return context.getContents(context)
}
reader, err := context.resolver.FileContentsByLocation(context.location)
if err != nil {
return nil, err
}
defer internal.CloseAndLogError(reader, location.AccessPath)
defer internal.CloseAndLogError(reader, context.location.AccessPath)

// TODO: there may be room for improvement here, as this may use an excessive amount of memory. Alternate approach is to leverage a RuneReader.
contents, err := io.ReadAll(reader)
Expand All @@ -268,8 +283,8 @@ func singleCPE(cpeString string) []cpe.CPE {

// sharedLibraries returns a list of all shared libraries found within a binary, currently
// supporting: elf, macho, and windows pe
func sharedLibraries(resolver file.Resolver, location file.Location) ([]string, error) {
contents, err := getContents(resolver, location)
func sharedLibraries(context matcherContext) ([]string, error) {
contents, err := getContents(context)
if err != nil {
return nil, err
}
Expand All @@ -280,7 +295,7 @@ func sharedLibraries(resolver file.Resolver, location file.Location) ([]string,
if e != nil {
symbols, err := e.ImportedLibraries()
if err != nil {
log.Debugf("unable to read elf binary at: %s -- %s", location.RealPath, err)
log.Debugf("unable to read elf binary at: %s -- %s", context.location.RealPath, err)
}
return symbols, nil
}
Expand All @@ -289,7 +304,7 @@ func sharedLibraries(resolver file.Resolver, location file.Location) ([]string,
if m != nil {
symbols, err := m.ImportedLibraries()
if err != nil {
log.Debugf("unable to read macho binary at: %s -- %s", location.RealPath, err)
log.Debugf("unable to read macho binary at: %s -- %s", context.location.RealPath, err)
}
return symbols, nil
}
Expand All @@ -298,7 +313,7 @@ func sharedLibraries(resolver file.Resolver, location file.Location) ([]string,
if p != nil {
symbols, err := p.ImportedLibraries()
if err != nil {
log.Debugf("unable to read pe binary at: %s -- %s", location.RealPath, err)
log.Debugf("unable to read pe binary at: %s -- %s", context.location.RealPath, err)
}
return symbols, nil
}
Expand Down
2 changes: 1 addition & 1 deletion syft/pkg/cataloger/binary/classifier_cataloger.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ func catalog(resolver file.Resolver, cls Classifier) (packages []pkg.Package, er
return nil, err
}
for _, location := range locations {
pkgs, err := cls.EvidenceMatcher(resolver, cls, location)
pkgs, err := cls.EvidenceMatcher(cls, matcherContext{resolver: resolver, location: location})
if err != nil {
return nil, err
}
Expand Down
43 changes: 42 additions & 1 deletion syft/pkg/cataloger/binary/classifier_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ func Test_ClassifierCPEs(t *testing.T) {
require.NoError(t, err)
require.Len(t, ls, 1)

pkgs, err := test.classifier.EvidenceMatcher(resolver, test.classifier, ls[0])
pkgs, err := test.classifier.EvidenceMatcher(test.classifier, matcherContext{resolver: resolver, location: ls[0]})
require.NoError(t, err)

require.Len(t, pkgs, 1)
Expand Down Expand Up @@ -139,3 +139,44 @@ func TestClassifier_MarshalJSON(t *testing.T) {
})
}
}

func TestFileContentsVersionMatcher(t *testing.T) {
tests := []struct {
name string
pattern string
data string
expected string
}{
{
name: "simple version string regexp",
pattern: `some data (?P<version>[0-9]+\.[0-9]+\.[0-9]+) some data`,
data: "some data 1.2.3 some data",
expected: "1.2.3",
},
{
name: "version parts regexp",
pattern: `\x00\x23(?P<major>[0-9]+)\x00\x23(?P<minor>[0-9]+)\x00\x23(?P<patch>[0-9]+)\x00\x23`,
data: "\x00\x239\x00\x239\x00\x239\x00\x23",
expected: "9.9.9",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mockGetContent := func(context matcherContext) ([]byte, error) {
return []byte(tt.data), nil
}
fn := FileContentsVersionMatcher(tt.pattern)
p, err := fn(Classifier{}, matcherContext{
getContents: mockGetContent,
})

if err != nil {
t.Errorf("Unexpected error %#v", err)
}

if p[0].Version != tt.expected {
t.Errorf("Versions don't match.\ngot\n%q\n\nexpected\n%q", p[0].Version, tt.expected)
}
})
}
}
Binary file modified syft/pkg/cataloger/binary/test-fixtures/version-parts.txt
View file
Binary file not shown.

0 comments on commit ded5922

Please sign in to comment.