Skip to content

Commit

Permalink
port javascript cataloger to new generic cataloger pattern (#1308)
Browse files Browse the repository at this point in the history 
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
  • Loading branch information
@wagoodman
wagoodman committed Nov 2, 2022
1 parent 35f0f29 commit 9634b42
Show file tree
Hide file tree
Showing 17 changed files with 685 additions and 650 deletions.
20 changes: 1 addition & 19 deletions syft/pkg/catalog.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package pkg

import (
"sort"
"sync"

"github.com/jinzhu/copier"
Expand Down Expand Up @@ -199,24 +198,7 @@ func (c *Catalog) Sorted(types ...Type) (pkgs []Package) {
pkgs = append(pkgs, p)
}

sort.SliceStable(pkgs, func(i, j int) bool {
if pkgs[i].Name == pkgs[j].Name {
if pkgs[i].Version == pkgs[j].Version {
iLocations := pkgs[i].Locations.ToSlice()
jLocations := pkgs[j].Locations.ToSlice()
if pkgs[i].Type == pkgs[j].Type && len(iLocations) > 0 && len(jLocations) > 0 {
if iLocations[0].String() == jLocations[0].String() {
// compare IDs as a final fallback
return pkgs[i].ID() < pkgs[j].ID()
}
return iLocations[0].String() < jLocations[0].String()
}
return pkgs[i].Type < pkgs[j].Type
}
return pkgs[i].Version < pkgs[j].Version
}
return pkgs[i].Name < pkgs[j].Name
})
Sort(pkgs)

return pkgs
}
13 changes: 13 additions & 0 deletions syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,19 @@ func NewCatalogTester() *CatalogTester {
}
}

func (p *CatalogTester) FromDirectory(t *testing.T, path string) *CatalogTester {
t.Helper()

s, err := source.NewFromDirectory(path)
require.NoError(t, err)

resolver, err := s.FileResolver(source.AllLayersScope)
require.NoError(t, err)

p.resolver = resolver
return p
}

func (p *CatalogTester) FromFile(t *testing.T, path string) *CatalogTester {
t.Helper()

Expand Down
80 changes: 8 additions & 72 deletions syft/pkg/cataloger/javascript/cataloger.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,88 +4,24 @@ Package javascript provides a concrete Cataloger implementation for JavaScript e
package javascript

import (
"encoding/json"
"io"
"path"
"strings"

"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/source"
)

// NewJavascriptPackageCataloger returns a new JavaScript cataloger object based on detection of npm based packages.
func NewJavascriptPackageCataloger() *common.GenericCataloger {
globParsers := map[string]common.ParserFn{
"**/package.json": parsePackageJSON,
}

return common.NewGenericCataloger(nil, globParsers, "javascript-package-cataloger")
func NewJavascriptPackageCataloger() *generic.Cataloger {
return generic.NewCataloger("javascript-package-cataloger").
WithParserByGlobs(parsePackageJSON, "**/package.json")
}

// NewJavascriptLockCataloger returns a new Javascript cataloger object base on package lock files.
func NewJavascriptLockCataloger() *common.GenericCataloger {
globParsers := map[string]common.ParserFn{
"**/package-lock.json": parsePackageLock,
"**/yarn.lock": parseYarnLock,
"**/pnpm-lock.yaml": parsePnpmLock,
}

return common.NewGenericCataloger(nil, globParsers, "javascript-lock-cataloger", addLicenses)
func NewJavascriptLockCataloger() *generic.Cataloger {
return generic.NewCataloger("javascript-lock-cataloger").
WithParserByGlobs(parsePackageLock, "**/package-lock.json").
WithParserByGlobs(parseYarnLock, "**/yarn.lock").
WithParserByGlobs(parsePnpmLock, "**/pnpm-lock.yaml")
}

func NewNodeBinaryCataloger() *generic.Cataloger {
return generic.NewCataloger("node-binary-cataloger").
WithParserByMimeTypes(parseNodeBinary, internal.ExecutableMIMETypeSet.List()...)
}

func addLicenses(resolver source.FileResolver, location source.Location, p *pkg.Package) error {
dir := path.Dir(location.RealPath)
pkgPath := []string{dir, "node_modules"}
pkgPath = append(pkgPath, strings.Split(p.Name, "/")...)
pkgPath = append(pkgPath, "package.json")
pkgFile := path.Join(pkgPath...)
locations, err := resolver.FilesByPath(pkgFile)
if err != nil {
log.Debugf("an error occurred attempting to read: %s - %+v", pkgFile, err)
return nil
}

if len(locations) == 0 {
return nil
}

for _, location := range locations {
contentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
log.Debugf("error getting file content reader for %s: %v", pkgFile, err)
return nil
}

contents, err := io.ReadAll(contentReader)
if err != nil {
log.Debugf("error reading file contents for %s: %v", pkgFile, err)
return nil
}

var pkgJSON packageJSON
err = json.Unmarshal(contents, &pkgJSON)
if err != nil {
log.Debugf("error parsing %s: %v", pkgFile, err)
return nil
}

licenses, err := pkgJSON.licensesFromJSON()
if err != nil {
log.Debugf("error getting licenses from %s: %v", pkgFile, err)
return nil
}

p.Licenses = append(p.Licenses, licenses...)
}

return nil
}
174 changes: 97 additions & 77 deletions syft/pkg/cataloger/javascript/cataloger_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,100 +3,120 @@ package javascript
import (
"testing"

"github.com/stretchr/testify/require"

"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
"github.com/anchore/syft/syft/source"
)

func Test_JavascriptCataloger(t *testing.T) {
expected := map[string]pkg.Package{
"@actions/core": {
Name: "@actions/core",
Version: "1.6.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Licenses: []string{"MIT"},
locationSet := source.NewLocationSet(source.NewLocation("package-lock.json"))
expectedPkgs := []pkg.Package{
{
Name: "@actions/core",
Version: "1.6.0",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/%40actions/core@1.6.0",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Licenses: []string{"MIT"},
},
"wordwrap": {
Name: "wordwrap",
Version: "0.0.3",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "ansi-regex",
Version: "3.0.0",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/ansi-regex@3.0.0",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"get-stdin": {
Name: "get-stdin",
Version: "5.0.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "cowsay",
Version: "1.4.0",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/cowsay@1.4.0",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Licenses: []string{"MIT"},
},
"minimist": {
Name: "minimist",
Version: "0.0.10",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "get-stdin",
Version: "5.0.1",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/get-stdin@5.0.1",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"optimist": {
Name: "optimist",
Version: "0.6.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "is-fullwidth-code-point",
Version: "2.0.0",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/is-fullwidth-code-point@2.0.0",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"string-width": {
Name: "string-width",
Version: "2.1.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "minimist",
Version: "0.0.10",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/minimist@0.0.10",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"strip-ansi": {
Name: "strip-ansi",
Version: "4.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "optimist",
Version: "0.6.1",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/optimist@0.6.1",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"strip-eof": {
Name: "wordwrap",
Version: "1.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "string-width",
Version: "2.1.1",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/string-width@2.1.1",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"ansi-regex": {
Name: "ansi-regex",
Version: "3.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "strip-ansi",
Version: "4.0.0",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/strip-ansi@4.0.0",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"is-fullwidth-code-point": {
Name: "is-fullwidth-code-point",
Version: "2.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "strip-eof",
Version: "1.0.0",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/strip-eof@1.0.0",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"cowsay": {
Name: "cowsay",
Version: "1.4.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Licenses: []string{"MIT"},
{
Name: "wordwrap",
Version: "0.0.3",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/wordwrap@0.0.3",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
}

s, err := source.NewFromDirectory("test-fixtures/pkg-lock")
require.NoError(t, err)

resolver, err := s.FileResolver(source.AllLayersScope)
require.NoError(t, err)

actual, _, err := NewJavascriptLockCataloger().Catalog(resolver)
if err != nil {
t.Fatalf("failed to parse package-lock.json: %+v", err)
}

var pkgs []*pkg.Package
for _, p := range actual {
p2 := p
pkgs = append(pkgs, &p2)
}
pkgtest.NewCatalogTester().
FromDirectory(t, "test-fixtures/pkg-lock").
Expects(expectedPkgs, nil).
TestCataloger(t, NewJavascriptLockCataloger())

assertPkgsEqual(t, pkgs, expected)
}
Loading

0 comments on commit 9634b42

Please sign in to comment.