Merge pull request #20 from nearform/support-package-json

Support package-lock.json Walker

.travis.yml

@@ -7,7 +7,7 @@ sudo: required
 
 services:
   - docker
-  
+
 before_install:
   - export PATH="$PATH:$GOPATH/bin"
   - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
@@ -20,5 +20,5 @@ notifications:
   email: false
 
 script:
-  - make test
+  - make verbose-test
   - $GOPATH/bin/goveralls -service=travis-ci

Makefile

@@ -20,19 +20,24 @@ build-test-docker-images:
 	docker build test_data/insecure-project/ -t gammaray-test-insecure-project:1.0.0
 
 dev-install: install build-test-docker-images
+	go get -u github.com/kyoh86/richgo
 	go get -u github.com/mgechev/revive
 	go get -u github.com/mna/pigeon
 
 ci-install: dev-install
 	go get -u github.com/mattn/goveralls
 
+verbose-test:
+	@richgo test -v -race ./... | sed ''/'PASS |'/s//`printf "✅\033[32mPASS\033[0m"`/'' | sed ''/'FAIL |'/s//`printf "❌\033[31mFAIL\033[0m"`/''
+	@go vet ./...
+
 test:
-	go test -v -race ./...
-	go vet ./...
+	@richgo test -v -race ./... | grep -v -P "     \|\s(?!ok)"| sed ''/'PASS |'/s//`printf "✅\033[32mPASS\033[0m"`/'' | sed ''/'FAIL |'/s//`printf "❌\033[31mFAIL\033[0m"`/''
+	@go vet ./...
 
 coverage:
-	go test -v -race -cover ./...
-	go vet ./...
+	@richgo test -v -race -cover ./... | grep -P "START|SKIP |PASS |FAIL |COVER|ok  	github.com/nearform/gammaray" | sed ''/'PASS | '/s//`printf "✅\033[32mPASS\033[0m"`/'' | sed ''/'FAIL | '/s//`printf "❌\033[31mFAIL\033[0m"`/'' | sed ''/'COVER|'/s//`printf "\033[34mCOVER\033[0m"`/'' | sed ''/'SKIP |'/s//`printf "⚠️\033[36m.SKIP\033[0m"`/''
+	@go vet ./...
 
 ci-test: test
 	goveralls

analyzer/analyzer.go

@@ -1,9 +1,12 @@
 package analyzer
 
 import (
+	"errors"
 	"fmt"
 	"log"
 
+	"github.com/nearform/gammaray/nodepackage"
+	"github.com/nearform/gammaray/packagelockrunner"
 	"github.com/nearform/gammaray/pathrunner"
 	"github.com/nearform/gammaray/vulnfetcher"
 	"github.com/nearform/gammaray/vulnfetcher/nodeswg"
@@ -14,16 +17,36 @@ import (
 const OSSIndexURL = "https://ossindex.net/api/v3/component-report"
 const nodeswgURL = "https://github.com/nodejs/security-wg/archive/master.zip"
 
-// Analyze analyzes a path to an installed (npm install) node package
-func Analyze(path string) (vulnfetcher.VulnerabilityReport, error) {
-	fmt.Println("Will scan folder <", path, ">")
-	packageList, err := pathrunner.Walk(path)
-	if err != nil {
-		return nil, err
+func runWalkers(path string, walkers []nodepackage.Walker) ([]nodepackage.NodePackage, error) {
+	var errs []error
+	var mainPackage []nodepackage.NodePackage
+	for _, walker := range walkers {
+		packageList, err := walker.Walk(path)
+		if packageList != nil {
+			if len(packageList) > 1 {
+				return packageList, nil
+			}
+			// only found the main package, but no dependency using this method
+			// just continue with other ways to check if we find more
+			// can happen for example with pathrunner, if no 'npm i' has been done
+			mainPackage = packageList
+		}
+		if err != nil {
+			fmt.Println("⚠️", walker.ErrorContext(err), ":\n", err)
+			errs = append(errs, err)
+		}
 	}
+	if mainPackage != nil {
+		return mainPackage, nil
+	}
+	if len(errs) == len(walkers) {
+		return nil, errors.New("could not find any dependencies and all strategies to find them failed")
+	}
+	return nil, nil
+}
 
-	// keep only valid packages
-	var packages []pathrunner.NodePackage
+func packagesCleanupAndDeduplication(packageList []nodepackage.NodePackage) []nodepackage.NodePackage {
+	packageMap := make(map[string]nodepackage.NodePackage)
 	for _, pkg := range packageList {
 		if pkg.Name == "" {
 			log.Print("Ignoring package with empty name")
@@ -32,8 +55,33 @@ func Analyze(path string) (vulnfetcher.VulnerabilityReport, error) {
 		if pkg.Version == "" {
 			pkg.Version = "*"
 		}
+		packageMap[pkg.Name+"@"+pkg.Version] = pkg
+	}
+
+	var packages []nodepackage.NodePackage
+	for _, pkg := range packageMap {
 		packages = append(packages, pkg)
 	}
+	return packages
+}
+
+// Analyze analyzes a path to an installed (npm install) node package
+func Analyze(path string, walkers ...nodepackage.Walker) (vulnfetcher.VulnerabilityReport, error) {
+	fmt.Println("Will scan folder <", path, ">")
+	if walkers == nil {
+		walkers = []nodepackage.Walker{
+			pathrunner.PathRunner{},
+			packagelockrunner.PackageLockRunner{},
+		}
+	}
+
+	packageList, err := runWalkers(path, walkers)
+	if err != nil {
+		return nil, err
+	}
+
+	// keep only valid packages
+	packages := packagesCleanupAndDeduplication(packageList)
 
 	ossFetcher := ossvulnfetcher.New(OSSIndexURL)
 	err = ossFetcher.Fetch()

analyzer/analyzer_test.go

@@ -44,7 +44,7 @@ func TestNotExistingProject(t *testing.T) {
 	if err == nil {
 		t.Errorf("TestNotExistingProject: ./does-not-exist does not exist, it should not be analyzed !")
 	}
-	if diff := cmp.Diff(err.Error(), "stat ./does-not-exist: no such file or directory"); diff != "" {
+	if diff := cmp.Diff(err.Error(), "could not find any dependencies and all strategies to find them failed"); diff != "" {
 		t.Errorf("TestNotExistingProject: err : (-got +want)\n%s", diff)
 	}
 

docker/docker.go

@@ -15,6 +15,7 @@ import (
 	docker "github.com/docker/docker/client"
 	unarr "github.com/gen2brain/go-unarr"
 	"github.com/nearform/gammaray/analyzer"
+	"github.com/nearform/gammaray/nodepackage"
 	"github.com/nearform/gammaray/vulnfetcher"
 	"golang.org/x/net/context"
 )
@@ -151,7 +152,7 @@ func readImageConfig(imageFolder string, manifest *DockerImageFiles) (*DockerCon
 }
 
 // ScanImage extracts an image and analyzes its layers
-func ScanImage(imageName string, projectPath string) (vulnfetcher.VulnerabilityReport, error) {
+func ScanImage(imageName string, projectPath string, walkers ...nodepackage.Walker) (vulnfetcher.VulnerabilityReport, error) {
 	ctx := context.Background()
 	cli, err := docker.NewEnvClient()
 	if err != nil {
@@ -195,5 +196,5 @@ func ScanImage(imageName string, projectPath string) (vulnfetcher.VulnerabilityR
 	}
 
 	fmt.Println("Analyze image stored in <", imageProjectPath, ">")
-	return analyzer.Analyze(path.Join(imageFolder, "snapshot", imageProjectPath))
+	return analyzer.Analyze(path.Join(imageFolder, "snapshot", imageProjectPath), walkers...)
 }

docker/docker_test.go

@@ -2,6 +2,7 @@ package docker
 
 import (
 	"context"
+	"fmt"
 	"log"
 	"os"
 	"path"
@@ -45,6 +46,13 @@ func TestScanImageInsecureProject(t *testing.T) {
 	}
 }
 
+func TestScanImageNotExisting(t *testing.T) {
+	_, err := ScanImage("gammaray-test-this-one-should not-exist", "")
+	if err == nil {
+		panic(fmt.Errorf("'gammaray-test-this-one-should not-exist' should not exist and create an error"))
+	}
+}
+
 func TestPullImageIfNecessaryOfficialNodeAlpine(t *testing.T) {
 	ctx := context.Background()
 	cli, err := docker.NewEnvClient()

main.go

@@ -8,22 +8,26 @@ import (
 	"github.com/jaffee/commandeer"
 	"github.com/nearform/gammaray/analyzer"
 	"github.com/nearform/gammaray/docker"
+	"github.com/nearform/gammaray/nodepackage"
+	"github.com/nearform/gammaray/packagelockrunner"
 	"github.com/nearform/gammaray/vulnfetcher"
 )
 
 // Args CLI arguments
 type Args struct {
-	Path    string `help:"path to installed Node package (locally or inside the container, depending if 'image' is provided)"`
-	Image   string `help:"analyze this docker image"`
-	LogFile string `help:"in which file to put the detailed logs"`
+	Path            string `help:"path to installed Node package (locally or inside the container, depending if 'image' is provided)"`
+	Image           string `help:"analyze this docker image"`
+	LogFile         string `help:"in which file to put the detailed logs"`
+	OnlyPackageLock bool   `help:"force <package-lock.json> usage (false: use it as a fallback)"`
 }
 
 // Defaults generate default CLI values
 func Defaults() *Args {
 	return &Args{
-		Path:    "",
-		Image:   "",
-		LogFile: ".gammaray.log",
+		Path:            "",
+		Image:           "",
+		LogFile:         ".gammaray.log",
+		OnlyPackageLock: false,
 	}
 }
 
@@ -52,21 +56,26 @@ func (m *Args) Run() error {
 
 // Analyze the path or docker image for vulnerabilities
 func (m *Args) Analyze() (vulnfetcher.VulnerabilityReport, error) {
-
+	var walkers []nodepackage.Walker
+	if m.OnlyPackageLock == true {
+		walkers = []nodepackage.Walker{
+			packagelockrunner.PackageLockRunner{},
+		}
+	}
 	if m.Image == "" && m.Path != "" {
-		return analyzer.Analyze(m.Path)
+		return analyzer.Analyze(m.Path, walkers...)
 	} else if m.Image != "" {
 		if m.Path != "" {
 			fmt.Println("Will scan folder <", m.Path, "> from docker image <", m.Image, ">")
 		} else {
 			fmt.Println("Will scan docker image <", m.Image, ">")
 		}
 
-		return docker.ScanImage(m.Image, m.Path)
+		return docker.ScanImage(m.Image, m.Path, walkers...)
 	} else if len(os.Args) > 1 {
 		lastArg := os.Args[len(os.Args)-1]
 		fmt.Println("⚠ Will use the last argument <", lastArg, "> as '-path' value.")
-		return analyzer.Analyze(lastArg)
+		return analyzer.Analyze(lastArg, walkers...)
 	}
 	return nil, fmt.Errorf("you need to at least properly define a path or a docker image")
 }

main_test.go

@@ -23,7 +23,8 @@ func TestHelloWorld(t *testing.T) {
 		Path:    "test_data/hello-world",
 	}).Run()
 	if err != nil {
-		panic(err)
+		t.Error(err)
+		return
 	}
 	info, error := os.Stat(testLogFile)
 	if error != nil {
@@ -45,7 +46,8 @@ func TestInsecureProject(t *testing.T) {
 		Path:    "test_data/insecure-project",
 	}).Run()
 	if err != nil {
-		panic(err)
+		t.Error(err)
+		return
 	}
 	info, error := os.Stat(testLogFile)
 	if error != nil {
@@ -71,7 +73,7 @@ func TestPathDoesNotExist(t *testing.T) {
 		t.Errorf("TestPathDoesNotExist: there should be an error when trying to analyze ./does-not-exist")
 	}
 
-	if diff := cmp.Diff(err.Error(), "stat ./does-not-exist: no such file or directory"); diff != "" {
+	if diff := cmp.Diff(err.Error(), "could not find any dependencies and all strategies to find them failed"); diff != "" {
 		t.Errorf("TestPathDoesNotExist: error : (-got +want)\n%s", diff)
 	}
 }
@@ -147,3 +149,27 @@ func TestMainHelloWorld(t *testing.T) {
 		t.Errorf("TestMainHelloWorld: log size should not be 0!")
 	}
 }
+
+func TestRunHelloWorld(t *testing.T) {
+	defer cleanLogs()
+
+	a := Args{Path: "test_data/hello-world"}
+
+	err := a.Run()
+	if err != nil {
+		t.Error(err)
+	}
+
+}
+
+func TestRunBadLogFile(t *testing.T) {
+	defer cleanLogs()
+
+	a := Args{Path: "test_data/hello-world", LogFile: "/dev/null"}
+
+	err := a.Run()
+	if err != nil {
+		t.Error(err)
+	}
+
+}

nodepackage/nodepackage.go

@@ -0,0 +1,13 @@
+package nodepackage
+
+// NodePackage represents a package.json (only the interesting fields)
+type NodePackage struct {
+	Name    string `json:"name"`
+	Version string `json:"version"`
+}
+
+// Walker interface is used to allow multiple backends using this interface to find out dependencies for a given dir
+type Walker interface {
+	Walk(dir string) ([]NodePackage, error)
+	ErrorContext(error) string
+}