implement nodejs feature detector, namespace detector

Signed-off-by: liang chenye <liangchenye@huawei.com>

cmd/clair/main.go

@@ -38,10 +38,12 @@ import (
 	_ "github.com/coreos/clair/worker/detectors/data/docker"
 
 	_ "github.com/coreos/clair/worker/detectors/feature/dpkg"
+	_ "github.com/coreos/clair/worker/detectors/feature/npm"
 	_ "github.com/coreos/clair/worker/detectors/feature/rpm"
 
 	_ "github.com/coreos/clair/worker/detectors/namespace/aptsources"
 	_ "github.com/coreos/clair/worker/detectors/namespace/lsbrelease"
+	_ "github.com/coreos/clair/worker/detectors/namespace/nodejs"
 	_ "github.com/coreos/clair/worker/detectors/namespace/osrelease"
 	_ "github.com/coreos/clair/worker/detectors/namespace/redhatrelease"
 

utils/tar.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"io/ioutil"
 	"os/exec"
+	"regexp"
 	"strings"
 )
 
@@ -94,7 +95,7 @@ func (r *TarReadCloser) Close() error {
 
 // SelectivelyExtractArchive extracts the specified files and folders
 // from targz data read from the given reader and store them in a map indexed by file paths
-func SelectivelyExtractArchive(r io.Reader, prefix string, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
+func SelectivelyExtractArchive(r io.Reader, prefix string, toExtract []*regexp.Regexp, maxFileSize int64) (map[string][]byte, error) {
 	data := make(map[string][]byte)
 
 	// Create a tar or tar/tar-gzip/tar-bzip2/tar-xz reader
@@ -123,8 +124,8 @@ func SelectivelyExtractArchive(r io.Reader, prefix string, toExtract []string, m
 
 		// Determine if we should extract the element
 		toBeExtracted := false
-		for _, s := range toExtract {
-			if strings.HasPrefix(filename, s) {
+		for _, re := range toExtract {
+			if re.MatchString(filename) {
 				toBeExtracted = true
 				break
 			}

utils/utils_test.go

@@ -18,6 +18,7 @@ import (
 	"bytes"
 	"os"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"testing"
 
@@ -71,13 +72,13 @@ func TestTar(t *testing.T) {
 		testArchivePath := filepath.Join(filepath.Dir(path), testDataDir, filename)
 
 		// Extract non compressed data
-		data, err = SelectivelyExtractArchive(bytes.NewReader([]byte("that string does not represent a tar or tar-gzip file")), "", []string{}, 0)
+		data, err = SelectivelyExtractArchive(bytes.NewReader([]byte("that string does not represent a tar or tar-gzip file")), "", []*regexp.Regexp{}, 0)
 		assert.Error(t, err, "Extracting non compressed data should return an error")
 
 		// Extract an archive
 		f, _ := os.Open(testArchivePath)
 		defer f.Close()
-		data, err = SelectivelyExtractArchive(f, "", []string{"test/"}, 0)
+		data, err = SelectivelyExtractArchive(f, "", []*regexp.Regexp{regexp.MustCompile("^test/")}, 0)
 		assert.Nil(t, err)
 
 		if c, n := data["test/test.txt"]; !n {
@@ -92,7 +93,7 @@ func TestTar(t *testing.T) {
 		// File size limit
 		f, _ = os.Open(testArchivePath)
 		defer f.Close()
-		data, err = SelectivelyExtractArchive(f, "", []string{"test"}, 50)
+		data, err = SelectivelyExtractArchive(f, "", []*regexp.Regexp{regexp.MustCompile("test")}, 50)
 		assert.Equal(t, ErrExtractedFileTooBig, err)
 	}
 }

worker/detectors/data.go

@@ -22,6 +22,7 @@ import (
 	"math"
 	"net/http"
 	"os"
+	"regexp"
 	"strings"
 	"sync"
 
@@ -34,7 +35,7 @@ type DataDetector interface {
 	//Support check if the input path and format are supported by the underling detector
 	Supported(path string, format string) bool
 	// Detect detects the required data from input path
-	Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (data map[string][]byte, err error)
+	Detect(layerReader io.ReadCloser, toExtract []*regexp.Regexp, maxFileSize int64) (data map[string][]byte, err error)
 }
 
 var (
@@ -70,7 +71,7 @@ func RegisterDataDetector(name string, f DataDetector) {
 }
 
 // DetectData finds the Data of the layer by using every registered DataDetector
-func DetectData(format, path string, headers map[string]string, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) {
+func DetectData(format, path string, headers map[string]string, toExtract []*regexp.Regexp, maxFileSize int64) (data map[string][]byte, err error) {
 	var layerReader io.ReadCloser
 	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
 		// Create a new HTTP request object.

worker/detectors/data/aci/aci.go

@@ -16,6 +16,7 @@ package aci
 
 import (
 	"io"
+	"regexp"
 	"strings"
 
 	"github.com/coreos/clair/utils"
@@ -36,6 +37,6 @@ func (detector *ACIDataDetector) Supported(path string, format string) bool {
 	return false
 }
 
-func (detector *ACIDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
+func (detector *ACIDataDetector) Detect(layerReader io.ReadCloser, toExtract []*regexp.Regexp, maxFileSize int64) (map[string][]byte, error) {
 	return utils.SelectivelyExtractArchive(layerReader, "rootfs/", toExtract, maxFileSize)
 }

worker/detectors/data/docker/docker.go

@@ -16,6 +16,7 @@ package docker
 
 import (
 	"io"
+	"regexp"
 	"strings"
 
 	"github.com/coreos/clair/utils"
@@ -36,6 +37,6 @@ func (detector *DockerDataDetector) Supported(path string, format string) bool {
 	return false
 }
 
-func (detector *DockerDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
+func (detector *DockerDataDetector) Detect(layerReader io.ReadCloser, toExtract []*regexp.Regexp, maxFileSize int64) (map[string][]byte, error) {
 	return utils.SelectivelyExtractArchive(layerReader, "", toExtract, maxFileSize)
 }

worker/detectors/feature/dpkg/dpkg.go

@@ -30,6 +30,7 @@ var (
 
 	dpkgSrcCaptureRegexp      = regexp.MustCompile(`Source: (?P<name>[^\s]*)( \((?P<version>.*)\))?`)
 	dpkgSrcCaptureRegexpNames = dpkgSrcCaptureRegexp.SubexpNames()
+	dpkgRegexp                = regexp.MustCompile("^var/lib/dpkg/status$")
 )
 
 // DpkgFeaturesDetector implements FeaturesDetector and detects dpkg packages
@@ -110,6 +111,6 @@ func (detector *DpkgFeaturesDetector) Detect(data map[string][]byte) ([]database
 
 // GetRequiredFiles returns the list of files required for Detect, without
 // leading /
-func (detector *DpkgFeaturesDetector) GetRequiredFiles() []string {
-	return []string{"var/lib/dpkg/status"}
+func (detector *DpkgFeaturesDetector) GetRequiredFiles() []*regexp.Regexp {
+	return []*regexp.Regexp{dpkgRegexp}
 }

worker/detectors/feature/npm/npm.go

@@ -0,0 +1,89 @@
+// Copyright 2016 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package npm
+
+import (
+	"encoding/json"
+	"regexp"
+
+	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/utils/types"
+	"github.com/coreos/clair/worker/detectors"
+	"github.com/coreos/pkg/capnslog"
+)
+
+var (
+	log = capnslog.NewPackageLogger("github.com/coreos/clair", "npm")
+
+	nodejsRegexp = regexp.MustCompile("(node_modules|nodejs)/.*/package.json$")
+)
+
+// NpmFeaturesDetector implements FeaturesDetector and detects nodejs packages
+type NpmFeaturesDetector struct{}
+
+type NodejsPkg struct {
+	Version string `json:"version"`
+	Name    string `json:"name"`
+}
+
+func init() {
+	detectors.RegisterFeaturesDetector("npm", &NpmFeaturesDetector{})
+}
+
+// Detect detects packages using *package.json from the input data
+func (detector *NpmFeaturesDetector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) {
+	// Create a map to store packages and ensure their uniqueness
+	packagesMap := make(map[string]database.FeatureVersion)
+	for filename, content := range data {
+		if !nodejsRegexp.MatchString(filename) {
+			continue
+		}
+
+		var nodejsPkg NodejsPkg
+		err := json.Unmarshal(content, &nodejsPkg)
+		if err != nil {
+			log.Warningf("could not parse nodejs package file '%s': %s. skipping", filename, err.Error())
+			continue
+		}
+
+		version, err := types.NewVersion(nodejsPkg.Version)
+		if err != nil {
+			log.Warningf("could not parse package version '%s': %s. skipping", nodejsPkg.Version, err.Error())
+			continue
+		}
+
+		pkg := database.FeatureVersion{
+			Feature: database.Feature{
+				Name: nodejsPkg.Name,
+			},
+			Version: version,
+		}
+		packagesMap[pkg.Feature.Name+"#"+pkg.Version.String()] = pkg
+	}
+
+	// Convert the map to a slice
+	packages := make([]database.FeatureVersion, 0, len(packagesMap))
+	for _, pkg := range packagesMap {
+		packages = append(packages, pkg)
+	}
+
+	return packages, nil
+}
+
+// GetRequiredFiles returns the list of files required for Detect, without
+// leading /
+func (detector *NpmFeaturesDetector) GetRequiredFiles() []*regexp.Regexp {
+	return []*regexp.Regexp{nodejsRegexp}
+}

worker/detectors/feature/npm/npm_test.go

@@ -0,0 +1,46 @@
+// Copyright 2016 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package npm
+
+import (
+	"testing"
+
+	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/utils/types"
+	"github.com/coreos/clair/worker/detectors/feature"
+)
+
+var npmPackagesTests = []feature.FeatureVersionTest{
+	{
+		FeatureVersions: []database.FeatureVersion{
+			{
+				Feature: database.Feature{Name: "npm"},
+				Version: types.NewVersionUnsafe("1.3.10"),
+			},
+			{
+				Feature: database.Feature{Name: "hawk"},
+				Version: types.NewVersionUnsafe("4.0.1"),
+			},
+		},
+		Data: map[string][]byte{
+			"usr/lib/nodejs/npm/package.json":              feature.LoadFileForTest("npm/testdata/npm/package.json"),
+			"usr/local/lib/node_modules/hawk/package.json": feature.LoadFileForTest("npm/testdata/hawk/package.json"),
+		},
+	},
+}
+
+func TestNpmFeaturesDetector(t *testing.T) {
+	feature.TestFeaturesDetector(t, &NpmFeaturesDetector{}, npmPackagesTests)
+}

worker/detectors/feature/npm/testdata/hawk/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "hawk",
+  "description": "HTTP Hawk Authentication Scheme",
+  "version": "4.0.1",
+  "author": {
+    "name": "Eran Hammer",
+    "email": "eran@hammer.io",
+    "url": "http://hueniverse.com"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/hueniverse/hawk"
+  },
+  "engines": {
+    "node": ">=4.0.0"
+  },
+  "dependencies": {
+    "hoek": "3.x.x",
+    "boom": "3.x.x",
+    "cryptiles": "3.x.x",
+    "sntp": "2.x.x"
+  },
+  "license": "BSD-3-Clause",
+  "_id": "hawk@4.1.2",
+  "_from": "hawk@"
+}

worker/detectors/feature/npm/testdata/npm/package.json

@@ -0,0 +1,19 @@
+{
+  "version": "1.3.10",
+  "name": "npm",
+  "homepage": "https://npmjs.org/doc/",
+  "author": "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me)",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/isaacs/npm"
+  },
+  "dependencies": {
+    "semver": "~2.1.0",
+    "github-url-from-git": "1.1.1"
+  },
+  "engines": {
+    "node": ">=0.6",
+    "npm": "1"
+  },
+  "license": "Artistic-2.0"
+}

worker/detectors/feature/rpm/rpm.go

@@ -18,6 +18,7 @@ import (
 	"bufio"
 	"io/ioutil"
 	"os"
+	"regexp"
 	"strings"
 
 	"github.com/coreos/clair/database"
@@ -28,7 +29,10 @@ import (
 	"github.com/coreos/pkg/capnslog"
 )
 
-var log = capnslog.NewPackageLogger("github.com/coreos/clair", "rpm")
+var (
+	log       = capnslog.NewPackageLogger("github.com/coreos/clair", "rpm")
+	rpmRegexp = regexp.MustCompile("^var/lib/rpm/Packages$")
+)
 
 // RpmFeaturesDetector implements FeaturesDetector and detects rpm packages
 // It requires the "rpm" binary to be in the PATH
@@ -115,6 +119,6 @@ func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database.
 
 // GetRequiredFiles returns the list of files required for Detect, without
 // leading /
-func (detector *RpmFeaturesDetector) GetRequiredFiles() []string {
-	return []string{"var/lib/rpm/Packages"}
+func (detector *RpmFeaturesDetector) GetRequiredFiles() []*regexp.Regexp {
+	return []*regexp.Regexp{rpmRegexp}
 }

worker/detectors/features.go

@@ -16,6 +16,7 @@ package detectors
 
 import (
 	"fmt"
+	"regexp"
 	"sync"
 
 	"github.com/coreos/clair/database"
@@ -27,7 +28,7 @@ type FeaturesDetector interface {
 	Detect(map[string][]byte) ([]database.FeatureVersion, error)
 	// GetRequiredFiles returns the list of files required for Detect, without
 	// leading /.
-	GetRequiredFiles() []string
+	GetRequiredFiles() []*regexp.Regexp
 }
 
 var (
@@ -70,7 +71,7 @@ func DetectFeatures(data map[string][]byte) ([]database.FeatureVersion, error) {
 
 // GetRequiredFilesFeatures returns the list of files required for Detect for every
 // registered FeaturesDetector, without leading /.
-func GetRequiredFilesFeatures() (files []string) {
+func GetRequiredFilesFeatures() (files []*regexp.Regexp) {
 	for _, detector := range featuresDetectors {
 		files = append(files, detector.GetRequiredFiles()...)
 	}