Merge pull request #749 from cnorthwood/tarutil-glob

tarutil: allow file names to be specified by regexp

ext/featurefmt/apk/apk.go

@@ -90,5 +90,5 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.LayerFeature, e
 }
 
 func (l lister) RequiredFilenames() []string {
-	return []string{"lib/apk/db/installed"}
+	return []string{"^lib/apk/db/installed"}
 }

ext/featurefmt/dpkg/dpkg.go

@@ -38,7 +38,7 @@ var (
 type lister struct{}
 
 func (l lister) RequiredFilenames() []string {
-	return []string{"var/lib/dpkg/status"}
+	return []string{"^var/lib/dpkg/status"}
 }
 
 func init() {

ext/featurefmt/driver.go

@@ -35,10 +35,11 @@ type Lister interface {
 	// ListFeatures produces a list of Features present in an image layer.
 	ListFeatures(tarutil.FilesMap) ([]database.LayerFeature, error)
 
-	// RequiredFilenames returns the list of files required to be in the FilesMap
-	// provided to the ListFeatures method.
+	// RequiredFilenames returns a list of patterns for filenames that will
+	// be in the FilesMap provided to the ListFeatures method.
 	//
-	// Filenames must not begin with "/".
+	// The patterns are expressed as regexps, and will be matched against
+	// full paths that do not include the leading "/".
 	RequiredFilenames() []string
 }
 
@@ -102,8 +103,9 @@ func ListFeatures(files tarutil.FilesMap, toUse []database.Detector) ([]database
 	return features, nil
 }
 
-// RequiredFilenames returns all files required by the give extensions. Any
-// extension metadata that has non feature-detector type will be skipped.
+// RequiredFilenames returns all file patterns that will be passed to the
+// given extensions. These patterns are expressed as regexps. Any extension
+// metadata that has non feature-detector type will be skipped.
 func RequiredFilenames(toUse []database.Detector) (files []string) {
 	listersM.RLock()
 	defer listersM.RUnlock()

ext/featurefmt/rpm/rpm.go

@@ -46,7 +46,7 @@ func init() {
 }
 
 func (l lister) RequiredFilenames() []string {
-	return []string{"var/lib/rpm/Packages"}
+	return []string{"^var/lib/rpm/Packages"}
 }
 
 func isIgnored(packageName string) bool {

ext/featurens/alpinerelease/alpinerelease.go

@@ -62,5 +62,5 @@ func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) {
 }
 
 func (d detector) RequiredFilenames() []string {
-	return []string{alpineReleasePath}
+	return []string{"^" + alpineReleasePath}
 }

ext/featurens/aptsources/aptsources.go

@@ -88,5 +88,5 @@ func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) {
 }
 
 func (d detector) RequiredFilenames() []string {
-	return []string{"etc/apt/sources.list"}
+	return []string{`^etc/apt/sources\.list`}
 }

ext/featurens/driver.go

@@ -39,10 +39,11 @@ type Detector interface {
 	// layer.
 	Detect(tarutil.FilesMap) (*database.Namespace, error)
 
-	// RequiredFilenames returns the list of files required to be in the FilesMap
-	// provided to the Detect method.
+	// RequiredFilenames returns a list of patterns for filenames that will
+	// be in the FilesMap provided to the Detect method.
 	//
-	// Filenames must not begin with "/".
+	// The patterns are expressed as regexps, and will be matched against
+	// full paths that do not include the leading "/".
 	RequiredFilenames() []string
 }
 
@@ -108,8 +109,9 @@ func Detect(files tarutil.FilesMap, toUse []database.Detector) ([]database.Layer
 	return namespaces, nil
 }
 
-// RequiredFilenames returns all files required by the give extensions. Any
-// extension metadata that has non namespace-detector type will be skipped.
+// RequiredFilenames returns all file patterns that will be passed to the
+// given extensions. These patterns are expressed as regexps. Any extension
+// metadata that has non namespace-detector type will be skipped.
 func RequiredFilenames(toUse []database.Detector) (files []string) {
 	detectorsM.RLock()
 	defer detectorsM.RUnlock()

ext/featurens/lsbrelease/lsbrelease.go

@@ -94,5 +94,5 @@ func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) {
 }
 
 func (d *detector) RequiredFilenames() []string {
-	return []string{"etc/lsb-release"}
+	return []string{"^etc/lsb-release"}
 }

ext/featurens/osrelease/osrelease.go

@@ -34,6 +34,8 @@ var (
 	osReleaseOSRegexp      = regexp.MustCompile(`^ID=(.*)`)
 	osReleaseVersionRegexp = regexp.MustCompile(`^VERSION_ID=(.*)`)
 
+	filenames = []string{"etc/os-release", "usr/lib/os-release"}
+
 	// blacklistFilenames are files that should exclude this detector.
 	blacklistFilenames = []string{
 		"etc/oracle-release",
@@ -57,7 +59,7 @@ func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) {
 		}
 	}
 
-	for _, filePath := range d.RequiredFilenames() {
+	for _, filePath := range filenames {
 		f, hasFile := files[filePath]
 		if !hasFile {
 			continue
@@ -100,5 +102,5 @@ func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) {
 }
 
 func (d detector) RequiredFilenames() []string {
-	return []string{"etc/os-release", "usr/lib/os-release"}
+	return []string{"^(etc|usr/lib)/os-release"}
 }

ext/featurens/redhatrelease/redhatrelease.go

@@ -33,6 +33,8 @@ var (
 	oracleReleaseRegexp = regexp.MustCompile(`(?P<os>Oracle) (Linux Server release) (?P<version>[\d]+)`)
 	centosReleaseRegexp = regexp.MustCompile(`(?P<os>[^\s]*) (Linux release|release) (?P<version>[\d]+)`)
 	redhatReleaseRegexp = regexp.MustCompile(`(?P<os>Red Hat Enterprise Linux) (Client release|Server release|Workstation release) (?P<version>[\d]+)`)
+
+	filenames = []string{"etc/oracle-release", "etc/centos-release", "etc/redhat-release", "etc/system-release"}
 )
 
 type detector struct{}
@@ -42,7 +44,7 @@ func init() {
 }
 
 func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) {
-	for _, filePath := range d.RequiredFilenames() {
+	for _, filePath := range filenames {
 		f, hasFile := files[filePath]
 		if !hasFile {
 			continue
@@ -83,5 +85,5 @@ func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) {
 }
 
 func (d detector) RequiredFilenames() []string {
-	return []string{"etc/oracle-release", "etc/centos-release", "etc/redhat-release", "etc/system-release"}
+	return []string{`^etc/(oracle|centos|redhat|system)-release`}
 }

pkg/tarutil/tarutil.go

@@ -25,6 +25,7 @@ import (
 	"io"
 	"io/ioutil"
 	"os/exec"
+	"regexp"
 	"strings"
 )
 
@@ -50,7 +51,8 @@ var (
 type FilesMap map[string][]byte
 
 // ExtractFiles decompresses and extracts only the specified files from an
-// io.Reader representing an archive.
+// io.Reader representing an archive. The files to be extracted are specified
+// by regexp
 func ExtractFiles(r io.Reader, filenames []string) (FilesMap, error) {
 	data := make(map[string][]byte)
 
@@ -78,7 +80,7 @@ func ExtractFiles(r io.Reader, filenames []string) (FilesMap, error) {
 		// Determine if we should extract the element
 		toBeExtracted := false
 		for _, s := range filenames {
-			if strings.HasPrefix(filename, s) {
+			if match, err := regexp.MatchString(s, filename); err == nil && match {
 				toBeExtracted = true
 				break
 			}

pkg/tarutil/tarutil_test.go

@@ -57,6 +57,26 @@ func TestExtract(t *testing.T) {
 	}
 }
 
+func TestExtractRegex(t *testing.T) {
+	for _, filename := range testTarballs {
+		f, err := os.Open(testfilepath(filename))
+		assert.Nil(t, err)
+		defer f.Close()
+
+		data, err := ExtractFiles(f, []string{`.+\.txt`})
+		assert.Nil(t, err)
+
+		if c, n := data["test/test.txt"]; !n {
+			assert.Fail(t, "test/test.txt should have been extracted")
+		} else {
+			assert.NotEqual(t, 0, len(c) > 0, "test/test.txt file is empty")
+		}
+		if _, n := data["test.txt"]; !n {
+			assert.Fail(t, "test.txt should also have been extracted")
+		}
+	}
+}
+
 func TestExtractUncompressedData(t *testing.T) {
 	for _, filename := range testTarballs {
 		f, err := os.Open(testfilepath(filename))