COPY command

executor/cmd/root.go

@@ -98,7 +98,7 @@ func execute() error {
 	// Currently only supports single stage builds
 	for _, stage := range stages {
 		for _, cmd := range stage.Commands {
-			dockerCommand, err := commands.GetCommand(cmd)
+			dockerCommand, err := commands.GetCommand(cmd, srcContext)
 			if err != nil {
 				return err
 			}

integration_tests/context/arr[0].txt

@@ -0,0 +1 @@
+hello

integration_tests/context/bar/bam/bat

@@ -0,0 +1 @@
+bat

integration_tests/context/bar/bat

@@ -0,0 +1 @@
+bat

integration_tests/context/bar/baz

@@ -0,0 +1 @@
+baz

integration_tests/context/foo

@@ -0,0 +1 @@
+foo

integration_tests/dockerfiles/Dockerfile_test_copy

@@ -0,0 +1,14 @@
+FROM gcr.io/distroless/base
+COPY context/foo foo
+COPY context/foo /foodir/
+COPY context/bar/b* bar/
+COPY context/fo? /foo2
+COPY context/bar/doesnotexist* context/foo hello
+COPY ./context/empty /empty
+COPY ./ dir/
+COPY . newdir
+COPY context/bar /baz/
+COPY ["context/foo", "/tmp/foo" ]
+COPY context/b* /baz/
+COPY context/foo context/bar/ba? /test/
+COPY context/arr[[]0].txt /mydir/

integration_tests/dockerfiles/config_test_copy.json

@@ -0,0 +1,12 @@
+[
+  {
+    "Image1": "gcr.io/kbuild-test/docker-test-copy:latest",
+    "Image2": "gcr.io/kbuild-test/kbuild-test-copy:latest",
+    "DiffType": "File",
+    "Diff": {
+      "Adds": null,
+      "Dels": null,
+      "Mods": null
+    }
+  }
+]

integration_tests/integration_test_yaml.go

@@ -49,6 +49,13 @@ var fileTests = []struct {
 		context:        "integration_tests/dockerfiles/",
 		repo:           "test-run-2",
 	},
+	{
+		description:    "test copy",
+		dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_copy",
+		configPath:     "/workspace/integration_tests/dockerfiles/config_test_copy.json",
+		context:        "/workspace/integration_tests/",
+		repo:           "test-copy",
+	},
 }
 
 var structureTests = []struct {
@@ -126,7 +133,7 @@ func main() {
 		kbuildImage := testRepo + kbuildPrefix + test.repo
 		kbuild := step{
 			Name: executorImage,
-			Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath},
+			Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath, "--context", test.context},
 		}
 
 		// Pull the kbuild image

pkg/commands/commands.go

@@ -34,10 +34,12 @@ type DockerCommand interface {
 	FilesToSnapshot() []string
 }
 
-func GetCommand(cmd instructions.Command) (DockerCommand, error) {
+func GetCommand(cmd instructions.Command, buildcontext string) (DockerCommand, error) {
 	switch c := cmd.(type) {
 	case *instructions.RunCommand:
 		return &RunCommand{cmd: c}, nil
+	case *instructions.CopyCommand:
+		return &CopyCommand{cmd: c, buildcontext: buildcontext}, nil
 	case *instructions.ExposeCommand:
 		return &ExposeCommand{cmd: c}, nil
 	case *instructions.EnvCommand:

pkg/commands/copy.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package commands
+
+import (
+	"github.com/GoogleCloudPlatform/k8s-container-builder/pkg/util"
+	"github.com/containers/image/manifest"
+	"github.com/docker/docker/builder/dockerfile/instructions"
+	"github.com/sirupsen/logrus"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type CopyCommand struct {
+	cmd           *instructions.CopyCommand
+	buildcontext  string
+	snapshotFiles []string
+}
+
+func (c *CopyCommand) ExecuteCommand(config *manifest.Schema2Config) error {
+	srcs := c.cmd.SourcesAndDest[:len(c.cmd.SourcesAndDest)-1]
+	dest := c.cmd.SourcesAndDest[len(c.cmd.SourcesAndDest)-1]
+
+	logrus.Infof("cmd: copy %s", srcs)
+	logrus.Infof("dest: %s", dest)
+
+	// Get a map of [src]:[files rooted at src]
+	srcMap, err := util.ResolveSources(c.cmd.SourcesAndDest, c.buildcontext)
+	if err != nil {
+		return err
+	}
+	// For each source, iterate through each file within and copy it over
+	for src, files := range srcMap {
+		for _, file := range files {
+			fi, err := os.Stat(filepath.Join(c.buildcontext, file))
+			if err != nil {
+				return err
+			}
+			destPath, err := util.DestinationFilepath(file, src, dest, config.WorkingDir, c.buildcontext)
+			if err != nil {
+				return err
+			}
+			// If source file is a directory, we want to create a directory ...
+			if fi.IsDir() {
+				logrus.Infof("Creating directory %s", destPath)
+				if err := os.MkdirAll(destPath, fi.Mode()); err != nil {
+					return err
+				}
+			} else {
+				// ... Else, we want to copy over a file
+				logrus.Infof("Copying file %s to %s", file, destPath)
+				srcFile, err := os.Open(filepath.Join(c.buildcontext, file))
+				if err != nil {
+					return err
+				}
+				defer srcFile.Close()
+				if err := util.CreateFile(destPath, srcFile, fi.Mode()); err != nil {
+					return err
+				}
+			}
+			// Append the destination file to the list of files that should be snapshotted later
+			c.snapshotFiles = append(c.snapshotFiles, destPath)
+		}
+	}
+	return nil
+}
+
+// FilesToSnapshot should return an empty array if still nil; no files were changed
+func (c *CopyCommand) FilesToSnapshot() []string {
+	return c.snapshotFiles
+}
+
+// CreatedBy returns some information about the command for the image config
+func (c *CopyCommand) CreatedBy() string {
+	return strings.Join(c.cmd.SourcesAndDest, " ")
+}

pkg/util/command_util.go

@@ -0,0 +1,163 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"github.com/docker/docker/builder/dockerfile/instructions"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// ContainsWildcards returns true if any entry in paths contains wildcards
+func ContainsWildcards(paths []string) bool {
+	for _, path := range paths {
+		if strings.ContainsAny(path, "*?[") {
+			return true
+		}
+	}
+	return false
+}
+
+// ResolveSources resolves the given sources if the sources contains wildcards
+// It returns a map of [src]:[files rooted at src]
+func ResolveSources(srcsAndDest instructions.SourcesAndDest, root string) (map[string][]string, error) {
+	srcs := srcsAndDest[:len(srcsAndDest)-1]
+	// If sources contain wildcards, we first need to resolve them to actual paths
+	if ContainsWildcards(srcs) {
+		logrus.Debugf("Resolving srcs %v...", srcs)
+		files, err := RelativeFiles("", root)
+		if err != nil {
+			return nil, err
+		}
+		srcs, err = matchSources(srcs, files)
+		if err != nil {
+			return nil, err
+		}
+		logrus.Debugf("Resolved sources to %v", srcs)
+	}
+	// Now, get a map of [src]:[files rooted at src]
+	srcMap, err := SourcesToFilesMap(srcs, root)
+	if err != nil {
+		return nil, err
+	}
+	// Check to make sure the sources are valid
+	return srcMap, IsSrcsValid(srcsAndDest, srcMap)
+}
+
+// matchSources returns a list of sources that match wildcards
+func matchSources(srcs, files []string) ([]string, error) {
+	var matchedSources []string
+	for _, src := range srcs {
+		src = filepath.Clean(src)
+		for _, file := range files {
+			matched, err := filepath.Match(src, file)
+			if err != nil {
+				return nil, err
+			}
+			if matched {
+				matchedSources = append(matchedSources, file)
+			}
+		}
+	}
+	return matchedSources, nil
+}
+
+func IsDestDir(path string) bool {
+	return strings.HasSuffix(path, "/") || path == "."
+}
+
+// DestinationFilepath returns the destination filepath from the build context to the image filesystem
+// If source is a file:
+//	If dest is a dir, copy it to /dest/relpath
+// 	If dest is a file, copy directly to dest
+// If source is a dir:
+//	Assume dest is also a dir, and copy to dest/relpath
+// If dest is not an absolute filepath, add /cwd to the beginning
+func DestinationFilepath(filename, srcName, dest, cwd, buildcontext string) (string, error) {
+	fi, err := os.Stat(filepath.Join(buildcontext, filename))
+	if err != nil {
+		return "", err
+	}
+	src, err := os.Stat(filepath.Join(buildcontext, srcName))
+	if err != nil {
+		return "", err
+	}
+	if src.IsDir() || IsDestDir(dest) {
+		relPath, err := filepath.Rel(srcName, filename)
+		if err != nil {
+			return "", err
+		}
+		if relPath == "." && !fi.IsDir() {
+			relPath = filepath.Base(filename)
+		}
+		destPath := filepath.Join(dest, relPath)
+		if filepath.IsAbs(dest) {
+			return destPath, nil
+		}
+		return filepath.Join(cwd, destPath), nil
+	}
+	if filepath.IsAbs(dest) {
+		return dest, nil
+	}
+	return filepath.Join(cwd, dest), nil
+}
+
+// SourcesToFilesMap returns a map of [src]:[files rooted at source]
+func SourcesToFilesMap(srcs []string, root string) (map[string][]string, error) {
+	srcMap := make(map[string][]string)
+	for _, src := range srcs {
+		src = filepath.Clean(src)
+		files, err := RelativeFiles(src, root)
+		if err != nil {
+			return nil, err
+		}
+		srcMap[src] = files
+	}
+	return srcMap, nil
+}
+
+// IsSrcsValid returns an error if the sources provided are invalid, or nil otherwise
+func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, srcMap map[string][]string) error {
+	srcs := srcsAndDest[:len(srcsAndDest)-1]
+	dest := srcsAndDest[len(srcsAndDest)-1]
+
+	totalFiles := 0
+	for _, files := range srcMap {
+		totalFiles += len(files)
+	}
+	if totalFiles == 0 {
+		return errors.New("copy failed: no source files specified")
+	}
+
+	if !ContainsWildcards(srcs) {
+		// If multiple sources and destination isn't a directory, return an error
+		if len(srcs) > 1 && !IsDestDir(dest) {
+			return errors.New("when specifying multiple sources in a COPY command, destination must be a directory and end in '/'")
+		}
+		return nil
+	}
+
+	// If there are wildcards, and the destination is a file, there must be exactly one file to copy over,
+	// Otherwise, return an error
+	if !IsDestDir(dest) && totalFiles > 1 {
+		return errors.New("when specifying multiple sources in a COPY command, destination must be a directory and end in '/'")
+	}
+	return nil
+}