Add support for passing build-time environment variables in build con…

…text

- The build-time environment variables are passed as environment-context for command(s)
run as part of the RUN primitve. These variables are not persisted in environment of
intermediate and final images when passed as context for RUN. The build environment
is prepended to the intermediate continer's command string for aiding cache lookups.
It also helps with build traceability. But this also makes the feature less secure from
point of view of passing build time secrets.

- The build-time environment variables also get used to expand the symbols used in certain
Dockerfile primitves like ADD, COPY, USER etc, without an explicit prior definiton using a
ENV primitive. These variables get persisted in the intermediate and final images
whenever they are expanded.

Signed-off-by: Madhav Puri <madhav.puri@gmail.com>

api/client/build.go

@@ -33,6 +33,7 @@ import (
 	"github.com/docker/docker/pkg/units"
 	"github.com/docker/docker/pkg/urlutil"
 	"github.com/docker/docker/registry"
+	"github.com/docker/docker/runconfig"
 	"github.com/docker/docker/utils"
 )
 
@@ -62,6 +63,8 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
 	flCPUSetCpus := cmd.String([]string{"-cpuset-cpus"}, "", "CPUs in which to allow execution (0-3, 0,1)")
 	flCPUSetMems := cmd.String([]string{"-cpuset-mems"}, "", "MEMs in which to allow execution (0-3, 0,1)")
 	flCgroupParent := cmd.String([]string{"-cgroup-parent"}, "", "Optional parent cgroup for the container")
+	flEnv := opts.NewListOpts(opts.ValidateEnv)
+	cmd.Var(&flEnv, []string{"-build-env"}, "Set build-time environment variables")
 
 	ulimits := make(map[string]*ulimit.Ulimit)
 	flUlimits := opts.NewUlimitOpt(ulimits)
@@ -298,6 +301,14 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
 	}
 	headers.Add("X-Registry-Config", base64.URLEncoding.EncodeToString(buf))
 
+	// collect all the build-time environment variables for the container
+	envVariables := runconfig.ConvertKVStringsToMap(flEnv.GetAll())
+	buf, err = json.Marshal(envVariables)
+	if err != nil {
+		return err
+	}
+	headers.Add("X-Docker-BuildEnv", base64.URLEncoding.EncodeToString(buf))
+
 	if context != nil {
 		headers.Set("Content-Type", "application/tar")
 	}

api/server/server.go

@@ -1250,6 +1250,8 @@ func (s *Server) postBuild(version version.Version, w http.ResponseWriter, r *ht
 	var (
 		authConfigs        = map[string]cliconfig.AuthConfig{}
 		authConfigsEncoded = r.Header.Get("X-Registry-Config")
+		buildEnvEncoded    = r.Header.Get("X-Docker-BuildEnv")
+		buildEnv           = map[string]string{}
 		buildConfig        = builder.NewBuildConfig()
 	)
 
@@ -1262,6 +1264,13 @@ func (s *Server) postBuild(version version.Version, w http.ResponseWriter, r *ht
 		}
 	}
 
+	if buildEnvEncoded != "" {
+		buildEnvJson := base64.NewDecoder(base64.URLEncoding, strings.NewReader(buildEnvEncoded))
+		if err := json.NewDecoder(buildEnvJson).Decode(&buildEnv); err != nil {
+			return err
+		}
+	}
+
 	w.Header().Set("Content-Type", "application/json")
 
 	if boolValue(r, "forcerm") && version.GreaterThanOrEqualTo("1.12") {
@@ -1294,6 +1303,7 @@ func (s *Server) postBuild(version version.Version, w http.ResponseWriter, r *ht
 	buildConfig.CPUSetCpus = r.FormValue("cpusetcpus")
 	buildConfig.CPUSetMems = r.FormValue("cpusetmems")
 	buildConfig.CgroupParent = r.FormValue("cgroupparent")
+	buildConfig.BuildEnv = buildEnv
 
 	var buildUlimits = []*ulimit.Ulimit{}
 	ulimitsJson := r.FormValue("ulimits")

builder/dispatchers.go

@@ -343,15 +343,51 @@ func run(b *builder, args []string, attributes map[string]bool, original string)
 		return err
 	}
 
+	// stash the cmd
 	cmd := b.Config.Cmd
-	// set Cmd manually, this is special case only for Dockerfiles
-	b.Config.Cmd = config.Cmd
 	runconfig.Merge(b.Config, config)
+	// stash the config environment
+	env := b.Config.Env
 
 	defer func(cmd *runconfig.Command) { b.Config.Cmd = cmd }(cmd)
+	defer func(env []string) { b.Config.Env = env }(env)
+
+	// derive the net build-time environment for this run. We let config
+	// environment override the build time environment.
+	// This means that we take the b.BuildEnv list of env vars and remove
+	// any of those variables that are defined as part of the container. In other
+	// words, anything in b.Config.Env. What's left is the list of build-time env
+	// vars that we need to add to each RUN command - note the list could be empty.
+	//
+	// We don't persist the build time environment with container's config
+	// environment, but just sort and pre-pend it to the command string at time
+	// of commit.
+	// This helps with tracing back the image's actual environment at the time
+	// of RUN, without leaking it to the final image. It also aids cache
+	// lookup for same image built with same build time environment.
+	cmdBuildEnv := []string{}
+	configEnv := runconfig.ConvertKVStringsToMap(b.Config.Env)
+	for key, val := range b.BuildEnv {
+		if _, ok := configEnv[key]; !ok {
+			cmdBuildEnv = append(cmdBuildEnv, fmt.Sprintf("%s=%s", key, val))
+		}
+	}
 
-	logrus.Debugf("[BUILDER] Command to be executed: %v", b.Config.Cmd)
+	// derive the command to use for probeCache() and to commit in this container.
+	// Note that we only do this if there are any build-time env vars.  Also, we
+	// use the special argument "|#" at the start of the args array. This will
+	// avoid conflicts with any RUN command since commands can not
+	// start with | (vertical bar). The "#" (number of build envs) is there to
+	// help ensure proper cache matches. We don't want a RUN command
+	// that starts with "foo=abc" to be considered part of a build-time env var.
+	saveCmd := config.Cmd
+	if len(cmdBuildEnv) > 0 {
+		sort.Strings(cmdBuildEnv)
+		tmpEnv := append([]string{fmt.Sprintf("|%d", len(cmdBuildEnv))}, cmdBuildEnv...)
+		saveCmd = runconfig.NewCommand(append(tmpEnv, saveCmd.Slice()...)...)
+	}
 
+	b.Config.Cmd = saveCmd
 	hit, err := b.probeCache()
 	if err != nil {
 		return err
@@ -360,6 +396,13 @@ func run(b *builder, args []string, attributes map[string]bool, original string)
 		return nil
 	}
 
+	// set Cmd manually, this is special case only for Dockerfiles
+	b.Config.Cmd = config.Cmd
+	// set build-time environment for 'run'.
+	b.Config.Env = append(b.Config.Env, cmdBuildEnv...)
+
+	logrus.Debugf("[BUILDER] Command to be executed: %v", b.Config.Cmd)
+
 	c, err := b.create()
 	if err != nil {
 		return err
@@ -374,6 +417,12 @@ func run(b *builder, args []string, attributes map[string]bool, original string)
 	if err != nil {
 		return err
 	}
+
+	// revert to original config environment and set the command string to
+	// have the build-time env vars in it (if any) so that future cache look-ups
+	// properly match it.
+	b.Config.Env = env
+	b.Config.Cmd = saveCmd
 	if err := b.commit(c.ID, cmd, "run"); err != nil {
 		return err
 	}

builder/evaluator.go

@@ -108,6 +108,9 @@ type builder struct {
 
 	Config *runconfig.Config // runconfig for cmd, run, entrypoint etc.
 
+	// build-time environment variables for expansion/substitution and commands in 'run'.
+	BuildEnv map[string]string
+
 	// both of these are controlled by the Remove and ForceRemove options in BuildOpts
 	TmpContainers map[string]struct{} // a map of containers used for removes
 
@@ -320,13 +323,26 @@ func (b *builder) dispatch(stepN int, ast *parser.Node) error {
 	msgList := make([]string, n)
 
 	var i int
+	// Append the build-time environment to config-environment.
+	// This allows builder config to override the variables, making the behavior similar to
+	// a shell script i.e. `ENV foo bar` overrides value of `foo` passed in build
+	// context. But `ENV foo $foo` will use the value from build context if one
+	// isn't already been defined by a previous ENV primitive.
+	// Note, we get this behavior because we know that ProcessWord() will
+	// stop on the first occurrence of a variable name and not notice
+	// a subsequent one. So, putting the BuildEnv list after the Config.Env
+	// list, in 'envs', is safe.
+	envs := b.Config.Env
+	for key, val := range b.BuildEnv {
+		envs = append(envs, fmt.Sprintf("%s=%s", key, val))
+	}
 	for ast.Next != nil {
 		ast = ast.Next
 		var str string
 		str = ast.Value
 		if _, ok := replaceEnvAllowed[cmd]; ok {
 			var err error
-			str, err = ProcessWord(ast.Value, b.Config.Env)
+			str, err = ProcessWord(ast.Value, envs)
 			if err != nil {
 				return err
 			}

builder/job.go

@@ -65,6 +65,7 @@ type Config struct {
 	CgroupParent   string
 	Ulimits        []*ulimit.Ulimit
 	AuthConfigs    map[string]cliconfig.AuthConfig
+	BuildEnv       map[string]string
 
 	Stdout  io.Writer
 	Context io.ReadCloser
@@ -210,6 +211,7 @@ func Build(d *daemon.Daemon, buildConfig *Config) error {
 		ulimits:         buildConfig.Ulimits,
 		cancelled:       buildConfig.WaitCancelled(),
 		id:              stringid.GenerateRandomID(),
+		BuildEnv:        buildConfig.BuildEnv,
 	}
 
 	defer func() {

docs/reference/api/docker_remote_api.md

@@ -257,6 +257,12 @@ Volumes are now initialized when the container is created.
 **New!**
 You can now copy data which is contained in a volume.
 
+`POST /build`
+
+**New!**
+This endpoint now optionally takes a serialized array of build-time environment
+variables.
+
 ## v1.15
 
 ### Full documentation

docs/reference/api/docker_remote_api_v1.16.md

@@ -1221,6 +1221,18 @@ Query Parameters:
 
 -   **Content-type** – should be set to `"application/tar"`.
 -   **X-Registry-Config** – base64-encoded ConfigFile object
+-   **X-Docker-BuildEnv** – base64-encoded JSON map of string pairs for build-time
+    environment variables. These are used to expand/substitute
+    the corresponding variables used in the Dockerfile primitives, without
+    an explicit prior definition by the ENV primitive. These are also used as
+    environment context for the command(s) run as part of the 'RUN' primitive
+    of the Dockerfile, if there is no prior explicit definition of the variable by the ENV primitive.
+    Normally, these variables are not persisted in the resulting Docker image. This gives
+    the flexibility to build an image by passing host specific environment variables (like
+    http_proxy) that will be used on the RUN commands without affecting portability
+    of the generated image.
+    However, as with any variable, they can be persisted in the final image if they are used in an
+    ENV command (e.g. ENV myName=$myName will save myName in the image).
 
 Status Codes:
 

docs/reference/commandline/build.md

@@ -17,6 +17,7 @@ weight=1
 
       -f, --file=""            Name of the Dockerfile (Default is 'PATH/Dockerfile')
       --force-rm=false         Always remove intermediate containers
+      --build-env=[]           Set build-time environment variables
       --no-cache=false         Do not use cache when building the image
       --pull=false             Always attempt to pull a newer version of the image
       -q, --quiet=false        Suppress the verbose output generated by the containers
@@ -251,3 +252,53 @@ flag](/reference/run/#specifying-custom-cgroups).
 Using the `--ulimit` option with `docker build` will cause each build step's 
 container to be started using those [`--ulimit`
 flag values](/reference/run/#setting-ulimits-in-a-container).
+
+Sometimes building an image will require access to certain environment variables
+that are specific to the build host like `http-proxy` or source versions for pulling
+intermediate files etc. Most likely these variables can't be persisted in the built
+image for portability reasons. Hence it's not always desirable to use the `ENV` primitive
+of the Dockerfile.
+
+    $ docker build --build-env HTTP_PROXY=http://10.20.30.2:1234 .
+
+This will allow passing the built-time environment variables which can be
+accessed like regular environment variables in the `RUN` primitive of the
+Dockerfile, without persisting them in the intermediate or final images.
+
+> **Note:** It is not recommended to use build-time environment variables for
+>  passing secrets like github keys, user credentials etc.
+
+> **Note:** Environment variables defined using `ENV` primitive of `Dockerfile`
+> will override the build-time environment variables.
+
+> **Note:** When desirable, variables passed from command-line can be persisted
+> in the final image by using `ENV` primitive and build-time environment variables
+> as shown below.
+
+    $ docker build --build-env CONT_IMG_VER=v2.0.1 - <<MARK
+    -> FROM ubuntu
+    -> ENV CONT_IMG_VER ${CONT_IMG_VER:-v1.0.0}
+    -> MARK
+
+In the above example, the `ENV` command will do two things. First, it will set
+`CONT_IMG_VER` to `v2.0.1` as the build-time environment variable was
+defined for it. Second, it will persist that variable in the resulting image. Note
+that if `docker build` were run without setting the build-time environment
+variable, then `CONT_IMG_VER` would still be persisted in the image but its
+value would be `v1.0.0`.
+
+There are also cases where users would like to set the values of the variables
+defined in the Dockerfile, without having an explicit `ENV` primitive.
+
+    $ docker build --build-env foo=bar .
+
+This will allow passing the value for a variable named `foo` and substitute all its
+occurrences with value `bar` in `Dockerfile` primitives.
+
+> **Note:** The variable expansion is only supported for certain `Dockerfile`
+> primitives as described [here](https://docs.docker.com/reference/builder/#environment-replacement)
+
+> **Note:** Environment variables defined using 'ENV' primitive of Dockerfile
+> will override the values of build variables passed as argument. This behavior is similar
+> to as a shell script where a locally scoped variable overrides the variables passed
+> as arguments or inherited from environment, from it's point of definition.