Add support for build-time environment variables to the 'build' API

A build-time environment variable gets used only while processing
the 'RUN' primitive of a DockerFile. Such a variable is only
accessible during 'RUN' and is 'not' persisted with the intermediate
and final docker images, thereby addressing the portability concerns
of the images generated with 'build'

Signed-off-by: Madhav Puri <madhav.puri@gmail.com>

api/client/commands.go

@@ -83,6 +83,11 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
 	rm := cmd.Bool([]string{"#rm", "-rm"}, true, "Remove intermediate containers after a successful build")
 	forceRm := cmd.Bool([]string{"-force-rm"}, false, "Always remove intermediate containers, even after unsuccessful builds")
 	pull := cmd.Bool([]string{"-pull"}, false, "Always attempt to pull a newer version of the image")
+	flEnv := opts.NewListOpts(opts.ValidateEnv)
+	flEnvFile := opts.NewListOpts(nil)
+	cmd.Var(&flEnv, []string{"e", "-env"}, "Set build-time environment variables")
+	cmd.Var(&flEnvFile, []string{"-env-file"}, "Read in a line delimited file of build-time environment variables")
+
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -231,6 +236,23 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
 	}
 	headers.Add("X-Registry-Config", base64.URLEncoding.EncodeToString(buf))
 
+	// collect all the environment variables for the container
+	envVariables := []string{}
+	for _, ef := range flEnvFile.GetAll() {
+		parsedVars, err := opts.ParseEnvFile(ef)
+		if err != nil {
+			return err
+		}
+		envVariables = append(envVariables, parsedVars...)
+	}
+	// parse the '-e' and '--env' after, to allow override the env-file
+	envVariables = append(envVariables, flEnv.GetAll()...)
+	buf, err = json.Marshal(envVariables)
+	if err != nil {
+		return err
+	}
+	headers.Add("X-BuildEnv", base64.URLEncoding.EncodeToString(buf))
+
 	if context != nil {
 		headers.Set("Content-Type", "application/tar")
 	}

api/server/server.go

@@ -991,6 +991,8 @@ func postBuild(eng *engine.Engine, version version.Version, w http.ResponseWrite
 		authConfig        = &registry.AuthConfig{}
 		configFileEncoded = r.Header.Get("X-Registry-Config")
 		configFile        = &registry.ConfigFile{}
+		buildEnvEncoded   = r.Header.Get("X-BuildEnv")
+		buildEnv          = []string{}
 		job               = eng.Job("build")
 	)
 
@@ -1016,6 +1018,13 @@ func postBuild(eng *engine.Engine, version version.Version, w http.ResponseWrite
 		}
 	}
 
+	if buildEnvEncoded != "" {
+		buildEnvJson := base64.NewDecoder(base64.URLEncoding, strings.NewReader(buildEnvEncoded))
+		if err := json.NewDecoder(buildEnvJson).Decode(&buildEnv); err != nil {
+			return err
+		}
+	}
+
 	if version.GreaterThanOrEqualTo("1.8") {
 		job.SetenvBool("json", true)
 		streamJSON(job, w, true)
@@ -1041,6 +1050,7 @@ func postBuild(eng *engine.Engine, version version.Version, w http.ResponseWrite
 	job.Setenv("forcerm", r.FormValue("forcerm"))
 	job.SetenvJson("authConfig", authConfig)
 	job.SetenvJson("configFile", configFile)
+	job.SetenvList("buildEnv", buildEnv)
 
 	if err := job.Run(); err != nil {
 		if !job.Stdout.Used() {

builder/dispatchers.go

@@ -226,10 +226,16 @@ func run(b *Builder, args []string, attributes map[string]bool, original string)
 	// set Cmd manually, this is special case only for Dockerfiles
 	b.Config.Cmd = config.Cmd
 	runconfig.Merge(b.Config, config)
+	// set build-time environment for 'run'. We let dockerfile
+	// environment override build-time environment.
+	env := b.Config.Env
+	b.Config.Env = append(b.BuildEnv, b.Config.Env...)
 
 	defer func(cmd []string) { b.Config.Cmd = cmd }(cmd)
+	defer func(env []string) { b.Config.Env = env }(env)
 
 	log.Debugf("[BUILDER] Command to be executed: %v", b.Config.Cmd)
+	log.Debugf("[BUILDER] Environment (applied in order): %v", b.Config.Env)
 
 	hit, err := b.probeCache()
 	if err != nil {
@@ -253,6 +259,8 @@ func run(b *Builder, args []string, attributes map[string]bool, original string)
 	if err != nil {
 		return err
 	}
+	// revert to original config, we don't persist build time environment
+	b.Config.Env = env
 	if err := b.commit(c.ID, cmd, "run"); err != nil {
 		return err
 	}

builder/evaluator.go

@@ -101,6 +101,9 @@ type Builder struct {
 
 	Config *runconfig.Config // runconfig for cmd, run, entrypoint etc.
 
+	// build-time environment variables for 'run'. These are not persisted with final or intermediate build images.
+	BuildEnv []string
+
 	// both of these are controlled by the Remove and ForceRemove options in BuildOpts
 	TmpContainers map[string]struct{} // a map of containers used for removes
 

builder/job.go

@@ -39,11 +39,13 @@ func (b *BuilderJob) CmdBuild(job *engine.Job) engine.Status {
 		pull           = job.GetenvBool("pull")
 		authConfig     = &registry.AuthConfig{}
 		configFile     = &registry.ConfigFile{}
+		buildEnv       = []string{}
 		tag            string
 		context        io.ReadCloser
 	)
 	job.GetenvJson("authConfig", authConfig)
 	job.GetenvJson("configFile", configFile)
+	buildEnv = job.GetenvList("buildEnv")
 
 	repoName, tag = parsers.ParseRepositoryTag(repoName)
 	if repoName != "" {
@@ -118,6 +120,7 @@ func (b *BuilderJob) CmdBuild(job *engine.Job) engine.Status {
 		StreamFormatter: sf,
 		AuthConfig:      authConfig,
 		AuthConfigFile:  configFile,
+		BuildEnv:        buildEnv,
 	}
 
 	id, err := builder.Run(context)

docs/man/docker-build.1.md

@@ -6,6 +6,8 @@ docker-build - Build a new image from the source code at PATH
 
 # SYNOPSIS
 **docker build**
+[**-e**|**--env**[=*[]*]]
+[**--env-file**[=*[]*]]
 [**--force-rm**[=*false*]]
 [**--no-cache**[=*false*]]
 [**-q**|**--quiet**[=*false*]]
@@ -30,6 +32,19 @@ When a Git repository is set as the **URL**, the repository is used
 as context.
 
 # OPTIONS
+**-e**, **--env**=*environment*
+   Set build-time environment variables. This option allows you to specify
+arbitrary environment variables that are available for the command(s) that will
+be executed as part of 'RUN' primitive of Dockerfile. Such a variable is only
+accessible during 'RUN' and is not persisted with the intermediate
+and final docker images, keeping the generated image portable across the
+actual deployment environments. This gives the flexibility to build
+an image by passing host specific environment variables (like http_proxy) without
+changing the Dockerfile.
+
+**--env-file**=[]
+   Read in a line delimited file of build-time environment variables.
+
 **--force-rm**=*true*|*false*
    Always remove intermediate containers, even after unsuccessful builds. The default is *false*.
 

docs/sources/reference/api/docker_remote_api.md

@@ -84,6 +84,12 @@ Volumes are now initialized when the container is created.
 **New!**
 You can now copy data which is contained in a volume.
 
+`POST /build`
+
+**New!**
+This endpoint now optionally takes a serialized array of build-time environment
+variables.
+
 ## v1.15
 
 ### Full Documentation

docs/sources/reference/api/docker_remote_api_v1.16.md

@@ -1170,7 +1170,15 @@ Query Parameters:
     Request Headers:
 
 -   **Content-type** – should be set to `"application/tar"`.
--   **X-Registry-Config** – base64-encoded ConfigFile objec
+-   **X-Registry-Config** – base64-encoded ConfigFile object
+-   **X-BuildEnv** – base64-encoded JSON array of strings of build-time environment
+    variables in the form <var>=<value>. These can be accessed like regular
+    environment variables in the 'RUN' primitive of the Dockerfile. And they are
+    not persisted in the intermediate and final images. This can be useful for
+    building images that require access to certain environment variables that
+    are specific to the build host like http-proxy; user credentials for
+    pulling intermediate files etc.
+
 
 Status Codes:
 

docs/sources/reference/commandline/cli.md

@@ -448,6 +448,8 @@ To kill the container, use `docker kill`.
 
     Build a new image from the source code at PATH
 
+      -e, --env=[]         Set build-time environment variables
+      --env-file=[]        Read in a line delimited file of build-time environment variables
       --force-rm=false     Always remove intermediate containers, even after unsuccessful builds
       --no-cache=false     Do not use cache when building the image
       -q, --quiet=false    Suppress the verbose output generated by the containers
@@ -606,6 +608,21 @@ schema.
 > children) for security reasons, and to ensure repeatable builds on remote
 > Docker hosts. This is also the reason why `ADD ../file` will not work.
 
+Sometimes building an image will require access to certain environment variables
+that are specific to the build host like http-proxy; user credentials for
+pulling intermediate files etc. Most likely these variables can't be persisted
+in the built image for portatbility or security reasons i.e. it's not desirable
+to use the 'ENV' primitive of the Dockerfile.
+
+    $ docker build -e HTTP_PROXY=http://10.20.30.2:1234 .
+
+This will allow passing the built-time environment variables, that can be
+accessed like regular environment variables in the 'RUN' primitive of the
+Dockerfile, without persisting them in the intermediate or final images.
+
+> **Note:** environment variables defined using 'ENV' primitive of Dockerfile
+> will override the build-time environment variables.
+
 ## commit
 
     Usage: docker commit [OPTIONS] CONTAINER [REPOSITORY[:TAG]]

integration-cli/docker_cli_build_test.go

@@ -4092,3 +4092,60 @@ CMD cat /foo/file`,
 
 	logDone("build - volumes retain contents in build")
 }
+
+func TestBuildBuildTimeEnv(t *testing.T) {
+	envKey := "foo"
+	envVal := "bar"
+	buildCmd := exec.Command(dockerBinary, "build", "-t", "bldenvtest", "-e",
+		fmt.Sprintf("%s=%s", envKey, envVal), "-")
+	buildCmd.Stdin = strings.NewReader(fmt.Sprintf("FROM busybox\n"+
+		"RUN echo $%s\n"+
+		"CMD echo $%s\n",
+		envKey, envKey))
+
+	if out, _, err := runCommandWithOutput(buildCmd); err != nil || !strings.Contains(out, envVal) {
+		if err != nil {
+			t.Fatalf("build failed to complete: %v %v", out, err)
+		}
+		defer func() { deleteImages("bldenvtest") }()
+		t.Fatalf("failed to access environment variable in output: '%v' "+
+			"expected: '%v'", out, envVal)
+	}
+
+	runCmd := exec.Command(dockerBinary, "run", "bldenvtest")
+	if out, _, err := runCommandWithOutput(runCmd); out != "\n" || err != nil {
+		t.Fatalf("run produced invalid output: '%q', expected '%q'", out, "")
+	}
+
+	logDone("build - build an image with build time environment variables")
+}
+
+func TestBuildBuildTimeEnvOverride(t *testing.T) {
+	envKey := "foo"
+	envVal := "bar"
+	envValOveride := "barOverride"
+	buildCmd := exec.Command(dockerBinary, "build", "-t", "bldenvtest", "-e",
+		fmt.Sprintf("%s=%s", envKey, envVal), "-")
+	buildCmd.Stdin = strings.NewReader(fmt.Sprintf("FROM busybox\n"+
+		"ENV %s %s \n"+
+		"RUN echo $%s\n"+
+		"CMD echo $%s\n",
+		envKey, envValOveride,
+		envKey, envKey))
+
+	if out, _, err := runCommandWithOutput(buildCmd); err != nil || !strings.Contains(out, envValOveride) {
+		if err != nil {
+			t.Fatalf("build failed to complete: %v %v", out, err)
+		}
+		defer func() { deleteImages("bldenvtest") }()
+		t.Fatalf("failed to access environment variable in output: '%v' "+
+			"expected: '%v'", out, envValOveride)
+	}
+
+	runCmd := exec.Command(dockerBinary, "run", "bldenvtest")
+	if out, _, err := runCommandWithOutput(runCmd); !strings.Contains(out, envValOveride) || err != nil {
+		t.Fatalf("run produced invalid output: '%q', expected '%q'", out, envValOveride)
+	}
+
+	logDone("build - build an image with build time environment variables override")
+}