add e2e tests to verify env variables priority

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/AlecAivazis/survey/v2 v2.3.2
 	github.com/buger/goterm v1.0.4
 	github.com/cnabio/cnab-to-oci v0.3.1-beta1
-	github.com/compose-spec/compose-go v1.2.6
+	github.com/compose-spec/compose-go v1.2.7
 	github.com/containerd/console v1.0.3
 	github.com/containerd/containerd v1.6.2
 	github.com/distribution/distribution/v3 v3.0.0-20210316161203-a01c71e2477e
@@ -69,7 +69,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
@@ -123,7 +123,7 @@ require (
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.0 // indirect
 	k8s.io/apimachinery v0.23.4 // indirect; see replace for the actual version used
 	k8s.io/client-go v0.23.4 // indirect; see replace for the actual version used
 	k8s.io/klog/v2 v2.30.0 // indirect

go.sum

@@ -301,8 +301,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
 github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/compose-spec/compose-go v1.0.8/go.mod h1:REnCbBugoIdHB7S1sfkN/aJ7AJpNApGNjNiVjA9L8x4=
-github.com/compose-spec/compose-go v1.2.6 h1:9l2Y/yNn/OSngnkUBP8h8CchTmMcf1MW4BeUEaZXy8k=
-github.com/compose-spec/compose-go v1.2.6/go.mod h1:cg8yTeI+7rfQsEW9XHOLx0sNVjGKEXr6XwVB4fxmG3A=
+github.com/compose-spec/compose-go v1.2.7 h1:eqKGZhdOQEGKW/FmFDt4xyEPhCpbA5dr0PkcWD895aI=
+github.com/compose-spec/compose-go v1.2.7/go.mod h1:Jl9L8zJrt4aGY1XAz03DvHAu8V3/f00TK+uJL4BayDU=
 github.com/compose-spec/godotenv v1.1.1/go.mod h1:zF/3BOa18Z24tts5qnO/E9YURQanJTBUf7nlcCTNsyc=
 github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
 github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
@@ -870,8 +870,9 @@ github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
 github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
-github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
+github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ=
@@ -2139,8 +2140,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=

pkg/compose/run.go

@@ -116,6 +116,9 @@ func applyRunOptions(project *types.Project, service *types.ServiceConfig, opts
 	if len(opts.Environment) > 0 {
 		env := types.NewMappingWithEquals(opts.Environment)
 		projectEnv := env.Resolve(func(s string) (string, bool) {
+			if _, ok := service.Environment[s]; ok {
+				return "", false
+			}
 			v, ok := project.Environment[s]
 			return v, ok
 		}).RemoveEmpty()

pkg/e2e/compose_environment_test.go

@@ -0,0 +1,169 @@
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package e2e
+
+import (
+	"os"
+	"strings"
+	"testing"
+
+	"gotest.tools/v3/assert"
+	"gotest.tools/v3/icmd"
+)
+
+func TestEnvPriority(t *testing.T) {
+	c := NewParallelE2eCLI(t, binDir)
+
+	projectDir := "./fixtures/environment/env-priority"
+
+	t.Run("up", func(t *testing.T) {
+		c.RunDockerOrExitError("rmi", "env-compose-priority")
+		c.RunDockerComposeCmd("-f", "./fixtures/environment/env-priority/compose-with-env.yaml",
+			"--project-directory", projectDir, "up", "-d", "--build")
+	})
+
+	// Full options activated
+	// 1. Compose file <-- Result expected
+	// 2. Shell environment variables
+	// 3. Environment file
+	// 4. Dockerfile
+	// 5. Variable is not defined
+	t.Run("compose file priority", func(t *testing.T) {
+		os.Setenv("WHEREAMI", "shell") //nolint:errcheck
+		defer os.Unsetenv("WHEREAMI")  //nolint:errcheck
+
+		res := c.RunDockerComposeCmd("-f", "./fixtures/environment/env-priority/compose-with-env.yaml",
+			"--project-directory", projectDir, "--env-file", "./fixtures/environment/env-priority/.env.override",
+			"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
+
+		assert.Equal(t, strings.TrimSpace(res.Stdout()), "Compose File")
+	})
+
+	// No Compose file, all other options
+	// 1. Compose file
+	// 2. Shell environment variables <-- Result expected
+	// 3. Environment file
+	// 4. Dockerfile
+	// 5. Variable is not defined
+	t.Run("shell priority", func(t *testing.T) {
+		os.Setenv("WHEREAMI", "shell") //nolint:errcheck
+		defer os.Unsetenv("WHEREAMI")  //nolint:errcheck
+
+		res := c.RunDockerComposeCmd("-f", "./fixtures/environment/env-priority/compose.yaml",
+			"--project-directory", projectDir, "--env-file", "./fixtures/environment/env-priority/.env.override",
+			"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
+		assert.Equal(t, strings.TrimSpace(res.Stdout()), "shell")
+	})
+
+	//  No Compose file and env variable pass to the run command
+	// 1. Compose file
+	// 2. Shell environment variables <-- Result expected
+	// 3. Environment file
+	// 4. Dockerfile
+	// 5. Variable is not defined
+	t.Run("shell priority from run command", func(t *testing.T) {
+		res := c.RunDockerComposeCmd("-f", "./fixtures/environment/env-priority/compose.yaml",
+			"--project-directory", projectDir, "--env-file", "./fixtures/environment/env-priority/.env.override",
+			"run", "--rm", "-e", "WHEREAMI=shell-run", "env-compose-priority")
+		assert.Equal(t, strings.TrimSpace(res.Stdout()), "shell-run")
+	})
+
+	//  No Compose file & no env variable but override env file
+	// 1. Compose file
+	// 2. Shell environment variables
+	// 3. Environment file <-- Result expected
+	// 4. Dockerfile
+	// 5. Variable is not defined
+	t.Run("override env file", func(t *testing.T) {
+		res := c.RunDockerComposeCmd("-f", "./fixtures/environment/env-priority/compose.yaml",
+			"--project-directory", projectDir, "--env-file", "./fixtures/environment/env-priority/.env.override",
+			"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
+		assert.Equal(t, strings.TrimSpace(res.Stdout()), "override")
+	})
+
+	//  No Compose file & no env variable but override env file
+	// 1. Compose file
+	// 2. Shell environment variables
+	// 3. Environment file <-- Result expected
+	// 4. Dockerfile
+	// 5. Variable is not defined
+	t.Run("env file", func(t *testing.T) {
+		res := c.RunDockerComposeCmd("-f", "./fixtures/environment/env-priority/compose.yaml",
+			"--project-directory", projectDir, "run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
+		assert.Equal(t, strings.TrimSpace(res.Stdout()), "Env File")
+	})
+
+	//  No Compose file & no env variable, using an empty override env file
+	// 1. Compose file
+	// 2. Shell environment variables
+	// 3. Environment file
+	// 4. Dockerfile   <-- Result expected
+	// 5. Variable is not defined
+	t.Run("use Dockerfile", func(t *testing.T) {
+		res := c.RunDockerComposeCmd("-f", "./fixtures/environment/env-priority/compose.yaml",
+			"--project-directory", projectDir, "--env-file", "./fixtures/environment/env-priority/.env.empty",
+			"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
+		assert.Equal(t, strings.TrimSpace(res.Stdout()), "Dockerfile")
+	})
+
+	t.Run("down", func(t *testing.T) {
+		c.RunDockerComposeCmd("--project-directory", projectDir, "down")
+	})
+}
+
+func TestEnvInterpolation(t *testing.T) {
+	c := NewParallelE2eCLI(t, binDir)
+
+	projectDir := "./fixtures/environment/env-interpolation"
+
+	//  No variable defined in the Compose file and env variable pass to the run command
+	// 1. Compose file
+	// 2. Shell environment variables <-- Result expected
+	// 3. Environment file
+	// 4. Dockerfile
+	// 5. Variable is not defined
+	t.Run("shell priority from run command", func(t *testing.T) {
+		os.Setenv("WHEREAMI", "shell") //nolint:errcheck
+		defer os.Unsetenv("WHEREAMI")  //nolint:errcheck
+		res := c.RunDockerComposeCmd("-f", "./fixtures/environment/env-interpolation/compose.yaml",
+			"--project-directory", projectDir, "config")
+
+		res.Assert(t, icmd.Expected{Out: `IMAGE: default_env:shell`})
+	})
+}
+
+func TestCommentsInEnvFile(t *testing.T) {
+	c := NewParallelE2eCLI(t, binDir)
+
+	projectDir := "./fixtures/environment/env-file-comments"
+
+	t.Run("comments in env files", func(t *testing.T) {
+		c.RunDockerOrExitError("rmi", "env-file-comments")
+
+		c.RunDockerComposeCmd("-f", "./fixtures/environment/env-file-comments/compose.yaml",
+			"--project-directory", projectDir, "up", "-d", "--build")
+
+		res := c.RunDockerComposeCmd("-f", "./fixtures/environment/env-file-comments/compose.yaml",
+			"--project-directory", projectDir, "run", "--rm",
+			"-e", "COMMENT", "-e", "NO_COMMENT", "env-file-comments")
+
+		res.Assert(t, icmd.Expected{Out: `COMMENT=1234`})
+		res.Assert(t, icmd.Expected{Out: `NO_COMMENT=1234#5`})
+
+		c.RunDockerComposeCmd("--project-directory", projectDir, "down", "--rmi", "all")
+	})
+}

pkg/e2e/fixtures/environment/env-file-comments/.env

@@ -0,0 +1,2 @@
+COMMENT=1234#5
+NO_COMMENT="1234#5"

pkg/e2e/fixtures/environment/env-file-comments/Dockerfile

@@ -0,0 +1,18 @@
+#   Copyright 2020 Docker Compose CLI authors
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+FROM alpine
+ENV  COMMENT=Dockerfile
+ENV  NO_COMMENT=Dockerfile
+CMD ["sh", "-c", "printenv", "|", "grep", "COMMENT"]

pkg/e2e/fixtures/environment/env-file-comments/compose.yaml

@@ -0,0 +1,5 @@
+services:
+  env-file-comments:
+    build:
+      context: .
+    image: env-file-comments

pkg/e2e/fixtures/environment/env-interpolation/.env

@@ -0,0 +1,2 @@
+WHEREAMI=Env File
+IMAGE=default_env:${WHEREAMI}

pkg/e2e/fixtures/environment/env-interpolation/compose.yaml

@@ -0,0 +1,6 @@
+services:
+  env-interpolation:
+    image: bash
+    environment:
+      IMAGE: ${IMAGE}
+    command: echo "$IMAGE"

pkg/e2e/fixtures/environment/env-priority/.env

@@ -0,0 +1 @@
+WHEREAMI=Env File

pkg/e2e/fixtures/environment/env-priority/.env.override

@@ -0,0 +1 @@
+WHEREAMI=override

pkg/e2e/fixtures/environment/env-priority/Dockerfile

@@ -0,0 +1,17 @@
+#   Copyright 2020 Docker Compose CLI authors
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+FROM alpine
+ENV  WHEREAMI=Dockerfile
+CMD ["printenv", "WHEREAMI"]

pkg/e2e/fixtures/environment/env-priority/compose-with-env.yaml

@@ -0,0 +1,7 @@
+services:
+  env-compose-priority:
+    image: env-compose-priority
+    build:
+      context: .
+    environment:
+      WHEREAMI: "Compose File"

pkg/e2e/fixtures/environment/env-priority/compose.yaml

@@ -0,0 +1,3 @@
+services:
+  env-compose-priority:
+    image: env-compose-priority