From 3bf3a1ae65d1d75c1292fee4c9cce9c253422f2c Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 16 Jul 2019 12:16:56 +0200 Subject: [PATCH 1/5] Dockerfile: Use APT_MIRROR for security.debian.org as well The fastly cdn mirror we're using also mirrors the debian security repository; ``` Welcome to deb.debian.org (fastly instance)! This is deb.debian.org. This service provides mirrors for the following Debian archive repositories: /debian/ /debian-debug/ /debian-ports/ /debian-security/ The server deb.debian.org does not have packages itself, but the name has SRV records in DNS that let apt in stretch and later find places. ``` Signed-off-by: Sebastiaan van Stijn (cherry picked from commit c8f43b5f6f7c83cfb5570f7f013c7efaa430d285) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index bc02ef809a09a..f93ed5c887f55 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,9 +27,9 @@ ARG CROSS="false" FROM golang:1.12.7 AS base -# allow replacing httpredir or deb mirror -ARG APT_MIRROR=deb.debian.org -RUN sed -ri "s/(httpredir|deb).debian.org/$APT_MIRROR/g" /etc/apt/sources.list +ARG APT_MIRROR +RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list \ + && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list FROM base AS criu # Install CRIU for checkpoint/restore support From c364e5d1baafb01e1214bfcd871ccdd9cd49772c Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 17 Jul 2019 13:59:16 +0200 Subject: [PATCH 2/5] Dockerfile: use GO_VERSION build-arg for overriding Go version This allows overriding the version of Go without making modifications in the source code, which can be useful to test against multiple versions. For example: make GO_VERSION=1.13beta1 shell Signed-off-by: Sebastiaan van Stijn (cherry picked from commit c6281bc4383b7f9eab617fd73601e8594c93365b) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 3 ++- Dockerfile.e2e | 4 +++- Dockerfile.simple | 4 +++- Dockerfile.windows | 4 +++- Makefile | 2 +- hack/ci/windows.ps1 | 2 +- 6 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index f93ed5c887f55..e4e1e5b504759 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,8 +25,9 @@ # ARG CROSS="false" +ARG GO_VERSION=1.12.7 -FROM golang:1.12.7 AS base +FROM golang:${GO_VERSION} AS base ARG APT_MIRROR RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list \ && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list diff --git a/Dockerfile.e2e b/Dockerfile.e2e index f24d29a364a54..9b6c0fe544192 100644 --- a/Dockerfile.e2e +++ b/Dockerfile.e2e @@ -1,4 +1,6 @@ -FROM golang:1.12.7-alpine as base +ARG GO_VERSION=1.12.7 + +FROM golang:${GO_VERSION}-alpine AS base RUN apk --no-cache add \ bash \ diff --git a/Dockerfile.simple b/Dockerfile.simple index 303b0cfd3567c..d7b63ce9cc189 100644 --- a/Dockerfile.simple +++ b/Dockerfile.simple @@ -5,7 +5,9 @@ # This represents the bare minimum required to build and test Docker. -FROM golang:1.12.7 +ARG GO_VERSION=1.12.7 + +FROM golang:${GO_VERSION} # allow replacing httpredir or deb mirror ARG APT_MIRROR=deb.debian.org diff --git a/Dockerfile.windows b/Dockerfile.windows index 7c034019d199e..c11b373c4fca4 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -165,10 +165,12 @@ FROM microsoft/windowsservercore # Use PowerShell as the default shell SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] +ARG GO_VERSION=1.12.7 + # Environment variable notes: # - GO_VERSION must be consistent with 'Dockerfile' used by Linux. # - FROM_DOCKERFILE is used for detection of building within a container. -ENV GO_VERSION=1.12.7 ` +ENV GO_VERSION=${GO_VERSION} ` GIT_VERSION=2.11.1 ` GOPATH=C:\go ` FROM_DOCKERFILE=1 diff --git a/Makefile b/Makefile index f778902fe6e75..1c97812b2cdb9 100644 --- a/Makefile +++ b/Makefile @@ -150,7 +150,7 @@ build: DOCKER_BUILD_ARGS += --build-arg=CROSS=$(DOCKER_CROSS) build: DOCKER_BUILDKIT ?= 1 build: bundles $(warning The docker client CLI has moved to github.com/docker/cli. For a dev-test cycle involving the CLI, run:${\n} DOCKER_CLI_PATH=/host/path/to/cli/binary make shell ${\n} then change the cli and compile into a binary at the same location.${\n}) - DOCKER_BUILDKIT="${DOCKER_BUILDKIT}" docker build ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} ${DOCKER_BUILD_OPTS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" . + DOCKER_BUILDKIT="${DOCKER_BUILDKIT}" docker build --build-arg=GO_VERSION ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} ${DOCKER_BUILD_OPTS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" . bundles: mkdir bundles diff --git a/hack/ci/windows.ps1 b/hack/ci/windows.ps1 index 6d87f3256b30e..3e4c83a93db47 100644 --- a/hack/ci/windows.ps1 +++ b/hack/ci/windows.ps1 @@ -453,7 +453,7 @@ Try { Write-Host -ForegroundColor Cyan "`n`nINFO: Building the image from Dockerfile.windows at $(Get-Date)..." Write-Host $ErrorActionPreference = "SilentlyContinue" - $Duration=$(Measure-Command { docker build -t docker -f Dockerfile.windows . | Out-Host }) + $Duration=$(Measure-Command { docker build --build-arg=GO_VERSION -t docker -f Dockerfile.windows . | Out-Host }) $ErrorActionPreference = "Stop" if (-not($LastExitCode -eq 0)) { Throw "ERROR: Failed to build image from Dockerfile.windows" From 97ca6434e0cf3fe23fe1d8c45c4b776ee2cd8043 Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Thu, 18 Jul 2019 13:53:38 -0700 Subject: [PATCH 3/5] TESTING.md: document GO_VERSION Signed-off-by: Kir Kolyshkin (cherry picked from commit a55753877016eaa3ed577774c11c2428f95ac433) Signed-off-by: Sebastiaan van Stijn --- TESTING.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/TESTING.md b/TESTING.md index b2c53769ca052..15322f9eedf33 100644 --- a/TESTING.md +++ b/TESTING.md @@ -87,3 +87,10 @@ To run the integration test suite: ``` make test-integration ``` + +You can change a version of golang used for building stuff that is being tested +by setting `GO_VERSION` variable, for example: + +``` +make GO_VERSION=1.12.7 test +``` From 640193b2bb4ea2774a4dac7d8dfac23f76288957 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 20 Jul 2019 12:54:50 +0200 Subject: [PATCH 4/5] Windows: fix Golang version checks for GO_VERSION build-arg This check was used to make sure we don't bump Go versions independently (Linux/Windows). The Dockerfile switched to using a build-arg to allow overriding the Go version, which rendered this check non-functional. It also fails if Linux versions use a specific variant of the image; 08:41:31 ERROR: Failed 'ERROR: Mismatched GO versions between Dockerfile and Dockerfile.windows. Update your PR to ensure that both files are updated and in sync. ${GO_VERSION}-stretch ${GO_VERSION}' at 07/20/2019 08:41:31 08:41:31 At C:\gopath\src\github.com\docker\docker\hack\ci\windows.ps1:448 char:9 08:41:31 + Throw "ERROR: Mismatched GO versions between Dockerfile and D ... 08:41:31 + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This patch fixes the check by looking for the value of `GO_VERSION` instead of looking at the `FROM` line (which is harder to parse). Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 4fa57a8191b1d23c6466725b688519f83c0ac5dd) Signed-off-by: Sebastiaan van Stijn --- hack/ci/windows.ps1 | 17 +++-------------- hack/make.ps1 | 2 +- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/hack/ci/windows.ps1 b/hack/ci/windows.ps1 index 3e4c83a93db47..8828f73d01f63 100644 --- a/hack/ci/windows.ps1 +++ b/hack/ci/windows.ps1 @@ -426,20 +426,9 @@ Try { Write-Host -ForegroundColor Green "INFO: Location for testing is $env:TEMP" # CI Integrity check - ensure Dockerfile.windows and Dockerfile go versions match - $goVersionDockerfileWindows=$(Get-Content ".\Dockerfile.windows" | Select-String "^ENV GO_VERSION" | Select-object -First 1).ToString().Replace("ENV GO_VERSION=","").Replace("\","").Replace("``","").Trim() - $goVersionDockerfile=$(Get-Content ".\Dockerfile" | Select-String "^ENV GO_VERSION" | Select-object -First 1) - - # As of go 1.11, Dockerfile changed to be in the format like "FROM golang:1.11.0 AS base". - # If a version number ends with .0 (as in 1.11.0, a convention used in golang docker - # image versions), it needs to be removed (i.e. "1.11.0" becomes "1.11"). - if ($null -eq $goVersionDockerfile) { - $goVersionDockerfile=$(Get-Content ".\Dockerfile" | Select-String "^FROM golang:" | Select-object -First 1) - if ($null -ne $goVersionDockerfile) { - $goVersionDockerfile = $goVersionDockerfile.ToString().Split(" ")[1].Split(":")[1] -replace '\.0$','' - } - } else { - $goVersionDockerfile = $goVersionDockerfile.ToString().Split(" ")[2] - } + $goVersionDockerfileWindows=(Select-String -Path ".\Dockerfile.windows" -Pattern "^ARG[\s]+GO_VERSION=(.*)$").Matches.groups[1].Value + $goVersionDockerfile=(Select-String -Path ".\Dockerfile" -Pattern "^ARG[\s]+GO_VERSION=(.*)$").Matches.groups[1].Value + if ($null -eq $goVersionDockerfile) { Throw "ERROR: Failed to extract golang version from Dockerfile" } diff --git a/hack/make.ps1 b/hack/make.ps1 index 6e5bc45e50a62..3c51716ed39d7 100644 --- a/hack/make.ps1 +++ b/hack/make.ps1 @@ -134,7 +134,7 @@ Function Check-InContainer() { # outside of a container where it may be out of date with master. Function Verify-GoVersion() { Try { - $goVersionDockerfile=(Select-String -Path ".\Dockerfile" -Pattern "^FROM golang:").ToString().Split(" ")[1].SubString(7) -replace '\.0$','' + $goVersionDockerfile=(Select-String -Path ".\Dockerfile" -Pattern "^ARG[\s]+GO_VERSION=(.*)$").Matches.groups[1].Value.TrimEnd(".0") $goVersionInstalled=(go version).ToString().Split(" ")[2].SubString(2) } Catch [Exception] { From 2f1984c6df21fc1d7685876d0b0be5ef02f6ed2e Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 20 Jul 2019 10:32:08 +0200 Subject: [PATCH 5/5] Pin Dockerfile to -stretch variant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Golang base images switch to buster, which causes some breakage in networking and packages that are no longer available; (`btrfs-tools` is now an empty package, and `libprotobuf-c0-dev` is gone). Some of out tests also start faiilng on stretch, and will have to be investigated further; ``` 15:13:06 --- FAIL: TestRenameAnonymousContainer (3.37s) 15:13:06 rename_test.go:168: assertion failed: 0 (int) != 1 (inspect.State.ExitCode int): container a7fe866d588d65f353f42ffc5ea5288e52700384e1d90850e9c3d4dce8657666 exited with the wrong exitcode: 15:13:38 --- FAIL: TestHostnameDnsResolution (2.23s) 15:13:38 run_linux_test.go:128: assertion failed: 15:13:38 --- ← 15:13:38 +++ → 15:13:38 @@ -1 +1,2 @@ 15:13:38 +ping: bad address 'foobar' 15:13:38 15:13:38 15:13:38 run_linux_test.go:129: assertion failed: 0 (int) != 1 (res.ExitCode int) ``` Signed-off-by: Sebastiaan van Stijn (cherry picked from commit ed672bb523cb255d0b2b79837d9c45a7c3255000) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 +- Dockerfile.simple | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e4e1e5b504759..861655c64cba6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ ARG CROSS="false" ARG GO_VERSION=1.12.7 -FROM golang:${GO_VERSION} AS base +FROM golang:${GO_VERSION}-stretch AS base ARG APT_MIRROR RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list \ && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list diff --git a/Dockerfile.simple b/Dockerfile.simple index d7b63ce9cc189..ff64fa4f5443c 100644 --- a/Dockerfile.simple +++ b/Dockerfile.simple @@ -7,7 +7,7 @@ ARG GO_VERSION=1.12.7 -FROM golang:${GO_VERSION} +FROM golang:${GO_VERSION}-stretch # allow replacing httpredir or deb mirror ARG APT_MIRROR=deb.debian.org