all: Switch to go mod

Signed-off-by: Jonathan Rudenberg <jonathan@titanous.com>

blobstore/backend/gcs.go

@@ -126,7 +126,7 @@ func (b *gcsBackend) Copy(tx *postgres.DBTx, dst, src FileInfo) error {
 		return err
 	}
 
-	_, err := b.bucket.Object(src.ExternalID).CopyTo(context.Background(), b.bucket.Object(dst.ExternalID), nil)
+	_, err := b.bucket.Object(dst.ExternalID).CopierFrom(b.bucket.Object(src.ExternalID)).Run(context.Background())
 	return err
 }
 

builder/build.go

@@ -135,6 +135,9 @@ type Layer struct {
 	// are then added as inputs.
 	CGoBuild map[string]string `json:"cgobuild,omitempty"`
 
+	// GoBin is a list of binaries to build using 'gobin' and install into `/bin`.
+	GoBin []string `json:"gobin,omitempty"`
+
 	// Copy is a set of inputs to copy into the layer
 	Copy map[string]string `json:"copy,omitempty"`
 
@@ -348,8 +351,11 @@ func (b *Builder) Build(images []*Image) error {
 		}
 		for _, l := range image.Layers {
 			// build Go binaries using the Go image
-			if l.BuildWith == "" && (len(l.GoBuild) > 0 || len(l.CGoBuild) > 0) {
-				l.BuildWith = "go"
+			if len(l.GoBuild) > 0 || len(l.CGoBuild) > 0 || len(l.GoBin) > 0 {
+				if l.BuildWith == "" {
+					l.BuildWith = "go"
+				}
+				l.Inputs = append(l.Inputs, "go.mod", "go.sum")
 			}
 			if l.BuildWith != "" {
 				addDependency(build, l.BuildWith)
@@ -462,6 +468,12 @@ func (b *Builder) BuildImage(image *Image) error {
 			inputs = append(inputs, paths...)
 		}
 
+		if len(l.GoBin) > 0 {
+			for _, bin := range l.GoBin {
+				run = append(run, "GOBIN=/bin GOBIN_CACHE=/tmp/gobin gobin -m "+bin)
+			}
+		}
+
 		// if building Go binaries, load Go inputs for the configured
 		// GOOS / GOARCH and build with 'go build' / 'cgo build'
 		if len(l.GoBuild) > 0 || len(l.CGoBuild) > 0 {
@@ -480,7 +492,7 @@ func (b *Builder) BuildImage(image *Image) error {
 					return err
 				}
 				inputs = append(inputs, i...)
-				run = append(run, fmt.Sprintf("go build -o %s %s", l.GoBuild[dir], filepath.Join("github.com/flynn/flynn", dir)))
+				run = append(run, fmt.Sprintf("go build -o %s %s", l.GoBuild[dir], "./"+dir))
 			}
 			dirs = make([]string, 0, len(l.CGoBuild))
 			for dir := range l.CGoBuild {
@@ -493,7 +505,7 @@ func (b *Builder) BuildImage(image *Image) error {
 					return err
 				}
 				inputs = append(inputs, i...)
-				run = append(run, fmt.Sprintf("cgo build -o %s %s", l.CGoBuild[dir], filepath.Join("github.com/flynn/flynn", dir)))
+				run = append(run, fmt.Sprintf("cgo build -o %s %s", l.CGoBuild[dir], "./"+dir))
 			}
 		}
 
@@ -822,6 +834,12 @@ func (b *Builder) BuildLayer(l *Layer, id, name string, run []string, env map[st
 	linuxCapabilities := append(host.DefaultCapabilities, l.LinuxCapabilities...)
 	job.Config.LinuxCapabilities = &linuxCapabilities
 
+	if l.Limits == nil {
+		l.Limits = make(map[string]string)
+	}
+	if l.Limits["temp_disk"] == "" {
+		l.Limits["temp_disk"] = "1G"
+	}
 	for typ, v := range l.Limits {
 		limit, err := resource.ParseLimit(resource.Type(typ), v)
 		if err != nil {

builder/go-wrapper.sh

@@ -4,13 +4,28 @@
 # inside the Go image and sets some environment variables and flags
 # before running the go tool.
 
+set -eo pipefail
+
 export GOROOT="/usr/local/go"
 export GOPATH="/go"
+export GO111MODULE=on
+
+if [[ -z "$GOFLAGS" ]]; then
+  # make sure we don't overwrite flags for `go list` run by gobin
+  export GOFLAGS=-mod=vendor
+fi
 
 if [[ $(basename $0) != "cgo" ]]; then
   export CGO_ENABLED=0
 fi
 
+BIN="${GOROOT}/bin/go"
+if [[ $(basename $0) == "gobin" ]]; then
+  export GOPROXY=https://proxy.golang.org
+  export GOFLAGS=-mod=readonly
+  BIN=/usr/local/bin/gobin-noenv
+fi
+
 GO_LDFLAGS="-X github.com/flynn/flynn/pkg/version.version=${FLYNN_VERSION}"
 
 if [[ -n "${TUF_ROOT_KEYS}" ]]; then
@@ -21,4 +36,8 @@ if [[ -n "${TUF_REPOSITORY}" ]]; then
   GO_LDFLAGS="${GO_LDFLAGS} -X github.com/flynn/flynn/pkg/tufconfig.Repository=${TUF_REPOSITORY}"
 fi
 
-${GOROOT}/bin/go $1 -ldflags "${GO_LDFLAGS}" ${@:2}
+if [[ "$1" = "build" ]]; then
+	${BIN} $1 -ldflags "${GO_LDFLAGS}" ${@:2}
+else
+	${BIN} "$@"
+fi

builder/gobin/gobin.go

@@ -0,0 +1,8 @@
+package gobin
+
+import (
+	// import hack for gobin
+	_ "github.com/go-bindata/go-bindata"
+	// for github.com/flynn/go-tuf/cmd/tuf
+	_ "github.com/dustin/go-humanize"
+)

builder/img/go.sh

@@ -2,16 +2,20 @@
 
 set -eo pipefail
 
-version="1.12.8"
-shasum="bd26cd4962a362ed3c11835bca32c2e131c2ae050304f2c4df9fa6ded8db85d2"
+go_version="1.12.8"
+go_shasum="bd26cd4962a362ed3c11835bca32c2e131c2ae050304f2c4df9fa6ded8db85d2"
+protoc_version="3.3.0"
+protoc_shasum="feb112bbc11ea4e2f7ef89a359b5e1c04428ba6cfa5ee628c410eccbfe0b64c3"
+gobin_commit="ef6664e41f0bfe3007869844d318bb2bfa2627f9"
 dir="/usr/local"
 
 apt-get update
 apt-get install --yes git build-essential unzip
 apt-get clean
 
-curl -fsSLo /tmp/go.tar.gz "https://storage.googleapis.com/golang/go${version}.linux-amd64.tar.gz"
-echo "${shasum}  /tmp/go.tar.gz" | shasum -c -
+curl -fsSLo /tmp/go.tar.gz "https://storage.googleapis.com/golang/go${go_version}.linux-amd64.tar.gz"
+echo "${go_shasum}  /tmp/go.tar.gz" | shasum -c -
+rm -rf "${dir}/go"
 tar xzf /tmp/go.tar.gz -C "${dir}"
 rm /tmp/go.tar.gz
 
@@ -22,13 +26,19 @@ export PATH="${GOROOT}/bin:${PATH}"
 # install protobuf compiler
 tmpdir=$(mktemp --directory)
 trap "rm -rf ${tmpdir}" EXIT
-curl -sL https://github.com/google/protobuf/releases/download/v3.3.0/protoc-3.3.0-linux-x86_64.zip > "${tmpdir}/protoc.zip"
+curl -sL https://github.com/google/protobuf/releases/download/v${protoc_version}/protoc-${protoc_version}-linux-x86_64.zip > "${tmpdir}/protoc.zip"
+echo "${protoc_shasum}  ${tmpdir}/protoc.zip" | shasum -c -
 unzip -d "${tmpdir}/protoc" "${tmpdir}/protoc.zip"
+rm -rf /opt/protoc /usr/local/bin/protoc
 mv "${tmpdir}/protoc" /opt
 ln -s /opt/protoc/bin/protoc /usr/local/bin/protoc
 
-mkdir -p "${GOPATH}/src/github.com/flynn"
-ln -nfs "$(pwd)" "${GOPATH}/src/github.com/flynn/flynn"
-
 cp "builder/go-wrapper.sh" "/usr/local/bin/go"
 cp "builder/go-wrapper.sh" "/usr/local/bin/cgo"
+cp "builder/go-wrapper.sh" "/usr/local/bin/gobin"
+
+# install gobin
+git clone https://github.com/flynn/gobin "${tmpdir}/gobin"
+cd "${tmpdir}/gobin"
+git reset --hard ${gobin_commit}
+/usr/local/bin/go build -o /usr/local/bin/gobin-noenv

builder/manifest.json

@@ -34,17 +34,15 @@
     {
       "id": "busybox",
       "layers": [{
-        "script": "builder/img/busybox.sh",
-        "limits": { "temp_disk": "1G" }
+        "script": "builder/img/busybox.sh"
       }]
     },
     {
       "id": "go",
       "base": "ubuntu-bionic",
       "layers": [{
         "script": "builder/img/go.sh",
-        "inputs": ["builder/go-wrapper.sh"],
-        "limits": { "temp_disk": "1G" }
+        "inputs": ["builder/go-wrapper.sh"]
       }]
     },
     {
@@ -60,11 +58,10 @@
       "id": "build-tools",
       "base": "busybox",
       "layers": [{
-        "gobuild": {
-          "vendor/github.com/golang/dep/cmd/dep": "/bin/dep",
-          "vendor/github.com/flynn/go-tuf/cmd/tuf": "/bin/tuf",
-          "vendor/github.com/flynn/go-tuf/cmd/tuf-client": "/bin/tuf-client"
-        }
+        "gobin": [
+          "github.com/flynn/go-tuf/cmd/tuf",
+          "github.com/flynn/go-tuf/cmd/tuf-client"
+        ]
       }]
     },
     {
@@ -110,7 +107,6 @@
         },
         {
           "name": "host-binaries",
-          "limits": { "temp_disk": "1G" },
           "env": {
             "FLYNN_VERSION": "{{ .Version }}",
             "TUF_ROOT_KEYS": "{{ .TUFRootKeys }}"
@@ -446,8 +442,7 @@
           "inputs": [
             "slugbuilder/builder/buildpacks.txt",
             "slugbuilder/builder/install-buildpack"
-          ],
-          "limits": { "temp_disk": "1G" }
+          ]
         },
         {
           "name": "slugbuilder-binaries",
@@ -478,8 +473,7 @@
           "inputs": [
             "slugbuilder/builder/buildpacks.txt",
             "slugbuilder/builder/install-buildpack"
-          ],
-          "limits": { "temp_disk": "1G" }
+          ]
         },
         {
           "name": "slugbuilder-binaries",
@@ -535,14 +529,13 @@
         "script": "util/assetbuilder/build.sh",
         "env": {"TARGET": "dashboard"},
         "gobuild": {
-          "dashboard/app": "/bin/dashboard-compile",
-          "vendor/github.com/jteeuwen/go-bindata/go-bindata": "/bin/go-bindata"
+          "dashboard/app": "/bin/dashboard-compile"
         },
+	"gobin": ["github.com/go-bindata/go-bindata/go-bindata"],
         "inputs": [
           "dashboard/app/Gemfile",
           "dashboard/app/Gemfile.lock"
-        ],
-        "limits": { "temp_disk": "1G" }
+        ]
       }]
     },
     {
@@ -574,7 +567,7 @@
           "dashboard/app/vendor/javascripts/*.js",
           "dashboard/app/vendor/stylesheets/*.scss"
         ],
-        "limits": { "memory": "3G", "temp_disk": "1G" }
+        "limits": { "memory": "3G" }
       }]
     },
     {
@@ -587,8 +580,7 @@
         ],
         "gobuild": {
           "dashboard": "/bin/flynn-dashboard"
-        },
-        "limits": { "temp_disk": "1G" }
+        }
       }],
       "entrypoint": {
         "args": [
@@ -674,7 +666,6 @@
         },
         {
           "name": "test-binaries",
-          "limits": { "temp_disk": "1G" },
           "cgobuild": {
             "test":                  "/bin/flynn-test",
             "test/runner":           "/bin/flynn-test-runner"
@@ -710,8 +701,7 @@
         {
           "build_with": "ubuntu-bionic",
           "name": "test-apps-minio",
-          "script": "test/apps/minio.sh",
-          "limits": { "temp_disk": "1G" }
+          "script": "test/apps/minio.sh"
         },
         {
           "name": "test-apps-binaries",
@@ -739,8 +729,7 @@
         },
         "gobuild": {
           "cli": "/bin/flynn-${GOOS}-${GOARCH}"
-        },
-        "limits": { "temp_disk": "1G" }
+        }
       }]
     }
   },