Bring in build_tools 2.0

.circleci/config.yml

@@ -144,7 +144,7 @@ jobs:
           name: Build the ddev executables
 
       - run:
-          command: bin/linux/ddev version
+          command: .gotmp/bin/ddev version
           name: ddev version information
 
       # We only build the xz version of the docker images on tag build.

.circleci/generate_artifacts.sh

@@ -4,17 +4,19 @@
 # Optional second arg is whether to build xz version of ddev_docker_images.tar
 
 set -o errexit
+set -o pipefail
+set -o nounset
 
-ARTIFACTS=$1
+ARTIFACTS=${1:-/artifacts}
 # We only build the xz artifacts if $2 ($BUILD_XZ) is not empty.
-BUILD_XZ=$2
+BUILD_XZ=${2:-}
 BASE_DIR=$PWD
 
-sudo mkdir $ARTIFACTS && sudo chmod 777 $ARTIFACTS
+sudo mkdir -p $ARTIFACTS && sudo chmod 777 $ARTIFACTS
 export VERSION=$(git describe --tags --always --dirty)
 
 # Make sure we have all our docker images, and save them in a tarball
-$BASE_DIR/bin/linux/ddev version | awk '/drud\// {print $2;}' >/tmp/images.txt
+.gotmp/bin/ddev version | awk '/drud\// {print $2;}' >/tmp/images.txt
 for item in $(cat /tmp/images.txt); do
   docker pull $item
 done
@@ -25,23 +27,23 @@ if [ ! -z "$BUILD_XZ" ] ; then
 fi
 
 # Generate and place extra items like autocomplete
-bin/linux/ddev_gen_autocomplete
-for dir in bin/darwin/darwin_amd64 bin/linux bin/windows/windows_amd64; do
-  cp bin/ddev_bash_completion.sh $dir
+.gotmp/bin/ddev_gen_autocomplete
+for dir in .gotmp/bin/darwin_amd64 .gotmp/bin/windows_amd64; do
+  cp .gotmp/bin/ddev_bash_completion.sh $dir
 done
 
 # Generate macOS tarball/zipball
-cd $BASE_DIR/bin/darwin/darwin_amd64
+cd $BASE_DIR/.gotmp/bin/darwin_amd64
 tar -czf $ARTIFACTS/ddev_macos.$VERSION.tar.gz ddev ddev_bash_completion.sh
 zip $ARTIFACTS/ddev_macos.$VERSION.zip ddev ddev_bash_completion.sh
 
 # Generate linux tarball/zipball
-cd $BASE_DIR/bin/linux
+cd $BASE_DIR/.gotmp/bin
 tar -czf $ARTIFACTS/ddev_linux.$VERSION.tar.gz ddev ddev_bash_completion.sh
 zip $ARTIFACTS/ddev_linux.$VERSION.zip ddev ddev_bash_completion.sh
 
 # generate windows tarball/zipball
-cd $BASE_DIR/bin/windows/windows_amd64
+cd $BASE_DIR/.gotmp/bin/windows_amd64
 tar -czf $ARTIFACTS/ddev_windows.$VERSION.tar.gz ddev.exe ddev_bash_completion.sh
 zip $ARTIFACTS/ddev_windows.$VERSION.zip ddev.exe ddev_bash_completion.sh
 cp ddev_windows_installer*.exe $ARTIFACTS

.golangci.yml

@@ -0,0 +1,3 @@
+run:
+  modules-download-mode: vendor
+

Makefile

@@ -65,25 +65,25 @@ TESTOS = $(shell uname -s | tr '[:upper:]' '[:lower:]')
 TEST_TIMEOUT=120m
 BUILD_ARCH = $(shell go env GOARCH)
 ifeq ($(BUILD_OS),linux)
-    DDEV_BINARY_FULLPATH=$(PWD)/bin/$(BUILD_OS)/ddev
+    DDEV_BINARY_FULLPATH=$(PWD)/$(GOTMP)/bin/ddev
 endif
 
 ifeq ($(BUILD_OS),windows)
-    DDEV_BINARY_FULLPATH=$(PWD)/bin/$(BUILD_OS)/$(BUILD_OS)_$(BUILD_ARCH)/ddev.exe
+    DDEV_BINARY_FULLPATH=$(PWD)/$(GOTMP)/bin/$(BUILD_OS)_$(BUILD_ARCH)/ddev.exe
 endif
 
 ifeq ($(BUILD_OS),darwin)
-    DDEV_BINARY_FULLPATH=$(PWD)/bin/$(BUILD_OS)/$(BUILD_OS)_$(BUILD_ARCH)/ddev
+    DDEV_BINARY_FULLPATH=$(PWD)/$(GOTMP)/bin/$(BUILD_OS)_$(BUILD_ARCH)/ddev
 endif
 
 # Override test section with tests specific to ddev
 test: testpkg testcmd
 
 testcmd: $(BUILD_OS) setup
-	DDEV_NO_SENTRY=true CGO_ENABLED=0 DDEV_BINARY_FULLPATH=$(DDEV_BINARY_FULLPATH) go test -p 1 -timeout $(TEST_TIMEOUT) -v -installsuffix static -ldflags '$(LDFLAGS)' ./cmd/... $(TESTARGS)
+	DDEV_NO_SENTRY=true CGO_ENABLED=0 DDEV_BINARY_FULLPATH=$(DDEV_BINARY_FULLPATH) go test $(USEMODVENDOR) -p 1 -timeout $(TEST_TIMEOUT) -v -installsuffix static -ldflags '$(LDFLAGS)' ./cmd/... $(TESTARGS)
 
 testpkg: setup
-	DDEV_NO_SENTRY=true CGO_ENABLED=0 go test -p 1 -timeout $(TEST_TIMEOUT) -v -installsuffix static -ldflags '$(LDFLAGS)' ./pkg/... $(TESTARGS)
+	DDEV_NO_SENTRY=true CGO_ENABLED=0 go test $(USEMODVENDOR) -p 1 -timeout $(TEST_TIMEOUT) -v -installsuffix static -ldflags '$(LDFLAGS)' ./pkg/... $(TESTARGS)
 
 setup:
 	@(mv -f ~/.ddev/global_config.yaml ~/.ddev/global_config.yaml.bak 2>/dev/null && echo "Warning: Removed your global ddev config file") || true
@@ -94,10 +94,10 @@ setup:
 # Required static analysis targets used in circleci - these cause fail if they don't work
 staticrequired: golangci-lint
 
-windows_install: windows bin/windows/windows_amd64/sudo.exe bin/windows/windows_amd64/sudo_license.txt
+windows_install: windows $(GOTMP)/bin/windows_amd64/sudo.exe $(GOTMP)/bin/windows_amd64/sudo_license.txt
 	makensis -DVERSION=$(VERSION) winpkg/ddev.nsi  # brew install makensis, apt-get install nsis, or install on Windows
 
-bin/windows/windows_amd64/sudo.exe bin/windows/windows_amd64/sudo_license.txt:
+$(GOTMP)/bin/windows_amd64/sudo.exe $(GOTMP)/bin/windows_amd64/sudo_license.txt:
 	curl -sSL -o /tmp/sudo.zip -O  https://github.com/mattn/sudo/releases/download/$(WINDOWS_SUDO_VERSION)/sudo-x86_64.zip
-	unzip -o -d $(PWD)/bin/windows/windows_amd64 /tmp/sudo.zip
-	curl -sSL -o $(PWD)/bin/windows/windows_amd64/sudo_license.txt https://raw.githubusercontent.com/mattn/sudo/master/LICENSE
+	unzip -o -d $(GOTMP)/bin/windows_amd64 /tmp/sudo.zip
+	curl -sSL -o $(GOTMP)/bin/windows_amd64/sudo_license.txt https://raw.githubusercontent.com/mattn/sudo/master/LICENSE

build-tools/README.md

@@ -40,10 +40,9 @@ make linux
 make darwin
 make gofmt
 make govet
-make govendor
 make golint
 make codecoroner
-make static (gofmt, govet, golint, govendor)
+make static (gofmt, govet, golint)
 make test
 make container
 make push

build-tools/makefile_components/base_build_go.mak

@@ -4,8 +4,21 @@
 ##### contents into ../Makefile and commenting out the include and adding a
 ##### comment about what you did and why.
 
-
-.PHONY: all build test push clean container-clean bin-clean version static govendor gofmt govet golint golangci-lint container
+DOCKERBUILDCMD=docker run -t --rm -u $(shell id -u):$(shell id -g)                    \
+          	    -v "$(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)"                                \
+          	    -v "$(S)$$PWD:/workdir$(DOCKERMOUNTFLAG)"                              \
+          	    -e CGO_ENABLED=0                  \
+          	    -e GOOS=$@						  \
+          	    -w $(S)/workdir              \
+          	    $(BUILD_IMAGE)
+
+DOCKERTESTCMD=docker run -t --rm -u $(shell id -u):$(shell id -g)                    \
+          	    -v "$(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)"                                \
+          	    -v "$(S)$$PWD:/workdir$(DOCKERMOUNTFLAG)"                              \
+          	    -w $(S)/workdir              \
+          	    $(BUILD_IMAGE)
+
+.PHONY: all build test push clean container-clean bin-clean version static gofmt govet golint golangci-lint container
 GOTMP=.gotmp
 
 SHELL = /bin/bash
@@ -14,7 +27,7 @@ GOFILES = $(shell find $(SRC_DIRS) -name "*.go")
 
 BUILD_OS = $(shell go env GOHOSTOS)
 
-BUILD_IMAGE ?= drud/golang-build-container:v1.11
+BUILD_IMAGE ?= drud/golang-build-container:v1.11.4.2
 
 BUILD_BASE_DIR ?= $$PWD
 
@@ -35,6 +48,11 @@ VERSION_LDFLAGS := $(foreach v,$(VERSION_VARIABLES),-X "$(PKG)/pkg/version.$(v)=
 LDFLAGS := -extldflags -static $(VERSION_LDFLAGS)
 DOCKERMOUNTFLAG := :delegated
 
+# In go 1.11 -mod=vendor is not autodetected; it probably will be in 1.12
+# See https://github.com/golang/go/issues/27227
+USEMODVENDOR := $(shell if [ -d vendor ]; then echo "-mod=vendor"; fi)
+
+
 PWD=$(shell pwd)
 S =
 ifeq ($(BUILD_OS),windows)
@@ -47,143 +65,70 @@ build: $(BUILD_OS)
 
 linux darwin windows: $(GOFILES)
 	@echo "building $@ from $(SRC_AND_UNDER)"
-	@$(shell rm -f VERSION.txt)
-	@$(shell mkdir -p bin/$@ $(GOTMP)/{std/$@,bin,src/$(PKG)})
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                    \
-	    -v "$(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)"                                \
-	    -v "$(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)"                              \
-	    -v "$(S)$$PWD/bin/$@:/go/bin$(DOCKERMOUNTFLAG)"                        \
-	    -v "$(S)$$PWD/bin/$@:/go/bin/$@$(DOCKERMOUNTFLAG)"             \
-	    -v "$(S)$$PWD/$(GOTMP)/std/$@:/usr/local/go/pkg/$@_amd64_static$(DOCKERMOUNTFLAG)"  \
-	    -e CGO_ENABLED=0                  \
-	    -e GOOS=$@						  \
-	    -w $(S)/go/src/$(PKG)                 \
-	    $(BUILD_IMAGE)                    \
-        go install -installsuffix static -ldflags ' $(LDFLAGS) ' $(SRC_AND_UNDER)
+	@mkdir -p $(GOTMP)/{.cache,pkg,src,bin}
+	@$(DOCKERBUILDCMD) \
+        go install $(USEMODVENDOR) -installsuffix static -ldflags ' $(LDFLAGS) ' $(SRC_AND_UNDER)
 	@$(shell touch $@)
+	$( shell if [ -d $(GOTMP) ]; then chmod -R u+w $(GOTMP); fi )
 	@echo $(VERSION) >VERSION.txt
 
-govendor:
-	@echo -n "Using govendor to check for missing dependencies and unused dependencies: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                    \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                   \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                  \
-		-w $(S)/go/src/$(PKG)                                         \
-		$(BUILD_IMAGE)                                                     \
-		bash -c 'OUT=$$(govendor list +missing +unused); if [ -n "$$OUT" ]; then echo "$$OUT"; exit 1; fi'
-
 gofmt:
 	@echo "Checking gofmt: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                    \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                     \
+	@$(DOCKERTESTCMD) \
 		bash -c 'export OUT=$$(gofmt -l $(SRC_DIRS))  && if [ -n "$$OUT" ]; then echo "These files need gofmt -w: $$OUT"; exit 1; fi'
 
 govet:
 	@echo "Checking go vet: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                         \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $S/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                     \
+	@$(DOCKERTESTCMD) \
 		bash -c 'go vet $(SRC_AND_UNDER)'
 
 golint:
 	@echo "Checking golint: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                   \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                     \
+	@$(DOCKERTESTCMD) \
 		bash -c 'export OUT=$$(golint $(SRC_AND_UNDER)) && if [ -n "$$OUT" ]; then echo "Golint problems discovered: $$OUT"; exit 1; fi'
 
 errcheck:
 	@echo "Checking errcheck: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                   \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                     \
+	@$(DOCKERTESTCMD) \
 		errcheck $(SRC_AND_UNDER)
 
 staticcheck:
 	@echo "Checking staticcheck: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                         \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                     \
+	@$(DOCKERTESTCMD) \
 		staticcheck $(SRC_AND_UNDER)
 
-unused:
-	@echo "Checking unused variables and functions: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                         \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                     \
-		unused $(SRC_AND_UNDER)
-
-codecoroner:
-	@echo "Checking codecoroner for unused functions: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                         \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE) \
-		bash -c 'OUT=$$(codecoroner -tests -ignore vendor funcs $(SRC_AND_UNDER)); if [ -n "$$OUT" ]; then echo "$$OUT"; exit 1; fi'                                             \
-
-
 varcheck:
 	@echo "Checking unused globals and struct members: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                         \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                     \
-		varcheck $(SRC_AND_UNDER) && structcheck $(SRC_AND_UNDER)
+	@$(DOCKERTESTCMD) \
+		bash -c "varcheck $(SRC_AND_UNDER) && structcheck $(SRC_AND_UNDER)"
 
 misspell:
 	@echo "Checking for misspellings: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                         \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                     \
+	@$(DOCKERTESTCMD) \
 		misspell $(SRC_DIRS)
 
 gometalinter:
 	@echo "gometalinter: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                         \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                     \
+	@$(DOCKERTESTCMD) \
 		time gometalinter $(GOMETALINTER_ARGS) $(SRC_AND_UNDER)
 
 golangci-lint:
 	@echo "golangci-lint: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                       \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                            \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                         \
+	@$(DOCKERTESTCMD) \
 		time bash -c "golangci-lint run $(GOLANGCI_LINT_ARGS) $(SRC_AND_UNDER)"
 
 version:
 	@echo VERSION:$(VERSION)
 
 clean: container-clean bin-clean
-	@go clean -cache || echo "You're not running latest golang locally" # Make sure the local go cache is clean for testing
 
 container-clean:
 	@if docker image inspect $(DOCKER_REPO):$(VERSION) >/dev/null 2>&1; then docker rmi -f $(DOCKER_REPO):$(VERSION); fi
 	@rm -rf .container-* .dockerfile* .push-* linux darwin windows container VERSION.txt .docker_image
 
 bin-clean:
-	$(shell rm -rf $(GOTMP) bin .tmp)
+	@rm -rf bin
+	$(shell if [ -d $(GOTMP) ]; then chmod -R u+w $(GOTMP) && rm -rf $(GOTMP); fi )
 
 # print-ANYVAR prints the expanded variable
 print-%: ; @echo $* = $($*)

build-tools/makefile_components/base_test_go.mak

@@ -7,9 +7,8 @@
 TESTOS = $(BUILD_OS)
 
 test: build
-	@mkdir -p bin/linux
-	@mkdir -p $(GOTMP)/{src/$(PKG),pkg,bin,std/linux}
 	@echo "Testing $(SRC_AND_UNDER) with TESTARGS=$(TESTARGS)"
+	@mkdir -p $(GOTMP)/{.cache,pkg,src,bin}
 	@docker run -t --rm  -u $(shell id -u):$(shell id -g)                 \
 	    -v $(PWD)/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
 	    -v $(PWD):/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
@@ -18,7 +17,8 @@ test: build
 	    -e CGO_ENABLED=0	\
 	    -w /go/src/$(PKG)                                                  \
 	    $(BUILD_IMAGE)                                                     \
-        go test -v -installsuffix static -ldflags '$(LDFLAGS)' $(SRC_AND_UNDER) $(TESTARGS)
+        go test $(USEMODVENDOR) -v -installsuffix static -ldflags '$(LDFLAGS)' $(SRC_AND_UNDER) $(TESTARGS)
+	@$(shell chmod -R u+w $(GOTMP))
 
 # test_precompile allows a full compilation of _test.go files, without execution of the tests.
 # Setup and teardown in TestMain is still executed though, so this can cost some time.

cmd/ddev_gen_autocomplete/ddev_gen_autocomplete.go

@@ -8,7 +8,7 @@ import (
 	"github.com/drud/ddev/pkg/util"
 )
 
-var targetDir = "bin"
+var targetDir = ".gotmp/bin"
 
 func main() {
 	if _, err := os.Stat(targetDir); os.IsNotExist(err) {

go.mod

@@ -39,7 +39,7 @@ require (
 	github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a
 	github.com/magiconair/properties v1.8.0 // indirect
 	github.com/mattn/go-colorable v0.0.9 // indirect
-	github.com/mattn/go-isatty v0.0.3 // indirect
+	github.com/mattn/go-isatty v0.0.3
 	github.com/mattn/go-runewidth v0.0.2 // indirect
 	github.com/mattn/go-shellwords v1.0.3
 	github.com/mitchellh/go-homedir v0.0.0-20180523094522-3864e76763d9