Makefile changes to allow easy builds with or without vendoring. Also fixes version bug for both cases.

.circleci/config.yml

@@ -47,7 +47,7 @@ workflows:
 # https://circleci.com/blog/circleci-hacks-reuse-yaml-in-your-circleci-config-with-yaml/
 .defaults: &defaults
   docker:
-    - image: grafana/loki-build-image:0.7.4
+    - image: grafana/loki-build-image:0.8.0
   working_directory: /src/loki
 
 jobs:

.drone/drone.yml

@@ -12,28 +12,28 @@ workspace:
 
 steps:
 - name: test
-  image: grafana/loki-build-image:0.7.4
+  image: grafana/loki-build-image:0.8.0
   commands:
   - make BUILD_IN_CONTAINER=false test
   depends_on:
   - clone
 
 - name: lint
-  image: grafana/loki-build-image:0.7.4
+  image: grafana/loki-build-image:0.8.0
   commands:
   - make BUILD_IN_CONTAINER=false lint
   depends_on:
   - clone
 
 - name: check-generated-files
-  image: grafana/loki-build-image:0.7.4
+  image: grafana/loki-build-image:0.8.0
   commands:
   - make BUILD_IN_CONTAINER=false check-generated-files
   depends_on:
   - clone
 
 - name: check-mod
-  image: grafana/loki-build-image:0.7.4
+  image: grafana/loki-build-image:0.8.0
   commands:
   - make BUILD_IN_CONTAINER=false check-mod
   depends_on:
@@ -526,7 +526,7 @@ platform:
 
 steps:
 - name: trigger
-  image: grafana/loki-build-image:0.7.4
+  image: grafana/loki-build-image:0.8.0
   commands:
   - ./tools/deploy.sh
   environment:

.golangci.yml

@@ -3,9 +3,6 @@
 
 # options for analysis running
 run:
-  # trust vendored dependencies
-  modules-download-mode: vendor
-
   # default concurrency is a available CPU number
   concurrency: 16
 

Makefile

@@ -8,6 +8,24 @@
 .PHONY: benchmark-store, drone, check-mod
 
 SHELL = /usr/bin/env bash
+
+# Empty value = no -mod parameter is used.
+# If not empty, GOMOD is passed to -mod= parameter.
+# In Go 1.13, "readonly" and "vendor" are accepted.
+# In Go 1.14, "readonly", "vendor" and "mod" values are accepted.
+# If no value is specified, defaults to "vendor".
+#
+# Can be used from command line by using "GOMOD= make" (empty = no -mod parameter), or "GOMOD=vendor make" (default).
+
+GOMOD?=vendor
+ifeq ($(strip $(GOMOD)),) # Is empty?
+	MOD_FLAG=
+	GOLANGCI_ARG=
+else
+	MOD_FLAG=-mod=$(GOMOD)
+	GOLANGCI_ARG=--modules-download-mode=$(GOMOD)
+endif
+
 #############
 # Variables #
 #############
@@ -20,7 +38,7 @@ IMAGE_NAMES := $(foreach dir,$(DOCKER_IMAGE_DIRS),$(patsubst %,$(IMAGE_PREFIX)%,
 # make BUILD_IN_CONTAINER=false target
 # or you can override this with an environment variable
 BUILD_IN_CONTAINER ?= true
-BUILD_IMAGE_VERSION := 0.7.4
+BUILD_IMAGE_VERSION := 0.8.0
 
 # Docker image info
 IMAGE_PREFIX ?= grafana
@@ -40,15 +58,15 @@ DONT_FIND := -name tools -prune -o -name vendor -prune -o -name .git -prune -o -
 APP_GO_FILES := $(shell find . $(DONT_FIND) -name .y.go -prune -o -name .pb.go -prune -o -name cmd -prune -o -type f -name '*.go' -print)
 
 # Build flags
-VPREFIX := github.com/grafana/loki/vendor/github.com/prometheus/common/version
+VPREFIX := github.com/grafana/loki/pkg/build
 GO_LDFLAGS   := -s -w -X $(VPREFIX).Branch=$(GIT_BRANCH) -X $(VPREFIX).Version=$(IMAGE_TAG) -X $(VPREFIX).Revision=$(GIT_REVISION) -X $(VPREFIX).BuildUser=$(shell whoami)@$(shell hostname) -X $(VPREFIX).BuildDate=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
-GO_FLAGS     := -ldflags "-extldflags \"-static\" $(GO_LDFLAGS)" -tags netgo -mod vendor
-DYN_GO_FLAGS := -ldflags "$(GO_LDFLAGS)" -tags netgo -mod vendor
+GO_FLAGS     := -ldflags "-extldflags \"-static\" $(GO_LDFLAGS)" -tags netgo $(MOD_FLAG)
+DYN_GO_FLAGS := -ldflags "$(GO_LDFLAGS)" -tags netgo $(MOD_FLAG)
 # Per some websites I've seen to add `-gcflags "all=-N -l"`, the gcflags seem poorly if at all documented
 # the best I could dig up is -N disables optimizations and -l disables inlining which should make debugging match source better.
 # Also remove the -s and -w flags present in the normal build which strip the symbol table and the DWARF symbol table.
-DEBUG_GO_FLAGS     := -gcflags "all=-N -l" -ldflags "-extldflags \"-static\" $(GO_LDFLAGS)" -tags netgo -mod vendor
-DYN_DEBUG_GO_FLAGS := -gcflags "all=-N -l" -ldflags "$(GO_LDFLAGS)" -tags netgo -mod vendor
+DEBUG_GO_FLAGS     := -gcflags "all=-N -l" -ldflags "-extldflags \"-static\" $(GO_LDFLAGS)" -tags netgo $(MOD_FLAG)
+DYN_DEBUG_GO_FLAGS := -gcflags "all=-N -l" -ldflags "$(GO_LDFLAGS)" -tags netgo $(MOD_FLAG)
 
 NETGO_CHECK = @strings $@ | grep cgo_stub\\\.go >/dev/null || { \
        rm $@; \
@@ -186,7 +204,7 @@ promtail-clean-assets:
 # Rule to generate promtail static assets file
 $(PROMTAIL_GENERATED_FILE): $(PROMTAIL_UI_FILES)
 	@echo ">> writing assets"
-	GOFLAGS=-mod=vendor GOOS=$(shell go env GOHOSTOS) go generate -x -v ./pkg/promtail/server/ui
+	GOFLAGS="$(MOD_FLAG)" GOOS=$(shell go env GOHOSTOS) go generate -x -v ./pkg/promtail/server/ui
 
 cmd/promtail/promtail: $(APP_GO_FILES) $(PROMTAIL_GENERATED_FILE) cmd/promtail/main.go
 	CGO_ENABLED=$(PROMTAIL_CGO) go build $(PROMTAIL_GO_FLAGS) -o $@ ./$(@D)
@@ -219,14 +237,14 @@ publish: dist
 ########
 
 lint:
-	GO111MODULE=on GOGC=10 golangci-lint run -v
+	GO111MODULE=on GOGC=10 golangci-lint run -v $(GOLANGCI_ARG)
 
 ########
 # Test #
 ########
 
 test: all
-	GOGC=10 go test -mod=vendor -p=4 ./...
+	GOGC=10 go test $(MOD_FLAG) -p=4 ./...
 
 #########
 # Clean #
@@ -242,7 +260,7 @@ clean:
 	rm -rf dist/
 	rm -rf cmd/fluent-bit/out_loki.h
 	rm -rf cmd/fluent-bit/out_loki.so
-	go clean -mod=vendor ./...
+	go clean $(MOD_FLAG) ./...
 
 #########
 # YACCs #
@@ -482,8 +500,8 @@ build-image-push: build-image
 ########
 
 benchmark-store:
-	go run -mod=vendor ./pkg/storage/hack/main.go
-	go test -mod=vendor ./pkg/storage/ -bench=.  -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out
+	go run $(MOD_FLAG) ./pkg/storage/hack/main.go
+	go test $(MOD_FLAG) ./pkg/storage/ -bench=.  -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out
 
 # regenerate drone yaml
 drone:

cmd/docker-driver/Dockerfile

@@ -1,4 +1,4 @@
-ARG BUILD_IMAGE=grafana/loki-build-image:0.7.4
+ARG BUILD_IMAGE=grafana/loki-build-image:0.8.0
 # Directories in this file are referenced from the root of the project not this folder
 # This file is intented to be called from the root like so:
 # docker build -t grafana/loki -f cmd/loki/Dockerfile .

cmd/docker-driver/main.go

@@ -8,6 +8,7 @@ import (
 	"github.com/docker/go-plugins-helpers/sdk"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
+	_ "github.com/grafana/loki/pkg/build"
 	"github.com/prometheus/common/version"
 	"github.com/weaveworks/common/logging"
 )

cmd/fluent-bit/out_loki.go

@@ -2,16 +2,17 @@ package main
 
 import (
 	"C"
+	"fmt"
 	"time"
 	"unsafe"
 
 	"github.com/fluent/fluent-bit-go/output"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
+	_ "github.com/grafana/loki/pkg/build"
 	"github.com/prometheus/common/version"
 	"github.com/weaveworks/common/logging"
 )
-import "fmt"
 
 var plugin *loki
 var logger log.Logger

cmd/logcli/main.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"time"
 
+	_ "github.com/grafana/loki/pkg/build"
 	"github.com/grafana/loki/pkg/logcli/client"
 	"github.com/grafana/loki/pkg/logcli/labelquery"
 	"github.com/grafana/loki/pkg/logcli/output"

cmd/loki-canary/Dockerfile.cross

@@ -1,4 +1,4 @@
-ARG BUILD_IMAGE=grafana/loki-build-image:0.7.4
+ARG BUILD_IMAGE=grafana/loki-build-image:0.8.0
 # Directories in this file are referenced from the root of the project not this folder
 # This file is intented to be called from the root like so:
 # docker build -t grafana/promtail -f cmd/promtail/Dockerfile .

cmd/loki-canary/main.go

@@ -13,6 +13,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/prometheus/common/version"
 
+	_ "github.com/grafana/loki/pkg/build"
 	"github.com/grafana/loki/pkg/canary/comparator"
 	"github.com/grafana/loki/pkg/canary/reader"
 	"github.com/grafana/loki/pkg/canary/writer"
@@ -39,7 +40,7 @@ func main() {
 	flag.Parse()
 
 	if *printVersion {
-		fmt.Print(version.Print("loki-canary"))
+		fmt.Println(version.Print("loki-canary"))
 		os.Exit(0)
 	}
 

cmd/loki/Dockerfile.cross

@@ -1,4 +1,4 @@
-ARG BUILD_IMAGE=grafana/loki-build-image:0.7.4
+ARG BUILD_IMAGE=grafana/loki-build-image:0.8.0
 # Directories in this file are referenced from the root of the project not this folder
 # This file is intented to be called from the root like so:
 # docker build -t grafana/loki -f cmd/loki/Dockerfile .

cmd/loki/main.go

@@ -7,6 +7,7 @@ import (
 	"reflect"
 
 	"github.com/go-kit/kit/log/level"
+	_ "github.com/grafana/loki/pkg/build"
 	"github.com/grafana/loki/pkg/cfg"
 	"github.com/grafana/loki/pkg/loki"
 	"github.com/prometheus/client_golang/prometheus"
@@ -31,7 +32,7 @@ func main() {
 		os.Exit(1)
 	}
 	if *printVersion {
-		fmt.Print(version.Print("loki"))
+		fmt.Println(version.Print("loki"))
 		os.Exit(0)
 	}
 

cmd/promtail/Dockerfile.cross

@@ -1,4 +1,4 @@
-ARG BUILD_IMAGE=grafana/loki-build-image:0.7.4
+ARG BUILD_IMAGE=grafana/loki-build-image:0.8.0
 # Directories in this file are referenced from the root of the project not this folder
 # This file is intented to be called from the root like so:
 # docker build -t grafana/promtail -f cmd/promtail/Dockerfile .

cmd/promtail/main.go

@@ -14,6 +14,7 @@ import (
 	"github.com/prometheus/common/version"
 	"github.com/weaveworks/common/logging"
 
+	_ "github.com/grafana/loki/pkg/build"
 	"github.com/grafana/loki/pkg/cfg"
 	"github.com/grafana/loki/pkg/logentry/stages"
 	"github.com/grafana/loki/pkg/promtail"

loki-build-image/Dockerfile

@@ -9,12 +9,12 @@ RUN apk add --no-cache curl && \
 FROM alpine as golangci
 RUN apk add --no-cache curl && \
     cd / && \
-    curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.20.0
+    curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.21.0
 
 FROM alpine:edge as docker
 RUN apk add --no-cache docker-cli
 
-FROM golang:1.13.1-stretch
+FROM golang:1.13.4-stretch
 RUN apt-get update && \
     apt-get install -qy \
       musl \

pkg/build/build.go

@@ -0,0 +1,23 @@
+package build
+
+import "github.com/prometheus/common/version"
+
+// Version information passed to Prometheus version package.
+// Package path as used by linker changes based on vendoring being used or not,
+// so it's easier just to use stable Loki path, and pass it to
+// Prometheus in the code.
+var (
+	Version   string
+	Revision  string
+	Branch    string
+	BuildUser string
+	BuildDate string
+)
+
+func init() {
+	version.Version = Version
+	version.Revision = Revision
+	version.Branch = Branch
+	version.BuildUser = BuildUser
+	version.BuildDate = BuildDate
+}

pkg/promtail/server/ui/doc.go

@@ -7,4 +7,4 @@ import (
 	_ "github.com/shurcooL/vfsgen"
 )
 
-//go:generate go run -tags=dev assets_generate.go -build_flags=-mod=vendor
+//go:generate go run -tags=dev assets_generate.go -build_flags="$GOFLAGS"