use docker to build and improve test for objstore #150

.travis.yml

@@ -1,23 +1,10 @@
 language: go
 
-before_install:
-  - go get github.com/mattn/goveralls
-  - go get golang.org/x/tools/cmd/cover
-
 services:
-- postgresql
-- redis-server
-
-env:
-- GO111MODULE=on REDIS_HOSTS=localhost:6379 POSTGRES_HOST=localhost POSTGRES_PASSWORD=
+- docker
 
 go:
 - 1.13.x
 
-after_success:
-- GO111MODULE=off
-
 script:
-  - make ci
-  - make coveralls-test
-  - $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci
+- docker-compose run testserver make ci

Makefile

@@ -1,6 +1,6 @@
 # Copyright © 2018 by PACE Telematics GmbH. All rights reserved.
 # Created at 2018/08/24 by Vincent Landgraf
-.PHONY: install test jsonapi build integration ci
+.PHONY: install test jsonapi build integration ci ci-test
 
 JSONAPITEST=http/jsonapi/generator/internal
 JSONAPIGEN="./tools/jsonapigen/main.go"
@@ -43,8 +43,10 @@ integration:
 testserver:
 	docker-compose up
 
-coveralls-test:
+ci-test:
 	go test -mod=vendor -count=1 -v -cover -covermode=count -coverprofile=coverage.out -short ./...
 
-
-ci: test integration
+ci: ci-test
+	go get github.com/mattn/goveralls
+	go get golang.org/x/tools/cmd/cover
+	$(GOPATH)/bin/goveralls -coverprofile=coverage.out -service=travis-ci

backend/objstore/health_objstore.go

@@ -2,11 +2,13 @@ package objstore
 
 import (
 	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
 	"time"
 
 	"github.com/minio/minio-go/v6"
 	"github.com/pace/bricks/maintenance/health/servicehealthcheck"
-	"golang.org/x/xerrors"
 )
 
 // HealthCheck checks the state of the object storage client. It must not be changed
@@ -19,7 +21,7 @@ type HealthCheck struct {
 // HealthCheck checks if the object storage client is healthy. If the last result is outdated,
 // object storage is checked for upload and download,
 // otherwise returns the old result
-func (h *HealthCheck) HealthCheck() servicehealthcheck.HealthCheckResult {
+func (h *HealthCheck) HealthCheck(ctx context.Context) servicehealthcheck.HealthCheckResult {
 	if time.Since(h.state.LastChecked()) <= cfg.HealthCheckResultTTL {
 		// the last health check is not outdated, an can be reused.
 		return h.state.GetState()
@@ -28,8 +30,8 @@ func (h *HealthCheck) HealthCheck() servicehealthcheck.HealthCheckResult {
 	expContent := []byte(time.Now().Format(time.RFC3339))
 	expSize := int64(len(expContent))
 
-	// Try upload
-	_, err := h.Client.PutObject(
+	_, err := h.Client.PutObjectWithContext(
+		ctx,
 		cfg.HealthCheckBucketName,
 		cfg.HealthCheckObjectName,
 		bytes.NewReader(expContent),
@@ -39,32 +41,32 @@ func (h *HealthCheck) HealthCheck() servicehealthcheck.HealthCheckResult {
 		},
 	)
 	if err != nil {
-		h.state.SetErrorState(err)
+		h.state.SetErrorState(fmt.Errorf("failed to put object: %v", err))
 		return h.state.GetState()
 	}
 
 	// Try download
-	obj, err := h.Client.GetObject(
+	obj, err := h.Client.GetObjectWithContext(
+		ctx,
 		cfg.HealthCheckBucketName,
 		cfg.HealthCheckObjectName,
 		minio.GetObjectOptions{},
 	)
 	if err != nil {
-		h.state.SetErrorState(err)
+		h.state.SetErrorState(fmt.Errorf("failed to get object: %v", err))
 		return h.state.GetState()
 	}
+	defer obj.Close()
 
 	// Assert expectations
-	gotContent := make([]byte, expSize)
-	_, err = obj.Read(gotContent)
+	buf, err := ioutil.ReadAll(obj)
 	if err != nil {
-		h.state.SetErrorState(err)
+		h.state.SetErrorState(fmt.Errorf("failed to compare object: %v", err))
 		return h.state.GetState()
 	}
-	defer obj.Close()
 
-	if bytes.Compare(gotContent, expContent) == 0 {
-		h.state.SetErrorState(xerrors.New("objstore: unexpected health check caused by unexpected object content"))
+	if bytes.Compare(buf, expContent) != 0 {
+		h.state.SetErrorState(fmt.Errorf("unexpected content: %q <-> %q", string(buf), string(expContent)))
 		return h.state.GetState()
 	}
 

backend/objstore/health_objstore_test.go

@@ -27,15 +27,15 @@ func TestIntegrationHealthCheck(t *testing.T) {
 		t.SkipNow()
 	}
 	resp := setup()
-	if resp.StatusCode != 503 {
-		t.Errorf("Expected /health/check to respond with 503, got: %d", resp.StatusCode)
+	if resp.StatusCode != 200 {
+		t.Errorf("Expected /health/check to respond with 200, got: %d", resp.StatusCode)
 	}
 
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		log.Fatal(err)
 	}
-	if !strings.Contains(string(data), "objstore               ERR") {
+	if !strings.Contains(string(data), "objstore               OK") {
 		t.Errorf("Expected /health/check to return OK, got: %q", string(data[:]))
 	}
 }

backend/objstore/objstore.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/caarlos0/env"
 	"github.com/minio/minio-go/v6"
+	"github.com/minio/minio-go/v6/pkg/credentials"
 	"github.com/pace/bricks/http/transport"
 	"github.com/pace/bricks/maintenance/health/servicehealthcheck"
 	"github.com/pace/bricks/maintenance/log"
@@ -15,6 +16,7 @@ import (
 type config struct {
 	Endpoint        string `env:"S3_ENDPOINT" envDefault:"s3.amazonaws.com"`
 	AccessKeyID     string `env:"S3_ACCESS_KEY_ID"`
+	Region          string `env:"S3_REGION" envDefault:"us-east-1"`
 	SecretAccessKey string `env:"S3_SECRET_ACCESS_KEY"`
 	UseSSL          bool   `env:"S3_USE_SSL"`
 
@@ -43,16 +45,27 @@ func init() {
 	servicehealthcheck.RegisterHealthCheck(&HealthCheck{
 		Client: client,
 	}, "objstore")
+
+	ok, err := client.BucketExists(cfg.HealthCheckBucketName)
+	if err != nil {
+		log.Warnf("Failed to create check for bucket: %v", err)
+	}
+	if !ok {
+		err := client.MakeBucket(cfg.HealthCheckBucketName, cfg.Region)
+		if err != nil {
+			log.Warnf("Failed to create bucket: %v", err)
+		}
+	}
 }
 
 // Client with environment based configuration
 func Client() (*minio.Client, error) {
-	client, err := minio.New(cfg.Endpoint, cfg.AccessKeyID, cfg.SecretAccessKey, cfg.UseSSL)
-	if err != nil {
-		return nil, err
-	}
-	client.SetCustomTransport(newCustomTransport(cfg.Endpoint))
-	return client, nil
+	return CustomClient(cfg.Endpoint, &minio.Options{
+		Secure:       cfg.UseSSL,
+		Region:       cfg.Region,
+		BucketLookup: minio.BucketLookupAuto,
+		Creds:        credentials.NewStaticV4(cfg.AccessKeyID, cfg.SecretAccessKey, ""),
+	})
 }
 
 // CustomClient with customized client
@@ -61,6 +74,10 @@ func CustomClient(endpoint string, opts *minio.Options) (*minio.Client, error) {
 	if err != nil {
 		return nil, err
 	}
+	log.Logger().Info().Str("endpoint", endpoint).
+		Str("region", opts.Region).
+		Bool("ssl", opts.Secure).
+		Msg("S3 connection created")
 	client.SetCustomTransport(newCustomTransport(endpoint))
 	return client, nil
 }

backend/postgres/health_postgres.go

@@ -4,6 +4,7 @@
 package postgres
 
 import (
+	"context"
 	"time"
 
 	"github.com/go-pg/pg/orm"
@@ -29,7 +30,7 @@ func (h *HealthCheck) Init() error {
 
 // HealthCheck performs the read test on the database. If enabled, it performs a
 // write test as well.
-func (h *HealthCheck) HealthCheck() servicehealthcheck.HealthCheckResult {
+func (h *HealthCheck) HealthCheck(ctx context.Context) servicehealthcheck.HealthCheckResult {
 	if time.Since(h.state.LastChecked()) <= cfg.HealthCheckResultTTL {
 		// the last result of the Health Check is still not outdated
 		return h.state.GetState()

backend/postgres/health_postgres_test.go

@@ -4,6 +4,7 @@
 package postgres
 
 import (
+	"context"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -56,23 +57,25 @@ func (t *testPool) Exec(query interface{}, params ...interface{}) (res orm.Resul
 }
 
 func TestHealthCheckCaching(t *testing.T) {
+	ctx := context.Background()
+
 	// set the TTL to a minute because this is long enough to test that the result is cached
 	cfg.HealthCheckResultTTL = time.Minute
 	requiredErr := errors.New("TestHealthCheckCaching")
 	pool := &testPool{err: requiredErr}
 	h := &HealthCheck{Pool: pool}
-	res := h.HealthCheck()
+	res := h.HealthCheck(ctx)
 	// get the error for the first time
 	require.Equal(t, servicehealthcheck.Err, res.State)
 	require.Equal(t, "TestHealthCheckCaching", res.Msg)
-	res = h.HealthCheck()
+	res = h.HealthCheck(ctx)
 	pool.err = nil
 	// getting the cached error
 	require.Equal(t, servicehealthcheck.Err, res.State)
 	require.Equal(t, "TestHealthCheckCaching", res.Msg)
 	// Resetting the TTL to get a uncached result
 	cfg.HealthCheckResultTTL = 0
-	res = h.HealthCheck()
+	res = h.HealthCheck(ctx)
 	require.Equal(t, servicehealthcheck.Ok, res.State)
 	require.Equal(t, "", res.Msg)
 }

backend/redis/health_redis.go

@@ -4,6 +4,7 @@
 package redis
 
 import (
+	"context"
 	"time"
 
 	"github.com/go-redis/redis"
@@ -20,7 +21,7 @@ type HealthCheck struct {
 // HealthCheck checks if the redis is healthy. If the last result is outdated,
 // redis is checked for writeability and readability,
 // otherwise return the old result
-func (h *HealthCheck) HealthCheck() servicehealthcheck.HealthCheckResult {
+func (h *HealthCheck) HealthCheck(ctx context.Context) servicehealthcheck.HealthCheckResult {
 	if time.Since(h.state.LastChecked()) <= cfg.HealthCheckResultTTL {
 		// the last health check is not outdated, an can be reused.
 		return h.state.GetState()

docker-compose.yml

@@ -15,6 +15,17 @@ services:
     ports:
       - "6379:6379"
 
+  minio:
+    image: minio/minio
+    ports:
+      # HTTP UI
+      - "9000:9000"
+    command: server /data
+    environment:
+      - MINIO_ACCESS_KEY=client
+      - MINIO_SECRET_KEY=secret01
+      - MINIO_REGION_NAME=us-east-1
+
   jaeger:
     image: jaegertracing/all-in-one:latest
     ports:
@@ -49,10 +60,15 @@ services:
       - SENTRY_RELEASE=`git rev-parse --short HEAD`
       - PACE_LIVETEST_INTERVAL=10s
       - LOG_FORMAT=console
+      - S3_ENDPOINT=minio:9000
+      - S3_ACCESS_KEY_ID=client
+      - S3_SECRET_ACCESS_KEY=secret01
+      - S3_USE_SSL=false
     command: go run -mod=vendor ./tools/testserver
     depends_on:
       - postgres
       - redis
+      - minio
       - jaeger
       - prometheus
     restart: always

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a // indirect
 	github.com/mattn/go-isatty v0.0.8
+	github.com/mattn/goveralls v0.0.5 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/minio/minio-go/v6 v6.0.44
 	github.com/onsi/ginkgo v1.8.0 // indirect
@@ -42,6 +43,7 @@ require (
 	github.com/uber/jaeger-lib v1.5.0
 	github.com/zenazn/goji v0.9.0
 	go.uber.org/atomic v1.3.2 // indirect
+	golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7 // indirect
 	golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898
 )
 

go.sum

@@ -51,6 +51,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/goveralls v0.0.5 h1:spfq8AyZ0cCk57Za6/juJ5btQxeE1FaEGMdfcI+XO48=
+github.com/mattn/goveralls v0.0.5/go.mod h1:Xg2LHi51faXLyKXwsndxiW6uxEEQT9+3sjGzzwU4xy0=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/minio/minio-go/v6 v6.0.44 h1:CVwVXw+uCOcyMi7GvcOhxE8WgV+Xj8Vkf2jItDf/EGI=
@@ -72,6 +74,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.8.0 h1:1921Yw9Gc3iSc4VQh3PIoOqgPCZS7G/4xQNVUp8Mda8=
 github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.4.0 h1:YVIb/fVcOTMSqtqZWSKnHpSLBxu8DKgxq8z6RuBZwqI=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e h1:n/3MEhJQjQxrOUCzh1Y3Re6aJUUWRp2M9+Oc3eVn/54=
@@ -95,7 +98,9 @@ github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc=
 github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -112,6 +117,9 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo=
 golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -135,6 +143,10 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74 h1:4cFkmztxtMslUX2SctSl+blCyXfpzhGOy9LhKAqSMA4=
 golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200113040837-eac381796e91 h1:OOkytthzFBKHY5EfEgLUabprb0LtJVkQtNxAQ02+UE4=
+golang.org/x/tools v0.0.0-20200113040837-eac381796e91/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7 h1:EBZoQjiKKPaLbPrbpssUfuHtwM6KV/vb4U85g/cigFY=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

maintenance/health/health.go

@@ -9,6 +9,8 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/rs/zerolog"
+
 	"github.com/pace/bricks/maintenance/log"
 )
 
@@ -19,7 +21,9 @@ type handler struct {
 var readinessCheck = &handler{check: liveness}
 
 func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	h.check(w, r)
+	logger := log.Logger()
+	logger.Level(zerolog.DebugLevel)
+	h.check(w, r.WithContext(logger.WithContext(r.Context())))
 }
 
 // ReadinessCheck allows to set a different function for the readiness check. The default readiness check