Merge eb9a742 into d803866

.github/workflows/go.yml

@@ -8,6 +8,7 @@ on:
 
 jobs:
   build:
+    environment: qa
     runs-on: ubuntu-latest
     services:
       postgres:
@@ -35,6 +36,13 @@ jobs:
         go get ./...
       
     - name: Test
+      env:
+        GONODE_TEST_AWS_VAULT_S3_BUCKET: ${{ secrets.GONODE_TEST_AWS_VAULT_S3_BUCKET }}
+        GONODE_TEST_OFFLINE: ${{ secrets.GONODE_TEST_OFFLINE }}
+        GONODE_TEST_S3_ACCESS_KEY: ${{ secrets.GONODE_TEST_S3_ACCESS_KEY }}
+        GONODE_TEST_S3_ENDPOINT: ${{ secrets.GONODE_TEST_S3_ENDPOINT }}
+        GONODE_TEST_S3_REGION: ${{ secrets.GONODE_TEST_S3_REGION }}
+        GONODE_TEST_S3_SECRET:  ${{ secrets.GONODE_TEST_S3_SECRET }}
       run: |
         make test
 

Makefile

@@ -6,16 +6,11 @@ GONODE_MODULES = $(shell ls -d ./modules/* | grep -v go)
 GONODE_CORE = $(shell ls -d ./core/* | grep -v go)
 GOPATH = $(shell go env GOPATH)
 
-install:
-	$(call back,glide install)
-	$(call back,go get github.com/wadey/gocovmerge && go get golang.org/x/tools/cmd/cover && go get golang.org/x/tools/cmd/goimports && go get -u github.com/jteeuwen/go-bindata/...)
-
 test:
 	./app/assets/bindata.sh
 	mkdir -p data
 	echo "mode: atomic" > data/coverage.out
-
-	GONODE_TEST_OFFLINE=true GOPATH=${GOPATH} go test -v -failfast -covermode=atomic -coverprofile=data/coverage_core.out $(GONODE_CORE)
+	GOPATH=${GOPATH} go test -v -failfast -covermode=atomic -coverprofile=data/coverage_core.out $(GONODE_CORE)
 	GOPATH=${GOPATH} go test -v -failfast -covermode=atomic -coverprofile=data/coverage_modules.out $(GONODE_MODULES)
 	GOPATH=${GOPATH} go test -v -failfast -covermode=atomic -coverpkg ./... -coverprofile=data/coverage_integration.out ./test/modules
 	go vet $(GONODE_CORE) $(GONODE_MODULES) ./test/modules/

core/vault/vault.go

@@ -183,7 +183,7 @@ func (v *Vault) Put(name string, meta VaultMetadata, r io.Reader) (written int64
 		defer w.Close()
 	}
 
-	if _, err = io.Copy(w, buf); err != nil {
+	if written, err = io.Copy(w, buf); err != nil {
 		v.removeIfExists(vaultfile)
 		v.removeIfExists(metafile)
 
@@ -204,8 +204,6 @@ func (v *Vault) Put(name string, meta VaultMetadata, r io.Reader) (written int64
 		v.removeIfExists(vaultfile)
 		v.removeIfExists(metafile)
 		v.removeIfExists(binfile)
-
-		return
 	}
 
 	return

core/vault/vault_driver_s3.go

@@ -30,7 +30,6 @@ func (w *s3Writer) Write(b []byte) (int, error) {
 
 func (w *s3Writer) Close() error {
 	name := w.file.Name()
-
 	defer func() {
 		os.Remove(name)
 	}()

core/vault/vault_driver_s3_test.go

@@ -13,35 +13,24 @@ import (
 	"bytes"
 	"fmt"
 	"os"
-	"syscall"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 )
 
-// this is just a test to validata how the aws sdk behave
-func Test_Vault_Basic_S3_Usage(t *testing.T) {
-
-	if _, offline := syscall.Getenv("GONODE_TEST_OFFLINE"); offline == true {
-		t.Skip("OFFLINE TEST ONLY")
-		return
+func getEnv(name, def string) string {
+	value := os.Getenv(name)
+	if len(value) == 0 {
+		value = def
 	}
 
-	var err error
-	var headResult *s3.HeadObjectOutput
-	var getResult *s3.GetObjectOutput
-
-	root := os.Getenv("GONODE_TEST_AWS_VAULT_ROOT")
-	if len(root) == 0 {
-		root = "local"
-	}
+	return value
+}
 
-	profile := os.Getenv("GONODE_TEST_AWS_PROFILE")
-	if len(profile) == 0 {
-		profile = "gonode-test"
-	}
+func getChainCredentials() (*credentials.Credentials, error) {
+	profile := getEnv("GONODE_TEST_AWS_PROFILE", "gonode-test")
 
 	chainProvider := credentials.NewChainCredentials([]credentials.Provider{
 		&credentials.EnvProvider{},
@@ -53,21 +42,48 @@ func Test_Vault_Basic_S3_Usage(t *testing.T) {
 			Filename: os.Getenv("GONODE_TEST_AWS_CREDENTIALS_FILE"),
 			Profile:  profile,
 		},
+		&credentials.StaticProvider{Value: credentials.Value{
+			AccessKeyID:     getEnv("GONODE_TEST_S3_ACCESS_KEY", ""),
+			SecretAccessKey: getEnv("GONODE_TEST_S3_SECRET", ""),
+		}},
 	})
 
-	_, err = chainProvider.Get()
+	if _, err := chainProvider.Get(); err != nil {
+		return nil, err
+	}
+
+	return chainProvider, nil
+}
+
+func getDriver(chainProvider *credentials.Credentials) *DriverS3 {
+	return &DriverS3{
+		Bucket:      getEnv("GONODE_TEST_AWS_VAULT_S3_BUCKET", "gonode-qa"),
+		Root:        getEnv("GITHUB_RUN_ID", getEnv("GONODE_TEST_AWS_VAULT_ROOT", "local")),
+		Region:      getEnv("GONODE_TEST_S3_REGION", "eu-west-1"),
+		EndPoint:    getEnv("GONODE_TEST_S3_ENDPOINT", "s3-eu-west-1.amazonaws.com"),
+		Credentials: chainProvider,
+	}
+}
+
+// this is just a test to validata how the aws sdk behave
+func Test_Vault_Basic_S3_Usage(t *testing.T) {
+	if getEnv("GONODE_TEST_OFFLINE", "yes") == "yes" {
+		t.Skip("OFFLINE TEST ONLY")
+		return
+	}
+
+	var err error
+	var headResult *s3.HeadObjectOutput
+	var getResult *s3.GetObjectOutput
+
+	chainProvider, err := getChainCredentials()
 
 	if err != nil {
 		t.Skip("Unable to find credentials")
 	}
 
 	// init vault
-	v := &DriverS3{
-		Root:        root,
-		Region:      "eu-west-1",
-		EndPoint:    "s3-eu-west-1.amazonaws.com",
-		Credentials: chainProvider,
-	}
+	v := getDriver(chainProvider)
 
 	// init credentials information
 	config := &aws.Config{
@@ -79,15 +95,10 @@ func Test_Vault_Basic_S3_Usage(t *testing.T) {
 
 	s3client := s3.New(session.New(), config)
 
-	bucketName := os.Getenv("GONODE_TEST_AWS_VAULT_S3_BUCKET")
-	if len(bucketName) == 0 {
-		bucketName = "gonode-test"
-	}
-
 	key := fmt.Sprintf("%s/test/assd", v.Root)
 
 	headResult, err = s3client.HeadObject(&s3.HeadObjectInput{
-		Bucket: aws.String(bucketName),
+		Bucket: aws.String(v.Bucket),
 		Key:    aws.String("no-file"),
 	})
 
@@ -97,7 +108,7 @@ func Test_Vault_Basic_S3_Usage(t *testing.T) {
 	data := []byte("foobar et foo")
 
 	putObject := &s3.PutObjectInput{
-		Bucket:      aws.String(bucketName),
+		Bucket:      aws.String(v.Bucket),
 		Key:         aws.String(key),
 		Body:        bytes.NewReader(data),
 		ContentType: aws.String("application/octet-stream"),
@@ -106,15 +117,15 @@ func Test_Vault_Basic_S3_Usage(t *testing.T) {
 	_, err = s3client.PutObject(putObject)
 
 	headResult, err = s3client.HeadObject(&s3.HeadObjectInput{
-		Bucket: aws.String(bucketName),
+		Bucket: aws.String(v.Bucket),
 		Key:    aws.String(key),
 	})
 
 	assert.NoError(t, err)
 	assert.NotNil(t, headResult.ETag)
 
 	getObject := &s3.GetObjectInput{
-		Bucket: aws.String(bucketName),
+		Bucket: aws.String(v.Bucket),
 		Key:    aws.String(key),
 	}
 

core/vault/vault_driver_test.go

@@ -8,11 +8,9 @@ package vault
 import (
 	"fmt"
 	"os"
-	"syscall"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/stretchr/testify/assert"
 	//		"bytes"
@@ -35,48 +33,14 @@ func getVaultFs(algo string, key []byte) *Vault {
 }
 
 func getVaultS3(algo string, key []byte) *Vault {
-	root := os.Getenv("GONODE_TEST_AWS_VAULT_ROOT")
 
-	if len(os.Getenv("GITHUB_RUN_ID")) > 0 {
-		root += "/" + os.Getenv("GITHUB_RUN_ID")
-	}
-
-	if len(root) == 0 {
-		root = "local"
-	}
-
-	bucket := os.Getenv("GONODE_TEST_AWS_VAULT_BUCKET")
-	if len(bucket) == 0 {
-		bucket = "gonode-test"
-	}
-
-	fmt.Printf("bucket: %s, root: %s\n", bucket, root)
-
-	creds := credentials.NewChainCredentials([]credentials.Provider{
-		&credentials.EnvProvider{},
-		&credentials.SharedCredentialsProvider{
-			Filename: os.Getenv("HOME") + "/.aws/credentials",
-			Profile:  "gonode-test",
-		},
-		&credentials.SharedCredentialsProvider{
-			Filename: os.Getenv("GONODE_TEST_AWS_CREDENTIALS_FILE"),
-			Profile:  os.Getenv("GONODE_TEST_AWS_PROFILE"),
-		},
-	})
-
-	_, err := creds.Get()
+	creds, err := getChainCredentials()
 
 	if err != nil {
 		return nil
 	}
 
-	driver := &DriverS3{
-		Root:        root,
-		Region:      "eu-west-1",
-		EndPoint:    "s3-eu-west-1.amazonaws.com",
-		Bucket:      bucket,
-		Credentials: creds,
-	}
+	driver := getDriver(creds)
 
 	v := &Vault{
 		Algo:    algo,
@@ -88,14 +52,15 @@ func getVaultS3(algo string, key []byte) *Vault {
 
 	// delete objects
 	l, _ := driver.client.ListObjects(&s3.ListObjectsInput{
-		Bucket: aws.String(bucket),
-		Prefix: aws.String(root),
+		Bucket: aws.String(driver.Bucket),
+		Prefix: aws.String(driver.Root),
 	})
 
 	for _, o := range l.Contents {
+		fmt.Printf("Delete: %s / %s\n", driver.Bucket, *o.Key)
 		driver.client.DeleteObject(&s3.DeleteObjectInput{
 			Key:    o.Key,
-			Bucket: aws.String(bucket),
+			Bucket: aws.String(driver.Bucket),
 		})
 	}
 
@@ -148,13 +113,12 @@ func Test_Vault_Drivers_FS(t *testing.T) {
 }
 
 func Test_Vault_Drivers_S3(t *testing.T) {
-	if _, offline := syscall.Getenv("GONODE_TEST_OFFLINE"); offline == true {
+	if getEnv("GONODE_TEST_OFFLINE", "yes") == "yes" {
 		t.Skip("OFFLINE TEST ONLY")
-
 		return
 	}
 
-	//runTest("s3", t, getVaultS3)
+	runTest("s3", t, getVaultS3)
 }
 
 //func Test_Generate_Regression_Files(t *testing.T) {

core/vault/vault_test.go

@@ -8,7 +8,6 @@ package vault
 import (
 	"bytes"
 	"crypto/rand"
-	"fmt"
 	"io"
 	"testing"
 
@@ -30,10 +29,8 @@ func init() {
 	largeMessage = make([]byte, 1024*1024*1+2)
 	io.ReadFull(rand.Reader, largeMessage)
 
-	fmt.Println("Start generating XLarge message")
 	xLargeMessage = make([]byte, 1024*1024*10+3)
 	io.ReadFull(rand.Reader, xLargeMessage)
-	fmt.Println("End generating XLarge message")
 }
 
 // write/encrypted file
@@ -49,10 +46,10 @@ func RunTestVault(t *testing.T, v *Vault, plaintext []byte, msgPrefix string) {
 
 	written, err := v.Put(file, meta, reader)
 
-	assert.NoError(t, err, msgPrefix+"err returned")
+	assert.NoError(t, err, msgPrefix+": err returned")
 	assert.True(t, written >= int64(len(plaintext)), msgPrefix) // some cipher might add extra data
 	assert.True(t, written > 0, msgPrefix)                      // some cipher might add extra data
-	assert.True(t, v.Has(file), msgPrefix)
+	assert.True(t, v.Has(file), msgPrefix+": has file should be true")
 
 	invalid := []byte("Another invalid message with the same key")
 
@@ -75,13 +72,13 @@ func RunTestVault(t *testing.T, v *Vault, plaintext []byte, msgPrefix string) {
 	assert.Equal(t, plaintext, writer.Bytes(), msgPrefix)
 
 	// remove file
-	v.Remove(file)
+	err = v.Remove(file)
 	assert.NoError(t, err, msgPrefix)
 }
 
 // read stored encrypted files
 func RunRegressionTest(t *testing.T, v *Vault) {
-	file := "The secret file"
+	file := "The-secret-file"
 
 	assert.True(t, v.Has(file))