diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 54a3552..ba4b5a3 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -8,6 +8,7 @@ on: jobs: build: + environment: qa runs-on: ubuntu-latest services: postgres: @@ -35,6 +36,13 @@ jobs: go get ./... - name: Test + env: + GONODE_TEST_AWS_VAULT_S3_BUCKET: ${{ secrets.GONODE_TEST_AWS_VAULT_S3_BUCKET }} + GONODE_TEST_OFFLINE: ${{ secrets.GONODE_TEST_OFFLINE }} + GONODE_TEST_S3_ACCESS_KEY: ${{ secrets.GONODE_TEST_S3_ACCESS_KEY }} + GONODE_TEST_S3_ENDPOINT: ${{ secrets.GONODE_TEST_S3_ENDPOINT }} + GONODE_TEST_S3_REGION: ${{ secrets.GONODE_TEST_S3_REGION }} + GONODE_TEST_S3_SECRET: ${{ secrets.GONODE_TEST_S3_SECRET }} run: | make test diff --git a/Makefile b/Makefile index 46f2006..6a0b52f 100644 --- a/Makefile +++ b/Makefile @@ -6,16 +6,11 @@ GONODE_MODULES = $(shell ls -d ./modules/* | grep -v go) GONODE_CORE = $(shell ls -d ./core/* | grep -v go) GOPATH = $(shell go env GOPATH) -install: - $(call back,glide install) - $(call back,go get github.com/wadey/gocovmerge && go get golang.org/x/tools/cmd/cover && go get golang.org/x/tools/cmd/goimports && go get -u github.com/jteeuwen/go-bindata/...) - test: ./app/assets/bindata.sh mkdir -p data echo "mode: atomic" > data/coverage.out - - GONODE_TEST_OFFLINE=true GOPATH=${GOPATH} go test -v -failfast -covermode=atomic -coverprofile=data/coverage_core.out $(GONODE_CORE) + GOPATH=${GOPATH} go test -v -failfast -covermode=atomic -coverprofile=data/coverage_core.out $(GONODE_CORE) GOPATH=${GOPATH} go test -v -failfast -covermode=atomic -coverprofile=data/coverage_modules.out $(GONODE_MODULES) GOPATH=${GOPATH} go test -v -failfast -covermode=atomic -coverpkg ./... -coverprofile=data/coverage_integration.out ./test/modules go vet $(GONODE_CORE) $(GONODE_MODULES) ./test/modules/ diff --git a/core/vault/vault.go b/core/vault/vault.go index 1ed84e2..f0e5c68 100644 --- a/core/vault/vault.go +++ b/core/vault/vault.go @@ -183,7 +183,7 @@ func (v *Vault) Put(name string, meta VaultMetadata, r io.Reader) (written int64 defer w.Close() } - if _, err = io.Copy(w, buf); err != nil { + if written, err = io.Copy(w, buf); err != nil { v.removeIfExists(vaultfile) v.removeIfExists(metafile) @@ -204,8 +204,6 @@ func (v *Vault) Put(name string, meta VaultMetadata, r io.Reader) (written int64 v.removeIfExists(vaultfile) v.removeIfExists(metafile) v.removeIfExists(binfile) - - return } return diff --git a/core/vault/vault_driver_s3.go b/core/vault/vault_driver_s3.go index 624f2fc..08d9012 100644 --- a/core/vault/vault_driver_s3.go +++ b/core/vault/vault_driver_s3.go @@ -30,7 +30,6 @@ func (w *s3Writer) Write(b []byte) (int, error) { func (w *s3Writer) Close() error { name := w.file.Name() - defer func() { os.Remove(name) }() diff --git a/core/vault/vault_driver_s3_test.go b/core/vault/vault_driver_s3_test.go index d4a14c0..f58e28b 100644 --- a/core/vault/vault_driver_s3_test.go +++ b/core/vault/vault_driver_s3_test.go @@ -13,7 +13,6 @@ import ( "bytes" "fmt" "os" - "syscall" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" @@ -21,27 +20,17 @@ import ( "github.com/aws/aws-sdk-go/service/s3" ) -// this is just a test to validata how the aws sdk behave -func Test_Vault_Basic_S3_Usage(t *testing.T) { - - if _, offline := syscall.Getenv("GONODE_TEST_OFFLINE"); offline == true { - t.Skip("OFFLINE TEST ONLY") - return +func getEnv(name, def string) string { + value := os.Getenv(name) + if len(value) == 0 { + value = def } - var err error - var headResult *s3.HeadObjectOutput - var getResult *s3.GetObjectOutput - - root := os.Getenv("GONODE_TEST_AWS_VAULT_ROOT") - if len(root) == 0 { - root = "local" - } + return value +} - profile := os.Getenv("GONODE_TEST_AWS_PROFILE") - if len(profile) == 0 { - profile = "gonode-test" - } +func getChainCredentials() (*credentials.Credentials, error) { + profile := getEnv("GONODE_TEST_AWS_PROFILE", "gonode-test") chainProvider := credentials.NewChainCredentials([]credentials.Provider{ &credentials.EnvProvider{}, @@ -53,21 +42,48 @@ func Test_Vault_Basic_S3_Usage(t *testing.T) { Filename: os.Getenv("GONODE_TEST_AWS_CREDENTIALS_FILE"), Profile: profile, }, + &credentials.StaticProvider{Value: credentials.Value{ + AccessKeyID: getEnv("GONODE_TEST_S3_ACCESS_KEY", ""), + SecretAccessKey: getEnv("GONODE_TEST_S3_SECRET", ""), + }}, }) - _, err = chainProvider.Get() + if _, err := chainProvider.Get(); err != nil { + return nil, err + } + + return chainProvider, nil +} + +func getDriver(chainProvider *credentials.Credentials) *DriverS3 { + return &DriverS3{ + Bucket: getEnv("GONODE_TEST_AWS_VAULT_S3_BUCKET", "gonode-qa"), + Root: getEnv("GITHUB_RUN_ID", getEnv("GONODE_TEST_AWS_VAULT_ROOT", "local")), + Region: getEnv("GONODE_TEST_S3_REGION", "eu-west-1"), + EndPoint: getEnv("GONODE_TEST_S3_ENDPOINT", "s3-eu-west-1.amazonaws.com"), + Credentials: chainProvider, + } +} + +// this is just a test to validata how the aws sdk behave +func Test_Vault_Basic_S3_Usage(t *testing.T) { + if getEnv("GONODE_TEST_OFFLINE", "yes") == "yes" { + t.Skip("OFFLINE TEST ONLY") + return + } + + var err error + var headResult *s3.HeadObjectOutput + var getResult *s3.GetObjectOutput + + chainProvider, err := getChainCredentials() if err != nil { t.Skip("Unable to find credentials") } // init vault - v := &DriverS3{ - Root: root, - Region: "eu-west-1", - EndPoint: "s3-eu-west-1.amazonaws.com", - Credentials: chainProvider, - } + v := getDriver(chainProvider) // init credentials information config := &aws.Config{ @@ -79,15 +95,10 @@ func Test_Vault_Basic_S3_Usage(t *testing.T) { s3client := s3.New(session.New(), config) - bucketName := os.Getenv("GONODE_TEST_AWS_VAULT_S3_BUCKET") - if len(bucketName) == 0 { - bucketName = "gonode-test" - } - key := fmt.Sprintf("%s/test/assd", v.Root) headResult, err = s3client.HeadObject(&s3.HeadObjectInput{ - Bucket: aws.String(bucketName), + Bucket: aws.String(v.Bucket), Key: aws.String("no-file"), }) @@ -97,7 +108,7 @@ func Test_Vault_Basic_S3_Usage(t *testing.T) { data := []byte("foobar et foo") putObject := &s3.PutObjectInput{ - Bucket: aws.String(bucketName), + Bucket: aws.String(v.Bucket), Key: aws.String(key), Body: bytes.NewReader(data), ContentType: aws.String("application/octet-stream"), @@ -106,7 +117,7 @@ func Test_Vault_Basic_S3_Usage(t *testing.T) { _, err = s3client.PutObject(putObject) headResult, err = s3client.HeadObject(&s3.HeadObjectInput{ - Bucket: aws.String(bucketName), + Bucket: aws.String(v.Bucket), Key: aws.String(key), }) @@ -114,7 +125,7 @@ func Test_Vault_Basic_S3_Usage(t *testing.T) { assert.NotNil(t, headResult.ETag) getObject := &s3.GetObjectInput{ - Bucket: aws.String(bucketName), + Bucket: aws.String(v.Bucket), Key: aws.String(key), } diff --git a/core/vault/vault_driver_test.go b/core/vault/vault_driver_test.go index 205cc01..e9883ec 100644 --- a/core/vault/vault_driver_test.go +++ b/core/vault/vault_driver_test.go @@ -8,11 +8,9 @@ package vault import ( "fmt" "os" - "syscall" "testing" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/service/s3" "github.com/stretchr/testify/assert" // "bytes" @@ -35,48 +33,14 @@ func getVaultFs(algo string, key []byte) *Vault { } func getVaultS3(algo string, key []byte) *Vault { - root := os.Getenv("GONODE_TEST_AWS_VAULT_ROOT") - if len(os.Getenv("GITHUB_RUN_ID")) > 0 { - root += "/" + os.Getenv("GITHUB_RUN_ID") - } - - if len(root) == 0 { - root = "local" - } - - bucket := os.Getenv("GONODE_TEST_AWS_VAULT_BUCKET") - if len(bucket) == 0 { - bucket = "gonode-test" - } - - fmt.Printf("bucket: %s, root: %s\n", bucket, root) - - creds := credentials.NewChainCredentials([]credentials.Provider{ - &credentials.EnvProvider{}, - &credentials.SharedCredentialsProvider{ - Filename: os.Getenv("HOME") + "/.aws/credentials", - Profile: "gonode-test", - }, - &credentials.SharedCredentialsProvider{ - Filename: os.Getenv("GONODE_TEST_AWS_CREDENTIALS_FILE"), - Profile: os.Getenv("GONODE_TEST_AWS_PROFILE"), - }, - }) - - _, err := creds.Get() + creds, err := getChainCredentials() if err != nil { return nil } - driver := &DriverS3{ - Root: root, - Region: "eu-west-1", - EndPoint: "s3-eu-west-1.amazonaws.com", - Bucket: bucket, - Credentials: creds, - } + driver := getDriver(creds) v := &Vault{ Algo: algo, @@ -88,14 +52,15 @@ func getVaultS3(algo string, key []byte) *Vault { // delete objects l, _ := driver.client.ListObjects(&s3.ListObjectsInput{ - Bucket: aws.String(bucket), - Prefix: aws.String(root), + Bucket: aws.String(driver.Bucket), + Prefix: aws.String(driver.Root), }) for _, o := range l.Contents { + fmt.Printf("Delete: %s / %s\n", driver.Bucket, *o.Key) driver.client.DeleteObject(&s3.DeleteObjectInput{ Key: o.Key, - Bucket: aws.String(bucket), + Bucket: aws.String(driver.Bucket), }) } @@ -148,13 +113,12 @@ func Test_Vault_Drivers_FS(t *testing.T) { } func Test_Vault_Drivers_S3(t *testing.T) { - if _, offline := syscall.Getenv("GONODE_TEST_OFFLINE"); offline == true { + if getEnv("GONODE_TEST_OFFLINE", "yes") == "yes" { t.Skip("OFFLINE TEST ONLY") - return } - //runTest("s3", t, getVaultS3) + runTest("s3", t, getVaultS3) } //func Test_Generate_Regression_Files(t *testing.T) { diff --git a/core/vault/vault_test.go b/core/vault/vault_test.go index 742fc3f..26769b5 100644 --- a/core/vault/vault_test.go +++ b/core/vault/vault_test.go @@ -8,7 +8,6 @@ package vault import ( "bytes" "crypto/rand" - "fmt" "io" "testing" @@ -30,10 +29,8 @@ func init() { largeMessage = make([]byte, 1024*1024*1+2) io.ReadFull(rand.Reader, largeMessage) - fmt.Println("Start generating XLarge message") xLargeMessage = make([]byte, 1024*1024*10+3) io.ReadFull(rand.Reader, xLargeMessage) - fmt.Println("End generating XLarge message") } // write/encrypted file @@ -49,10 +46,10 @@ func RunTestVault(t *testing.T, v *Vault, plaintext []byte, msgPrefix string) { written, err := v.Put(file, meta, reader) - assert.NoError(t, err, msgPrefix+"err returned") + assert.NoError(t, err, msgPrefix+": err returned") assert.True(t, written >= int64(len(plaintext)), msgPrefix) // some cipher might add extra data assert.True(t, written > 0, msgPrefix) // some cipher might add extra data - assert.True(t, v.Has(file), msgPrefix) + assert.True(t, v.Has(file), msgPrefix+": has file should be true") invalid := []byte("Another invalid message with the same key") @@ -75,13 +72,13 @@ func RunTestVault(t *testing.T, v *Vault, plaintext []byte, msgPrefix string) { assert.Equal(t, plaintext, writer.Bytes(), msgPrefix) // remove file - v.Remove(file) + err = v.Remove(file) assert.NoError(t, err, msgPrefix) } // read stored encrypted files func RunRegressionTest(t *testing.T, v *Vault) { - file := "The secret file" + file := "The-secret-file" assert.True(t, v.Has(file))