Skip to content

Commit

Permalink
Read file support (#58)
Browse files Browse the repository at this point in the history
  • Loading branch information
@carrala
carrala committed May 10, 2021
1 parent 38ad962 commit b811f8c
Show file tree
Hide file tree
Showing 4 changed files with 127 additions and 70 deletions.
110 changes: 84 additions & 26 deletions client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ import (
const (
DefaultStorageURL = "https://api.e3db.com"
DefaultEncryptedFileName = "encrypted"
DefaultDownloadedFileName = "downloaded"
SecretUUID = "38bb737a-4ce0-5ead-8585-e13ea23b09a6"
SecretWriterUsernameMetadataKey = "username"
SecretSharedMetadataKey = "shared"
Expand Down Expand Up @@ -1429,7 +1430,7 @@ type Secret struct {
SecretValue string
Description string
FileName string
SecretID string
SecretID uuid.UUID
RealmName string
OwnerUsername string
NamespaceId string
Expand Down Expand Up @@ -1472,7 +1473,12 @@ func (c *ToznySDKV3) CreateSecret(ctx context.Context, secret CreateSecretOption
}
var createdRecord *pdsClient.Record
if secret.SecretType == FileSecretType {
createdRecord, err = c.WriteFile(ctx, recordType, plain, secret.FileName)
writeFileRequest := WriteFileOptions{
RecordType: recordType,
Plain: plain,
FileName: secret.FileName,
}
createdRecord, err = c.WriteFile(ctx, writeFileRequest)
if err != nil {
return nil, err
}
Expand All @@ -1491,46 +1497,52 @@ func (c *ToznySDKV3) CreateSecret(ctx context.Context, secret CreateSecretOption
return createdSecret, nil
}

type WriteFileOptions struct {
RecordType string
Plain map[string]string
FileName string
}

// WriteFile encrypts the file with specified fileName and uploads it, creating a new record in E3DB
func (c *ToznySDKV3) WriteFile(ctx context.Context, recordType string, plain map[string]string, fileName string) (*pdsClient.Record, error) {
func (c *ToznySDKV3) WriteFile(ctx context.Context, options WriteFileOptions) (*pdsClient.Record, error) {
keyRequest := pdsClient.GetOrCreateAccessKeyRequest{
WriterID: c.E3dbPDSClient.ClientID,
UserID: c.E3dbPDSClient.ClientID,
ReaderID: c.E3dbPDSClient.ClientID,
RecordType: recordType,
RecordType: options.RecordType,
}
ak, err := c.E3dbPDSClient.GetOrCreateAccessKey(ctx, keyRequest)
if err != nil {
return nil, err
}
// Encrypt the file
size, checksum, err := e3dbClients.EncryptFile(fileName, DefaultEncryptedFileName, ak)
encryptionInfo, err := e3dbClients.EncryptFile(options.FileName, DefaultEncryptedFileName, ak)
if err != nil {
return nil, err
}
defer func() {
err := os.Remove(DefaultEncryptedFileName)
err := os.Remove(encryptionInfo.EncryptedFileName)
if err != nil {
fmt.Println("CreateSecret: error deleting encrypted file")
fmt.Printf("WriteFile: Could not delete %s: %+v", encryptionInfo.EncryptedFileName, err)
}
}()
plain[SecretFilenameMetadataKey] = fileName
sizeKB := size / 1024
options.Plain[SecretFilenameMetadataKey] = options.FileName
sizeKB := encryptionInfo.Size / 1024
if sizeKB >= 1 {
plain[SecretFileSizeMetadataKey] = fmt.Sprintf("%d", sizeKB)
options.Plain[SecretFileSizeMetadataKey] = fmt.Sprintf("%d", sizeKB)
} else {
plain[SecretFileSizeMetadataKey] = "< 1"
options.Plain[SecretFileSizeMetadataKey] = "< 1"
}
// Write the whole file
recordToWrite := storageClient.Record{
Metadata: storageClient.Meta{
Type: recordType,
Type: options.RecordType,
WriterID: uuid.MustParse(c.StorageClient.ClientID),
UserID: uuid.MustParse(c.StorageClient.ClientID),
Plain: plain,
Plain: options.Plain,
FileMeta: &storageClient.FileMeta{
Size: int64(size),
Checksum: checksum,
Size: int64(encryptionInfo.Size),
Checksum: encryptionInfo.Checksum,
Compression: "raw",
},
},
Expand All @@ -1541,12 +1553,12 @@ func (c *ToznySDKV3) WriteFile(ctx context.Context, recordType string, plain map
}
uploadRequest := file.UploadRequest{
URL: pendingFileURL.FileURL,
EncryptedFileName: DefaultEncryptedFileName,
Checksum: checksum,
Size: size,
EncryptedFileName: encryptionInfo.EncryptedFileName,
Checksum: encryptionInfo.Checksum,
Size: encryptionInfo.Size,
}
uploadResp, err := file.UploadFile(uploadRequest)
if err != nil || uploadResp != 0 {
err = file.UploadFile(uploadRequest)
if err != nil {
return nil, err
}
// Register the file as being written
Expand All @@ -1573,6 +1585,52 @@ func (c *ToznySDKV3) WriteFile(ctx context.Context, recordType string, plain map
return fileRecord, nil
}

type ReadFileOptions struct {
RecordID uuid.UUID
DownloadFileName string
}

// ReadFile downloads and decrypts the file from the record
func (c *ToznySDKV3) ReadFile(ctx context.Context, options ReadFileOptions) error {
fileResp, err := c.E3dbPDSClient.GetFileRecord(ctx, options.RecordID)
if err != nil {
return err
}
fileURL := fileResp.Metadata.FileMeta.FileURL
downloadRequest := file.DownloadRequest{
URL: fileURL,
EncryptedFileName: DefaultDownloadedFileName,
}
// download file from URL and store in EncryptedFileName
downloadedPath, err := file.DownloadFile(downloadRequest)
if err != nil {
return fmt.Errorf("ReadFile: Err: %+v", err)
}
defer func() {
err := os.Remove(downloadedPath)
if err != nil {
fmt.Printf("ReadFile: Could not delete %s: %+v", downloadedPath, err)
}
}()
// get access key for the record type
keyRequest := pdsClient.GetOrCreateAccessKeyRequest{
WriterID: c.E3dbPDSClient.ClientID,
UserID: c.E3dbPDSClient.ClientID,
ReaderID: c.E3dbPDSClient.ClientID,
RecordType: fileResp.Metadata.Type,
}
ak, err := c.E3dbPDSClient.GetOrCreateAccessKey(ctx, keyRequest)
if err != nil {
return err
}
// decrypt the file with the access key
err = e3dbClients.DecryptFile(downloadedPath, options.DownloadFileName, ak)
if err != nil {
return err
}
return nil
}

// WriteRecord encrypts the data for the record and creates a new record in E3DB
func (c *ToznySDKV3) WriteRecord(ctx context.Context, data map[string]string, recordType string, plain map[string]string) (*pdsClient.Record, error) {
recordToWrite := pdsClient.WriteRecordRequest{
Expand Down Expand Up @@ -1867,7 +1925,7 @@ func (c *ToznySDKV3) ListSecrets(ctx context.Context, options ListSecretsOptions
}
listRequest := storageClient.ListGroupRecordsRequest{
GroupID: group.GroupID,
Max: options.Limit,
Max: options.Limit,
}
for {
listGroupRecords, err := c.StorageClient.GetSharedWithGroup(ctx, listRequest)
Expand Down Expand Up @@ -1923,7 +1981,7 @@ func (c *ToznySDKV3) ListSecrets(ctx context.Context, options ListSecretsOptions
}

type ViewSecretOptions struct {
SecretID string
SecretID uuid.UUID
MaxSecrets int
}

Expand All @@ -1941,17 +1999,17 @@ func (c *ToznySDKV3) ViewSecret(ctx context.Context, options ViewSecretOptions)
var nextToken string
for _, group := range groupList.Groups {
listRequest := storageClient.ListGroupRecordsRequest{
GroupID: group.GroupID,
GroupID: group.GroupID,
NextToken: nextToken,
Max: options.MaxSecrets,
Max: options.MaxSecrets,
}
for {
listGroupRecords, err := c.StorageClient.GetSharedWithGroup(ctx, listRequest)
if err != nil {
return nil, err
}
for _, record := range listGroupRecords.ResultList {
if record.Metadata.RecordID == options.SecretID {
if record.Metadata.RecordID == options.SecretID.String() {
secret = &record
groupID = group.GroupID.String()
break
Expand Down Expand Up @@ -2003,7 +2061,7 @@ func (c *ToznySDKV3) MakeSecretResponse(secretRecord *pdsClient.Record, groupID
secret := &Secret{
SecretName: secretRecord.Metadata.Plain[SecretNameMetadataKey],
SecretType: secretRecord.Metadata.Plain[SecretTypeMetadataKey],
SecretID: secretRecord.Metadata.RecordID,
SecretID: uuid.MustParse(secretRecord.Metadata.RecordID),
Description: secretRecord.Metadata.Plain[SecretDescriptionMetadataKey],
Version: secretRecord.Metadata.Plain[SecretVersionMetadataKey],
Record: secretRecord,
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ require (
github.com/jawher/mow.cli v1.2.0
github.com/mitchellh/go-homedir v1.1.0
github.com/stretchr/testify v1.7.0 // indirect
github.com/tozny/e3db-clients-go v0.0.145
github.com/tozny/e3db-clients-go v0.0.146
golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf
golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c
)
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,8 +141,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tozny/e3db-clients-go v0.0.145 h1:Bk76Q2kvvdEpu2S2jcGSgw+8iJBxFJoWrbMLuPpUmTI=
github.com/tozny/e3db-clients-go v0.0.145/go.mod h1:xqnK5S5r0qLrKCUms5Mi/3oij2ppNg2lk/8iggyn7IQ=
github.com/tozny/e3db-clients-go v0.0.146 h1:pSJKF5W9d/RSVj4QgQylBQIZUT3AWoflQ1t4AMg70Ko=
github.com/tozny/e3db-clients-go v0.0.146/go.mod h1:xqnK5S5r0qLrKCUms5Mi/3oij2ppNg2lk/8iggyn7IQ=
github.com/tozny/utils-go v0.0.35 h1:gPvhlQ8QCoLBUjIx1COfYy6o4dfSM8Lrh+2FV9Ask+g=
github.com/tozny/utils-go v0.0.35/go.mod h1:SHi9wnpPEEzAxbwcBhRd+jW32r+gY6S+AcWweuGytRw=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
Expand Down
81 changes: 40 additions & 41 deletions secrets_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package e3db

import (
"bytes"
"context"
"fmt"
"io/ioutil"
"os"
"testing"

Expand All @@ -15,11 +17,14 @@ var (
username = os.Getenv("TEST_IDENTITY_USERNAME")
password = os.Getenv("TEST_IDENTITY_PASSWORD")
baseURL = os.Getenv("TEST_IDENTITY_API_URL")
secret1ID = os.Getenv("TEST_CREATED_SECRET1_ID")
secret2ID = os.Getenv("TEST_CREATED_SECRET2_ID")
testCtx = context.Background()
plaintextFileName = "plainfile"
decryptedFileName = "decrypted"
)

func TestCreateAndListSecrets(t *testing.T) {
func TestListSecrets(t *testing.T) {
request := TozIDLoginRequest{
Username: username,
Password: password,
Expand All @@ -31,40 +36,6 @@ func TestCreateAndListSecrets(t *testing.T) {
if err != nil {
t.Fatalf("Could not log in %+v", err)
}
secretReq := CreateSecretOptions{
SecretName: fmt.Sprintf("client-%s", uuid.New().String()),
SecretType: CredentialSecretType,
SecretValue: uuid.New().String(),
Description: "a credential test",
RealmName: realmName,
}
validClient := `{
"version": "2",
"public_signing_key": "A5QXkIKW5dBN_IOhjGoUBtT-xuVmqRXDB2uaqiKuTao",
"private_signing_key": "qIqG9_81kd2gOY-yggIpahQG1MDnlBeQj7G4MHa5p0E1WapQxLVlyU6hXA6rp-Ci5DFf8g6GMaqy5t_H1g5Nqg",
"client_id": "4f20ca95-1b3b-b78f-b5bd-6d469ac804eb",
"api_key_id": "63807026e9a23850307429e52d2f607eaa5be43488cbb819b075ade91735b180",
"api_secret": "730e6b18dc9668fe1758304283c73060619f6596f11bf42bdd3f16d6fc6cd6d0",
"public_key": "6u73qLgJniPi9S2t99A7lNfvi3xjxMsPB_Z-CEGWZmo",
"private_key": "BnBt9_tquBvSAHL04bQm0HkQ7eXtvuj1WSHegQeho6E",
"api_url": "http://platform.local.tozny.com:8000",
"client_email": ""
}`
secretReq2 := CreateSecretOptions{
SecretName: fmt.Sprintf("cred-%s", uuid.New().String()),
SecretType: ClientSecretType,
SecretValue: validClient,
Description: "a client cred test",
RealmName: realmName,
}
secret1, err := sdk.CreateSecret(testCtx, secretReq)
if err != nil {
t.Fatalf("Could not create secret: Req: %+v Err: %+v", secretReq, err)
}
secret2, err := sdk.CreateSecret(testCtx, secretReq2)
if err != nil {
t.Fatalf("Could not create secret: Req: %+v Err: %+v", secretReq2, err)
}
listOptions := ListSecretsOptions{
RealmName: realmName,
Limit: 1000,
Expand All @@ -76,11 +47,12 @@ func TestCreateAndListSecrets(t *testing.T) {
}
found1 := false
found2 := false
// Check that the two pre-created secrets are in the list
for _, secret := range listSecrets.List {
if secret.Record.Metadata.RecordID == secret1.Record.Metadata.RecordID && secretReq.SecretValue == secret.Record.Data["secretValue"] {
if secret.Record.Metadata.RecordID == secret1ID {
found1 = true
}
if secret.Record.Metadata.RecordID == secret2.Record.Metadata.RecordID && secretReq2.SecretValue == secret.Record.Data["secretValue"] {
if secret.Record.Metadata.RecordID == secret2ID {
found2 = true
}
}
Expand Down Expand Up @@ -172,19 +144,19 @@ func TestCreateAndViewSecretSucceeds(t *testing.T) {
t.Fatalf("Could not create secret: Req: %+v Err: %+v", secretReq, err)
}
viewOptions := ViewSecretOptions{
SecretID: secretCreated.Record.Metadata.RecordID,
SecretID: secretCreated.SecretID,
MaxSecrets: 1000,
}
secretView, err := sdk.ViewSecret(testCtx, viewOptions)
if err != nil {
t.Fatalf("Could not view secret: Err: %+v", err)
}
if secretReq.SecretValue != secretView.Record.Data["secretValue"] {
if secretReq.SecretValue != secretView.SecretValue {
t.Fatalf("SecretValue doesn't match. Created: %s Viewed: %s", secretCreated.Record.Data["secretValue"], secretView.Record.Data["secretValue"])
}
}

func TestCreateAndViewFileSecretSucceeds(t *testing.T) {
func TestCreateAndReadFileSecretSucceeds(t *testing.T) {
request := TozIDLoginRequest{
Username: username,
Password: password,
Expand Down Expand Up @@ -219,10 +191,37 @@ func TestCreateAndViewFileSecretSucceeds(t *testing.T) {
FileName: plaintextFileName,
RealmName: realmName,
}
_, err = sdk.CreateSecret(testCtx, secretReq)
createdSecret, err := sdk.CreateSecret(testCtx, secretReq)
if err != nil {
t.Fatalf("Could not create secret: Req: %+v Err: %+v", secretReq, err)
}
readFileOptions := ReadFileOptions{
RecordID: createdSecret.SecretID,
DownloadFileName: decryptedFileName,
}
err = sdk.ReadFile(testCtx, readFileOptions)
if err != nil {
t.Fatalf("Could not read file: Err: %+v", err)
}
defer func() {
err := os.Remove(decryptedFileName)
if err != nil {
t.Logf("Could not delete %s: %+v", decryptedFileName, err)
}
}()
// Compare plaintext and decrypted file contents
plaintext, err := ioutil.ReadFile(plaintextFileName)
if err != nil {
t.Fatalf("Could not read %s file: %+v", plaintextFileName, err)
}
decrypted, err := ioutil.ReadFile(decryptedFileName)
if err != nil {
t.Fatalf("Could not read %s file: %+v", decryptedFileName, err)
}
compare := bytes.Equal(plaintext, decrypted)
if !compare {
t.Fatalf("%s and %s files do not match", plaintextFileName, decryptedFileName)
}
}

func mfaHandler(sessionResponse *IdentitySessionIntermediateResponse) (LoginActionData, error) {
Expand Down

0 comments on commit b811f8c

Please sign in to comment.