Skip to content

Commit

Permalink
test, refactor: remove unnecessary checks, add test to pipe
Browse files Browse the repository at this point in the history
  • Loading branch information
@denizsurmeli
denizsurmeli committed Sep 11, 2023
1 parent 5394a50 commit c177bb2
Show file tree
Hide file tree
Showing 5 changed files with 78 additions and 183 deletions.
82 changes: 17 additions & 65 deletions command/cp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -691,43 +691,21 @@ func (c Copy) doUpload(ctx context.Context, srcurl *url.URL, dsturl *url.URL, ex
}

metadata := storage.Metadata{UserDefined: extradata}
if c.acl != "" {
metadata.ACL = c.acl
}

if c.cacheControl != "" {
metadata.CacheControl = c.cacheControl
}

if c.expires != "" {
metadata.Expires = c.expires
}

if c.storageClass != "" {
metadata.StorageClass = string(c.storageClass)
}
metadata.ACL = c.acl
metadata.CacheControl = c.cacheControl
metadata.Expires = c.expires
metadata.StorageClass = string(c.storageClass)

if c.contentType != "" {
metadata.ContentType = c.contentType
} else {
metadata.ContentType = guessContentType(file)
}

if c.contentEncoding != "" {
metadata.ContentEncoding = c.contentEncoding
}

if c.contentDisposition != "" {
metadata.ContentDisposition = c.contentDisposition
}

if c.encryptionMethod != "" {
metadata.EncryptionMethod = c.encryptionMethod
}

if c.encryptionKeyID != "" {
metadata.EncryptionKeyID = c.encryptionKeyID
}
metadata.ContentEncoding = c.contentEncoding
metadata.ContentDisposition = c.contentDisposition
metadata.EncryptionMethod = c.encryptionMethod
metadata.EncryptionKeyID = c.encryptionKeyID

reader := newCountingReaderWriter(file, c.progressbar)
err = dstClient.Put(ctx, reader, dsturl, metadata, c.concurrency, c.partSize)
Expand Down Expand Up @@ -776,41 +754,15 @@ func (c Copy) doCopy(ctx context.Context, srcurl, dsturl *url.URL, extradata map
}

metadata := storage.Metadata{UserDefined: extradata}
if c.acl != "" {
metadata.ACL = c.acl
}

if c.cacheControl != "" {
metadata.CacheControl = c.cacheControl
}

if c.expires != "" {
metadata.Expires = c.expires
}

if c.storageClass != "" {
metadata.StorageClass = string(c.storageClass)
}

if c.contentType != "" {
metadata.ContentType = c.contentType
}

if c.contentEncoding != "" {
metadata.ContentEncoding = c.contentEncoding
}

if c.contentDisposition != "" {
metadata.ContentDisposition = c.contentDisposition
}

if c.encryptionMethod != "" {
metadata.EncryptionMethod = c.encryptionMethod
}

if c.encryptionKeyID != "" {
metadata.EncryptionKeyID = c.encryptionKeyID
}
metadata.ACL = c.acl
metadata.CacheControl = c.cacheControl
metadata.Expires = c.expires
metadata.StorageClass = string(c.storageClass)
metadata.ContentType = c.contentType
metadata.ContentEncoding = c.contentEncoding
metadata.ContentDisposition = c.contentDisposition
metadata.EncryptionMethod = c.encryptionMethod
metadata.EncryptionKeyID = c.encryptionKeyID

err = c.shouldOverride(ctx, srcurl, dsturl)
if err != nil {
Expand Down
40 changes: 8 additions & 32 deletions command/pipe.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,43 +222,19 @@ func (c Pipe) Run(ctx context.Context) error {
}

metadata := storage.Metadata{UserDefined: c.metadata}
if c.acl != "" {
metadata.ACL = c.acl
}

if c.cacheControl != "" {
metadata.CacheControl = c.cacheControl
}

if c.expires != "" {
metadata.Expires = c.expires
}

if c.storageClass != "" {
metadata.StorageClass = string(c.storageClass)
}

metadata.ACL = c.acl
metadata.CacheControl = c.cacheControl
metadata.Expires = c.expires
metadata.StorageClass = string(c.storageClass)
if c.contentType != "" {
metadata.ContentType = c.contentType
} else {
metadata.ContentType = guessContentTypeByExtension(c.dst)
}

if c.contentEncoding != "" {
metadata.ContentEncoding = c.contentEncoding
}

if c.contentDisposition != "" {
metadata.ContentDisposition = c.contentDisposition
}

if c.encryptionMethod != "" {
metadata.EncryptionMethod = c.encryptionMethod
}

if c.encryptionKeyID != "" {
metadata.EncryptionKeyID = c.encryptionKeyID
}
metadata.ContentEncoding = c.contentEncoding
metadata.ContentDisposition = c.contentDisposition
metadata.EncryptionMethod = c.encryptionMethod
metadata.EncryptionKeyID = c.encryptionKeyID

err = client.Put(ctx, &stdin{file: os.Stdin}, c.dst, metadata, c.concurrency, c.partSize)
if err != nil {
Expand Down
42 changes: 3 additions & 39 deletions e2e/cp_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -767,7 +767,9 @@ func TestCopySingleFileToS3WithAllMetadataFlags(t *testing.T) {
"--content-encoding", ContentEncoding,
"--sse", EncryptionMethod,
"--sse-kms-key-id", EncryptionKeyID,
srcpath, dstpath)
srcpath, dstpath,
)

result := icmd.RunCmd(cmd)

result.Assert(t, icmd.Success)
Expand All @@ -788,44 +790,6 @@ func TestCopySingleFileToS3WithAllMetadataFlags(t *testing.T) {

}

// NOTE: This test needs an implementation of the
// `ACL` feature in gofakes3.
func TestCopySingleFileToS3WithACLFlag(t *testing.T) {
t.Skip()

t.Parallel()

s3client, s5cmd := setup(t)

bucket := s3BucketFromTestName(t)

createBucket(t, s3client, bucket)

const (
filename = "index"
content = `testfilecontent`
acl = "public-read"
)

workdir := fs.NewDir(t, bucket, fs.WithFile(filename, content))
defer workdir.Remove()

srcpath := workdir.Join(filename)
dstpath := fmt.Sprintf("s3://%v/", bucket)

srcpath = filepath.ToSlash(srcpath)
cmd := s5cmd("cp", "--acl", acl, srcpath, dstpath)
result := icmd.RunCmd(cmd)

result.Assert(t, icmd.Success)

expected := fs.Expected(t, fs.WithFile(filename, content))
assert.Assert(t, fs.Equal(workdir.Path(), expected))

assert.Assert(t, ensureS3Object(s3client, bucket, filename, content, ensureACL(acl)))

}

// cp dir/file s3://bucket/ --metadata key1=val1 --metadata key2=val2 ...
func TestCopySingleFileToS3WithArbitraryMetadata(t *testing.T) {
t.Parallel()
Expand Down
71 changes: 49 additions & 22 deletions e2e/pipe_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,14 @@ package e2e
import (
"bytes"
"fmt"
"net/http"
"runtime"
"testing"
"time"

"github.com/aws/aws-sdk-go/aws"
"gotest.tools/v3/assert"
"gotest.tools/v3/fs"
"gotest.tools/v3/icmd"
)

Expand Down Expand Up @@ -461,44 +464,68 @@ func TestUploadStdinToS3WithStorageClassGlacier(t *testing.T) {
}

// pipe --content-disposition inline s3://bucket/object
func TestUploadStdinToToS3WithContentDisposition(t *testing.T) {
func TestUploadStdinToToS3WithAllMetadataFlags(t *testing.T) {
t.Parallel()

s3client, s5cmd := setup(t)

bucket := s3BucketFromTestName(t)

createBucket(t, s3client, bucket)

const (
// make sure that Put reads the file header and guess Content-Type correctly.
filename = "index.html"
content = `
<html lang="tr">
<head>
<meta charset="utf-8">
<body>
<header></header>
<main></main>
<footer></footer>
</body>
</html>
`
expectedContentType = "text/html; charset=utf-8"
expectedContentDisposition = "inline"
filename = "index"
content = `testfilecontent`
cacheControl = "public, max-age=3600"
expires = "2025-01-01T00:00:00Z"
storageClass = "STANDARD_IA"
ContentType = "text/html; charset=utf-8"
ContentDisposition = "inline"
ContentEncoding = "utf-8"
EncryptionMethod = "aws:kms"
EncryptionKeyID = "1234abcd-12ab-34cd-56ef-1234567890ab"
)

// expected expires flag is the parsed version of the date in RFC3339 format
parsedTime, err := time.Parse(time.RFC3339, expires)
if err != nil {
t.Fatal(err)
}

expectedExpires := parsedTime.Format(http.TimeFormat)

workdir := fs.NewDir(t, bucket, fs.WithFile(filename, content))
defer workdir.Remove()

dstpath := fmt.Sprintf("s3://%v/%v", bucket, filename)

reader := bytes.NewBufferString(content)

cmd := s5cmd("pipe", "--content-disposition", "inline", dstpath)
cmd := s5cmd("pipe",
"--cache-control", cacheControl,
"--expires", expires,
"--storage-class", storageClass,
"--content-type", ContentType,
"--content-disposition", ContentDisposition,
"--content-encoding", ContentEncoding,
"--sse", EncryptionMethod,
"--sse-kms-key-id", EncryptionKeyID,
dstpath,
)

result := icmd.RunCmd(cmd, icmd.WithStdin(reader))

result.Assert(t, icmd.Success)

assertLines(t, result.Stdout(), map[int]compareFunc{
0: suffix(`pipe %v`, dstpath),
})

// assert S3
assert.Assert(t, ensureS3Object(s3client, bucket, filename, content, ensureContentType(expectedContentType), ensureContentDisposition(expectedContentDisposition)))
assert.Assert(t, ensureS3Object(s3client, bucket, filename, content,
ensureExpires(expectedExpires),
ensureCacheControl(cacheControl),
ensureStorageClass(storageClass),
ensureContentType(ContentType),
ensureContentDisposition(ContentDisposition),
ensureContentEncoding(ContentEncoding),
ensureEncryptionMethod(EncryptionMethod),
ensureEncryptionKeyID(EncryptionKeyID),
))
}
26 changes: 1 addition & 25 deletions e2e/util_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,7 @@ func createBucket(t *testing.T, client *s3.S3, bucket string) {

input := &s3.CreateBucketInput{
Bucket: aws.String(bucket),
ACL: aws.String(s3.BucketCannedACLPublicRead),
}

_, err := client.CreateBucket(input)
Expand Down Expand Up @@ -571,7 +572,6 @@ func setBucketVersioning(t *testing.T, s3client *s3.S3, bucket string, versionin
var errS3NoSuchKey = fmt.Errorf("s3: no such key")

type ensureOpts struct {
acl *string
cacheControl *string
expires *string
storageClass *string
Expand All @@ -585,12 +585,6 @@ type ensureOpts struct {

type ensureOption func(*ensureOpts)

func ensureACL(acl string) ensureOption {
return func(opts *ensureOpts) {
opts.acl = &acl
}
}

func ensureCacheControl(cacheControl string) ensureOption {
return func(opts *ensureOpts) {
opts.cacheControl = &cacheControl
Expand Down Expand Up @@ -681,24 +675,6 @@ func ensureS3Object(
return fmt.Errorf("s3 %v/%v: (-want +got):\n%v", bucket, key, diff)
}

// ACL is not returned by GetObject, so we need to make a separate call.
// TODO: Implement GetObjectAcl endpoint
// in gofakes3
if opts.acl != nil {
getACLOutput, err := client.GetObjectAcl(&s3.GetObjectAclInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})

if err != nil {
return err
}

if diff := cmp.Diff(*opts.acl, *getACLOutput.Grants[0].Permission); diff != "" {
return fmt.Errorf("acl of %v/%v: (-want +got):\n%v", bucket, key, diff)
}
}

if opts.cacheControl != nil {
if diff := cmp.Diff(opts.cacheControl, output.CacheControl); diff != "" {
return fmt.Errorf("cache-control of %v/%v: (-want +got):\n%v", bucket, key, diff)
Expand Down

0 comments on commit c177bb2

Please sign in to comment.