Skip to content

Commit

Permalink
s3ng: Provide objectSize when uploading (#1940)
Browse files Browse the repository at this point in the history
  • Loading branch information
AndreasSko authored Aug 2, 2021
1 parent fcb7a30 commit 3c034f9
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 1 deletion.
7 changes: 7 additions & 0 deletions changelog/unreleased/provide-size-for-s3-upload.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Enhancement: Reduce memory usage when uploading with S3ng storage

The memory usage could be high when uploading files using the S3ng storage.
By providing the actual file size when triggering `PutObject`,
the overall memory usage is reduced.

https://github.com/cs3org/reva/pull/1940
12 changes: 11 additions & 1 deletion pkg/storage/fs/s3ng/blobstore/blobstore.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"context"
"io"
"net/url"
"os"

"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
Expand Down Expand Up @@ -60,7 +61,16 @@ func New(endpoint, region, bucket, accessKey, secretKey string) (*Blobstore, err

// Upload stores some data in the blobstore under the given key
func (bs *Blobstore) Upload(key string, reader io.Reader) error {
_, err := bs.client.PutObject(context.Background(), bs.bucket, key, reader, -1, minio.PutObjectOptions{ContentType: "application/octet-stream"})
size := int64(-1)
if file, ok := reader.(*os.File); ok {
info, err := file.Stat()
if err != nil {
return errors.Wrapf(err, "could not determine file size for object '%s'", key)
}
size = info.Size()
}

_, err := bs.client.PutObject(context.Background(), bs.bucket, key, reader, size, minio.PutObjectOptions{ContentType: "application/octet-stream"})

if err != nil {
return errors.Wrapf(err, "could not store object '%s' into bucket '%s'", key, bs.bucket)
Expand Down

0 comments on commit 3c034f9

Please sign in to comment.