diff --git a/blob/azureblob/azureblob.go b/blob/azureblob/azureblob.go index 61e5883468..5f619f2667 100644 --- a/blob/azureblob/azureblob.go +++ b/blob/azureblob/azureblob.go @@ -815,9 +815,17 @@ type writer struct { client *blockblob.Client uploadOpts *azblob.UploadStreamOptions - w *io.PipeWriter - donec chan struct{} - err error + // Ends of an io.Pipe, created when the first byte is written. + pw *io.PipeWriter + pr *io.PipeReader + + // Alternatively, upload is set to true when Upload was + // used to upload data. + upload bool + + donec chan struct{} // closed when done writing + // The following fields will be written before donec closes: + err error } // escapeKey does all required escaping for UTF-8 strings to work with Azure. @@ -916,50 +924,64 @@ func (b *bucket) NewTypedWriter(ctx context.Context, key string, contentType str }, nil } -// Write appends p to w. User must call Close to close the w after done writing. +// Write appends p to w.pw. User must call Close to close the w after done writing. func (w *writer) Write(p []byte) (int, error) { + // Avoid opening the pipe for a zero-length write; + // the concrete can do these for empty blobs. if len(p) == 0 { return 0, nil } - if w.w == nil { - pr, pw := io.Pipe() - w.w = pw - if err := w.open(pr); err != nil { - return 0, err - } + if w.pw == nil { + // We'll write into pw and use pr as an io.Reader for the + // Upload call to Azure. + w.pr, w.pw = io.Pipe() + w.open(w.pr, true) } - return w.w.Write(p) + return w.pw.Write(p) +} + +// Upload reads from r. Per the driver, it is guaranteed to be the only +// write call for this writer. +func (w *writer) Upload(r io.Reader) error { + w.upload = true + w.open(r, false) + return nil } -func (w *writer) open(pr *io.PipeReader) error { +// r may be nil if we're Closing and no data was written. +// If closePipeOnError is true, w.pr will be closed if there's an +// error uploading to Azure. +func (w *writer) open(r io.Reader, closePipeOnError bool) { go func() { defer close(w.donec) - var body io.Reader - if pr == nil { - body = http.NoBody - } else { - body = pr + if r == nil { + r = http.NoBody } - _, w.err = w.client.UploadStream(w.ctx, body, w.uploadOpts) + _, w.err = w.client.UploadStream(w.ctx, r, w.uploadOpts) if w.err != nil { - if pr != nil { - pr.CloseWithError(w.err) + if closePipeOnError { + w.pr.CloseWithError(w.err) + w.pr = nil } - return } }() - return nil } // Close completes the writer and closes it. Any error occurring during write will // be returned. If a writer is closed before any Write is called, Close will // create an empty file at the given key. func (w *writer) Close() error { - if w.w == nil { - w.open(nil) - } else if err := w.w.Close(); err != nil { - return err + if !w.upload { + if w.pr != nil { + defer w.pr.Close() + } + if w.pw == nil { + // We never got any bytes written. We'll write an http.NoBody. + w.open(nil, false) + } else if err := w.pw.Close(); err != nil { + return err + } } <-w.donec return w.err diff --git a/blob/azureblob/testdata/TestConformance/TestUploadDownload.replay b/blob/azureblob/testdata/TestConformance/TestUploadDownload.replay new file mode 100644 index 0000000000..9301e3be54 --- /dev/null +++ b/blob/azureblob/testdata/TestConformance/TestUploadDownload.replay @@ -0,0 +1,273 @@ +{ + "Initial": "AQAAAA7b69HoFIvtFP5c", + "Version": "0.2", + "Converter": { + "ScrubBody": [ + "\u003cBlock(l|L)ist\u003e\u003cLatest\u003e.*\u003c/Latest\u003e\u003c/Block(l|L)ist\u003e" + ], + "ClearHeaders": [ + "^X-Goog-.*Encryption-Key$", + "^X-Ms-Date$", + "^X-Ms-Version$", + "^User-Agent$" + ], + "RemoveRequestHeaders": [ + "^Authorization$", + "^Proxy-Authorization$", + "^Connection$", + "^Content-Type$", + "^Date$", + "^Host$", + "^Transfer-Encoding$", + "^Via$", + "^X-Forwarded-.*$", + "^X-Cloud-Trace-Context$", + "^X-Goog-Api-Client$", + "^X-Google-.*$", + "^X-Gfe-.*$" + ], + "RemoveResponseHeaders": [ + "^X-Google-.*$", + "^X-Gfe-.*$" + ], + "ClearParams": [ + "^blockid$" + ], + "RemoveParams": [ + "^se$", + "^sig$", + "^st$", + "^X-Ms-Date$" + ] + }, + "Entries": [ + { + "ID": "6d7f91b6c7cec13b", + "Request": { + "Method": "PUT", + "URL": "https://gocloudblobtests.blob.core.windows.net/go-cloud-bucket/blob-for-upload-download", + "Header": { + "Accept": [ + "application/xml" + ], + "Accept-Encoding": [ + "gzip" + ], + "Content-Length": [ + "11" + ], + "User-Agent": [ + "CLEARED" + ], + "X-Ms-Blob-Cache-Control": [ + "" + ], + "X-Ms-Blob-Content-Disposition": [ + "" + ], + "X-Ms-Blob-Content-Encoding": [ + "" + ], + "X-Ms-Blob-Content-Language": [ + "" + ], + "X-Ms-Blob-Content-Type": [ + "text" + ], + "X-Ms-Blob-Type": [ + "BlockBlob" + ], + "X-Ms-Date": [ + "CLEARED" + ], + "X-Ms-Version": [ + "CLEARED" + ] + }, + "MediaType": "application/octet-stream", + "BodyParts": [ + "dXAgYW5kIGRvd24=" + ] + }, + "Response": { + "StatusCode": 201, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Content-Length": [ + "0" + ], + "Content-Md5": [ + "G3VTPtWoaf9vOuAzbQwzIA==" + ], + "Date": [ + "Tue, 09 May 2023 05:32:24 GMT" + ], + "Etag": [ + "\"0x8DB504EC520C6BA\"" + ], + "Last-Modified": [ + "Tue, 09 May 2023 05:32:24 GMT" + ], + "Server": [ + "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" + ], + "X-Ms-Content-Crc64": [ + "wOkxZJKwY/U=" + ], + "X-Ms-Request-Id": [ + "c5d0dade-701e-0025-2a37-82b15e000000" + ], + "X-Ms-Request-Server-Encrypted": [ + "true" + ], + "X-Ms-Version": [ + "CLEARED" + ] + }, + "Body": "" + } + }, + { + "ID": "4ced1b1e3dd7f6ac", + "Request": { + "Method": "GET", + "URL": "https://gocloudblobtests.blob.core.windows.net/go-cloud-bucket/blob-for-upload-download", + "Header": { + "Accept": [ + "application/xml" + ], + "Accept-Encoding": [ + "gzip" + ], + "User-Agent": [ + "CLEARED" + ], + "X-Ms-Date": [ + "CLEARED" + ], + "X-Ms-Version": [ + "CLEARED" + ] + }, + "MediaType": "", + "BodyParts": [ + null + ] + }, + "Response": { + "StatusCode": 200, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Accept-Ranges": [ + "bytes" + ], + "Content-Disposition": [ + "" + ], + "Content-Length": [ + "11" + ], + "Content-Md5": [ + "G3VTPtWoaf9vOuAzbQwzIA==" + ], + "Content-Type": [ + "text" + ], + "Date": [ + "Tue, 09 May 2023 05:32:24 GMT" + ], + "Etag": [ + "\"0x8DB504EC520C6BA\"" + ], + "Last-Modified": [ + "Tue, 09 May 2023 05:32:24 GMT" + ], + "Server": [ + "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" + ], + "X-Ms-Blob-Type": [ + "BlockBlob" + ], + "X-Ms-Creation-Time": [ + "Tue, 09 May 2023 05:32:24 GMT" + ], + "X-Ms-Lease-State": [ + "available" + ], + "X-Ms-Lease-Status": [ + "unlocked" + ], + "X-Ms-Request-Id": [ + "c5d0daea-701e-0025-3437-82b15e000000" + ], + "X-Ms-Server-Encrypted": [ + "true" + ], + "X-Ms-Version": [ + "CLEARED" + ] + }, + "Body": "dXAgYW5kIGRvd24=" + } + }, + { + "ID": "ce0443c3194c87e1", + "Request": { + "Method": "DELETE", + "URL": "https://gocloudblobtests.blob.core.windows.net/go-cloud-bucket/blob-for-upload-download", + "Header": { + "Accept": [ + "application/xml" + ], + "Accept-Encoding": [ + "gzip" + ], + "User-Agent": [ + "CLEARED" + ], + "X-Ms-Date": [ + "CLEARED" + ], + "X-Ms-Version": [ + "CLEARED" + ] + }, + "MediaType": "", + "BodyParts": [ + null + ] + }, + "Response": { + "StatusCode": 202, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Content-Length": [ + "0" + ], + "Date": [ + "Tue, 09 May 2023 05:32:24 GMT" + ], + "Server": [ + "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" + ], + "X-Ms-Delete-Type-Permanent": [ + "true" + ], + "X-Ms-Request-Id": [ + "c5d0daef-701e-0025-3837-82b15e000000" + ], + "X-Ms-Version": [ + "CLEARED" + ] + }, + "Body": "" + } + } + ] +} \ No newline at end of file diff --git a/blob/blob.go b/blob/blob.go index fd1f9d47da..f530e48443 100644 --- a/blob/blob.go +++ b/blob/blob.go @@ -237,11 +237,33 @@ func (r *Reader) WriteTo(w io.Writer) (int64, error) { return nw, err } +// downloadAndClose is similar to WriteTo, but ensures it's the only read. +// This pattern is more optimal for some drivers. +func (r *Reader) downloadAndClose(w io.Writer) (err error) { + if r.bytesRead != 0 { + // Shouldn't happen. + return gcerr.Newf(gcerr.Internal, nil, "blob: downloadAndClose isn't the first read") + } + driverDownloader, ok := r.r.(driver.Downloader) + if ok { + err = driverDownloader.Download(w) + } else { + _, err = r.WriteTo(w) + } + cerr := r.Close() + if err == nil && cerr != nil { + err = cerr + } + return err +} + // readFromWriteTo is a helper for ReadFrom and WriteTo. // It reads data from r and writes to w, until EOF or a read/write error. // It returns the number of bytes read from r and the number of bytes // written to w. func readFromWriteTo(r io.Reader, w io.Writer) (int64, int64, error) { + // Note: can't use io.Copy because it will try to use r.WriteTo + // or w.WriteTo, which is recursive in this context. buf := make([]byte, 1024) var totalRead, totalWritten int64 for { @@ -458,6 +480,26 @@ func (w *Writer) ReadFrom(r io.Reader) (int64, error) { return nr, err } +// uploadAndClose is similar to ReadFrom, but ensures it's the only write. +// This pattern is more optimal for some drivers. +func (w *Writer) uploadAndClose(r io.Reader) (err error) { + if w.bytesWritten != 0 { + // Shouldn't happen. + return gcerr.Newf(gcerr.Internal, nil, "blob: uploadAndClose must be the first write") + } + driverUploader, ok := w.w.(driver.Uploader) + if ok { + err = driverUploader.Upload(r) + } else { + _, err = w.ReadFrom(r) + } + cerr := w.Close() + if err == nil && cerr != nil { + err = cerr + } + return err +} + // ListOptions sets options for listing blobs via Bucket.List. type ListOptions struct { // Prefix indicates that only blobs with a key starting with this prefix @@ -644,6 +686,8 @@ func (b *Bucket) ErrorAs(err error, i interface{}) bool { // ReadAll is a shortcut for creating a Reader via NewReader with nil // ReaderOptions, and reading the entire blob. +// +// Using Download may be more efficient. func (b *Bucket) ReadAll(ctx context.Context, key string) (_ []byte, err error) { b.mu.RLock() defer b.mu.RUnlock() @@ -658,6 +702,20 @@ func (b *Bucket) ReadAll(ctx context.Context, key string) (_ []byte, err error) return ioutil.ReadAll(r) } +// Download writes the content of a blob into an io.Writer w. +func (b *Bucket) Download(ctx context.Context, key string, w io.Writer, opts *ReaderOptions) error { + b.mu.RLock() + defer b.mu.RUnlock() + if b.closed { + return errClosed + } + r, err := b.NewReader(ctx, key, opts) + if err != nil { + return err + } + return r.downloadAndClose(w) +} + // List returns a ListIterator that can be used to iterate over blobs in a // bucket, in lexicographical order of UTF-8 encoded keys. The underlying // implementation fetches results in pages. @@ -934,6 +992,8 @@ func (b *Bucket) newRangeReader(ctx context.Context, key string, offset, length // // If opts.ContentMD5 is not set, WriteAll will compute the MD5 of p and use it // as the ContentMD5 option for the Writer it creates. +// +// Using Upload may be more efficient. func (b *Bucket) WriteAll(ctx context.Context, key string, p []byte, opts *WriterOptions) (err error) { realOpts := new(WriterOptions) if opts != nil { @@ -954,6 +1014,20 @@ func (b *Bucket) WriteAll(ctx context.Context, key string, p []byte, opts *Write return w.Close() } +// Upload reads from an io.Reader r and writes into a blob. +// +// opts.ContentType is required. +func (b *Bucket) Upload(ctx context.Context, key string, r io.Reader, opts *WriterOptions) error { + if opts == nil || opts.ContentType == "" { + return gcerr.Newf(gcerr.InvalidArgument, nil, "blob: Upload requires WriterOptions.ContentType") + } + w, err := b.NewWriter(ctx, key, opts) + if err != nil { + return err + } + return w.uploadAndClose(r) +} + // NewWriter returns a Writer that writes to the blob stored at key. // A nil WriterOptions is treated the same as the zero value. // diff --git a/blob/blob_test.go b/blob/blob_test.go index 0c6ebc456b..22bb8efee4 100644 --- a/blob/blob_test.go +++ b/blob/blob_test.go @@ -194,6 +194,77 @@ func (b *fakeLister) ListPaged(ctx context.Context, opts *driver.ListOptions) (* func (*fakeLister) Close() error { return nil } func (*fakeLister) ErrorCode(err error) gcerrors.ErrorCode { return gcerrors.Unknown } +type stubReader struct { + driver.Reader + downloaded bool +} + +func (r *stubReader) Download(w io.Writer) error { + r.downloaded = true + return nil +} + +func (*stubReader) Close() error { return nil } + +type stubWriter struct { + driver.Writer + uploaded bool +} + +func (w *stubWriter) Upload(r io.Reader) error { + w.uploaded = true + return nil +} + +func (*stubWriter) Close() error { return nil } + +// loaderBucket implements driver.Bucket's NewTypedWriter and NewRangedReader methods, +// returning stubReader and stubWriter. It is used to verify that the special driver.Uploader +// and driver.Downloader overrides work when called. +type loaderBucket struct { + driver.Bucket + w stubWriter + r stubReader +} + +func (b *loaderBucket) NewTypedWriter(ctx context.Context, key string, contentType string, opts *driver.WriterOptions) (driver.Writer, error) { + return &b.w, nil +} + +func (b *loaderBucket) NewRangeReader(ctx context.Context, key string, offset, length int64, opts *driver.ReaderOptions) (driver.Reader, error) { + return &b.r, nil +} + +func (*loaderBucket) Close() error { return nil } + +func TestUploader(t *testing.T) { + ctx := context.Background() + lb := &loaderBucket{} + b := NewBucket(lb) + defer b.Close() + err := b.Upload(ctx, "key", nil, &WriterOptions{ContentType: "text/html"}) + if err != nil { + t.Fatalf("Upload failed: %v", err) + } + if !lb.w.uploaded { + t.Error("Uploader wasn't called") + } +} + +func TestDownloader(t *testing.T) { + ctx := context.Background() + lb := &loaderBucket{} + b := NewBucket(lb) + defer b.Close() + err := b.Download(ctx, "key", nil, nil) + if err != nil { + t.Fatalf("Download failed: %v", err) + } + if !lb.r.downloaded { + t.Error("Downloader wasn't called") + } +} + // erroringBucket implements driver.Bucket. All interface methods that return // errors are implemented, and return errFake. // In addition, when passed the key "work", NewRangeReader and NewTypedWriter diff --git a/blob/driver/driver.go b/blob/driver/driver.go index 9d89b63e7f..6ac9b1eeff 100644 --- a/blob/driver/driver.go +++ b/blob/driver/driver.go @@ -50,11 +50,24 @@ type Reader interface { As(interface{}) bool } +// Downloader has an optional extra method for readers. +// It is similar to io.WriteTo, but without the count of bytes returned. +type Downloader interface { + // Download is similar to io.WriteTo, but without the count of bytes returned. + Download(w io.Writer) error +} + // Writer writes an object to the blob. type Writer interface { io.WriteCloser } +// Uploader has an optional extra method for writers. +type Uploader interface { + // Upload is similar to io.ReadFrom, but without the count of bytes returned. + Upload(r io.Reader) error +} + // WriterOptions controls behaviors of Writer. type WriterOptions struct { // BufferSize changes the default size in byte of the maximum part Writer can @@ -263,6 +276,11 @@ type Bucket interface { // exist, NewRangeReader must return an error for which ErrorCode returns // gcerrors.NotFound. // opts is guaranteed to be non-nil. + // + // The returned Reader *may* also implement Downloader if the underlying + // implementation can take advantage of that. The Download call is guaranteed + // to be the only call to the Reader. For such readers, offset will always + // be 0 and length will always be -1. NewRangeReader(ctx context.Context, key string, offset, length int64, opts *ReaderOptions) (Reader, error) // NewTypedWriter returns Writer that writes to an object associated with key. @@ -279,6 +297,10 @@ type Bucket interface { // // Implementations should abort an ongoing write if ctx is later canceled, // and do any necessary cleanup in Close. Close should then return ctx.Err(). + // + // The returned Writer *may* also implement Uploader if the underlying + // implementation can take advantage of that. The Upload call is guaranteed + // to be the only non-Close call to the Writer.. NewTypedWriter(ctx context.Context, key, contentType string, opts *WriterOptions) (Writer, error) // Copy copies the object associated with srcKey to dstKey. diff --git a/blob/drivertest/drivertest.go b/blob/drivertest/drivertest.go index 697530be67..6c10034587 100644 --- a/blob/drivertest/drivertest.go +++ b/blob/drivertest/drivertest.go @@ -237,6 +237,9 @@ func RunConformanceTests(t *testing.T, newHarness HarnessMaker, asTests []AsTest t.Run("TestConcurrentWriteAndRead", func(t *testing.T) { testConcurrentWriteAndRead(t, newHarness) }) + t.Run("TestUploadDownload", func(t *testing.T) { + testUploadDownload(t, newHarness) + }) t.Run("TestMetadata", func(t *testing.T) { testMetadata(t, newHarness) }) @@ -2126,6 +2129,42 @@ func testConcurrentWriteAndRead(t *testing.T, newHarness HarnessMaker) { wg.Wait() } +// testUploadDownload tests that Upload and Download work. For many drivers, +// these are implemented in the concrete type, but drivers that implement Reader.Download +// and/or Writer.Upload will have those called directly. +func testUploadDownload(t *testing.T, newHarness HarnessMaker) { + const key = "blob-for-upload-download" + const contents = "up and down" + ctx := context.Background() + h, err := newHarness(ctx, t) + if err != nil { + t.Fatal(err) + } + defer h.Close() + drv, err := h.MakeDriver(ctx) + if err != nil { + t.Fatal(err) + } + b := blob.NewBucket(drv) + defer b.Close() + + // Write a blob using Upload. + if err := b.Upload(ctx, key, strings.NewReader(contents), &blob.WriterOptions{ContentType: "text"}); err != nil { + t.Fatal(err) + } + defer b.Delete(ctx, key) + + // Read the blob using Download. + var bb bytes.Buffer + err = b.Download(ctx, key, &bb, nil) + if err != nil { + t.Fatal(err) + } + if bb.String() != contents { + t.Errorf("read data mismatch for key %s", key) + } +} + // testKeys tests a variety of weird keys. func testKeys(t *testing.T, newHarness HarnessMaker) { const keyPrefix = "weird-keys" diff --git a/blob/fileblob/fileblob.go b/blob/fileblob/fileblob.go index 3fc7ac281a..89673782fd 100644 --- a/blob/fileblob/fileblob.go +++ b/blob/fileblob/fileblob.go @@ -789,6 +789,11 @@ type writer struct { path string } +func (w *writer) Upload(r io.Reader) error { + _, err := w.ReadFrom(r) + return err +} + func (w *writer) Close() error { err := w.File.Close() if err != nil { diff --git a/blob/gcsblob/testdata/TestConformance/TestUploadDownload.replay b/blob/gcsblob/testdata/TestConformance/TestUploadDownload.replay new file mode 100644 index 0000000000..af911f9557 --- /dev/null +++ b/blob/gcsblob/testdata/TestConformance/TestUploadDownload.replay @@ -0,0 +1,276 @@ +{ + "Initial": "AQAAAA7b69WPOQKWSv5c", + "Version": "0.2", + "Converter": { + "ScrubBody": null, + "ClearHeaders": [ + "^X-Goog-.*Encryption-Key$", + "^Expires$", + "^Signature$" + ], + "RemoveRequestHeaders": [ + "^Authorization$", + "^Proxy-Authorization$", + "^Connection$", + "^Content-Type$", + "^Date$", + "^Host$", + "^Transfer-Encoding$", + "^Via$", + "^X-Forwarded-.*$", + "^X-Cloud-Trace-Context$", + "^X-Goog-Api-Client$", + "^X-Google-.*$", + "^X-Gfe-.*$" + ], + "RemoveResponseHeaders": [ + "^X-Google-.*$", + "^X-Gfe-.*$" + ], + "ClearParams": [ + "^Expires$", + "^Signature$" + ], + "RemoveParams": null + }, + "Entries": [ + { + "ID": "d0e7f1e950944982", + "Request": { + "Method": "POST", + "URL": "https://storage.googleapis.com/upload/storage/v1/b/go-cloud-blob-test-bucket/o?alt=json\u0026name=blob-for-upload-download\u0026prettyPrint=false\u0026projection=full\u0026uploadType=multipart", + "Header": { + "Accept-Encoding": [ + "gzip" + ], + "User-Agent": [ + "google-api-go-client/0.5 go-cloud/blob/0.1.0" + ] + }, + "MediaType": "multipart/related", + "BodyParts": [ + "eyJidWNrZXQiOiJnby1jbG91ZC1ibG9iLXRlc3QtYnVja2V0IiwiY29udGVudFR5cGUiOiJ0ZXh0IiwibmFtZSI6ImJsb2ItZm9yLXVwbG9hZC1kb3dubG9hZCJ9Cg==", + "dXAgYW5kIGRvd24=" + ] + }, + "Response": { + "StatusCode": 200, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Alt-Svc": [ + "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + ], + "Cache-Control": [ + "no-cache, no-store, max-age=0, must-revalidate" + ], + "Content-Length": [ + "2894" + ], + "Content-Type": [ + "application/json; charset=UTF-8" + ], + "Date": [ + "Tue, 09 May 2023 05:48:00 GMT" + ], + "Etag": [ + "COaNwuLE5/4CEAE=" + ], + "Expires": [ + "CLEARED" + ], + "Pragma": [ + "no-cache" + ], + "Server": [ + "UploadServer" + ], + "Vary": [ + "Origin", + "X-Origin" + ], + "X-Guploader-Customer": [ + "apiary_cloudstorage_single_post_uploads" + ], + "X-Guploader-Request-Class": [ + "LATENCY_SENSITIVE" + ], + "X-Guploader-Request-Result": [ + "success" + ], + "X-Guploader-Upload-Result": [ + "success" + ], + "X-Guploader-Uploadid": [ + "ADPycdvwDj4z7c1q8sY5kWy2twefs6xu4s9G_aewNmj9jDNnH6EDRTnNTLsOZhpDZewf_ucqhD8I25RQD9QE9Tni4jy4tQ" + ] + }, + "Body": "eyJraW5kIjoic3RvcmFnZSNvYmplY3QiLCJpZCI6ImdvLWNsb3VkLWJsb2ItdGVzdC1idWNrZXQvYmxvYi1mb3ItdXBsb2FkLWRvd25sb2FkLzE2ODM2MTEyODA1MDg2NDYiLCJzZWxmTGluayI6Imh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL3N0b3JhZ2UvdjEvYi9nby1jbG91ZC1ibG9iLXRlc3QtYnVja2V0L28vYmxvYi1mb3ItdXBsb2FkLWRvd25sb2FkIiwibWVkaWFMaW5rIjoiaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2Rvd25sb2FkL3N0b3JhZ2UvdjEvYi9nby1jbG91ZC1ibG9iLXRlc3QtYnVja2V0L28vYmxvYi1mb3ItdXBsb2FkLWRvd25sb2FkP2dlbmVyYXRpb249MTY4MzYxMTI4MDUwODY0NiZhbHQ9bWVkaWEiLCJuYW1lIjoiYmxvYi1mb3ItdXBsb2FkLWRvd25sb2FkIiwiYnVja2V0IjoiZ28tY2xvdWQtYmxvYi10ZXN0LWJ1Y2tldCIsImdlbmVyYXRpb24iOiIxNjgzNjExMjgwNTA4NjQ2IiwibWV0YWdlbmVyYXRpb24iOiIxIiwiY29udGVudFR5cGUiOiJ0ZXh0Iiwic3RvcmFnZUNsYXNzIjoiUkVHSU9OQUwiLCJzaXplIjoiMTEiLCJtZDVIYXNoIjoiRzNWVFB0V29hZjl2T3VBemJRd3pJQT09IiwiY3JjMzJjIjoiMzVjNnJnPT0iLCJldGFnIjoiQ09hTnd1TEU1LzRDRUFFPSIsInRpbWVDcmVhdGVkIjoiMjAyMy0wNS0wOVQwNTo0ODowMC41MTBaIiwidXBkYXRlZCI6IjIwMjMtMDUtMDlUMDU6NDg6MDAuNTEwWiIsInRpbWVTdG9yYWdlQ2xhc3NVcGRhdGVkIjoiMjAyMy0wNS0wOVQwNTo0ODowMC41MTBaIiwiYWNsIjpbeyJraW5kIjoic3RvcmFnZSNvYmplY3RBY2Nlc3NDb250cm9sIiwib2JqZWN0IjoiYmxvYi1mb3ItdXBsb2FkLWRvd25sb2FkIiwiZ2VuZXJhdGlvbiI6IjE2ODM2MTEyODA1MDg2NDYiLCJpZCI6ImdvLWNsb3VkLWJsb2ItdGVzdC1idWNrZXQvYmxvYi1mb3ItdXBsb2FkLWRvd25sb2FkLzE2ODM2MTEyODA1MDg2NDYvcHJvamVjdC1vd25lcnMtODkyOTQyNjM4MTI5Iiwic2VsZkxpbmsiOiJodHRwczovL3d3dy5nb29nbGVhcGlzLmNvbS9zdG9yYWdlL3YxL2IvZ28tY2xvdWQtYmxvYi10ZXN0LWJ1Y2tldC9vL2Jsb2ItZm9yLXVwbG9hZC1kb3dubG9hZC9hY2wvcHJvamVjdC1vd25lcnMtODkyOTQyNjM4MTI5IiwiYnVja2V0IjoiZ28tY2xvdWQtYmxvYi10ZXN0LWJ1Y2tldCIsImVudGl0eSI6InByb2plY3Qtb3duZXJzLTg5Mjk0MjYzODEyOSIsInJvbGUiOiJPV05FUiIsImV0YWciOiJDT2FOd3VMRTUvNENFQUU9IiwicHJvamVjdFRlYW0iOnsicHJvamVjdE51bWJlciI6Ijg5Mjk0MjYzODEyOSIsInRlYW0iOiJvd25lcnMifX0seyJraW5kIjoic3RvcmFnZSNvYmplY3RBY2Nlc3NDb250cm9sIiwib2JqZWN0IjoiYmxvYi1mb3ItdXBsb2FkLWRvd25sb2FkIiwiZ2VuZXJhdGlvbiI6IjE2ODM2MTEyODA1MDg2NDYiLCJpZCI6ImdvLWNsb3VkLWJsb2ItdGVzdC1idWNrZXQvYmxvYi1mb3ItdXBsb2FkLWRvd25sb2FkLzE2ODM2MTEyODA1MDg2NDYvcHJvamVjdC1lZGl0b3JzLTg5Mjk0MjYzODEyOSIsInNlbGZMaW5rIjoiaHR0cHM6Ly93d3cuZ29vZ2xlYXBpcy5jb20vc3RvcmFnZS92MS9iL2dvLWNsb3VkLWJsb2ItdGVzdC1idWNrZXQvby9ibG9iLWZvci11cGxvYWQtZG93bmxvYWQvYWNsL3Byb2plY3QtZWRpdG9ycy04OTI5NDI2MzgxMjkiLCJidWNrZXQiOiJnby1jbG91ZC1ibG9iLXRlc3QtYnVja2V0IiwiZW50aXR5IjoicHJvamVjdC1lZGl0b3JzLTg5Mjk0MjYzODEyOSIsInJvbGUiOiJPV05FUiIsImV0YWciOiJDT2FOd3VMRTUvNENFQUU9IiwicHJvamVjdFRlYW0iOnsicHJvamVjdE51bWJlciI6Ijg5Mjk0MjYzODEyOSIsInRlYW0iOiJlZGl0b3JzIn19LHsia2luZCI6InN0b3JhZ2Ujb2JqZWN0QWNjZXNzQ29udHJvbCIsIm9iamVjdCI6ImJsb2ItZm9yLXVwbG9hZC1kb3dubG9hZCIsImdlbmVyYXRpb24iOiIxNjgzNjExMjgwNTA4NjQ2IiwiaWQiOiJnby1jbG91ZC1ibG9iLXRlc3QtYnVja2V0L2Jsb2ItZm9yLXVwbG9hZC1kb3dubG9hZC8xNjgzNjExMjgwNTA4NjQ2L3Byb2plY3Qtdmlld2Vycy04OTI5NDI2MzgxMjkiLCJzZWxmTGluayI6Imh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL3N0b3JhZ2UvdjEvYi9nby1jbG91ZC1ibG9iLXRlc3QtYnVja2V0L28vYmxvYi1mb3ItdXBsb2FkLWRvd25sb2FkL2FjbC9wcm9qZWN0LXZpZXdlcnMtODkyOTQyNjM4MTI5IiwiYnVja2V0IjoiZ28tY2xvdWQtYmxvYi10ZXN0LWJ1Y2tldCIsImVudGl0eSI6InByb2plY3Qtdmlld2Vycy04OTI5NDI2MzgxMjkiLCJyb2xlIjoiUkVBREVSIiwiZXRhZyI6IkNPYU53dUxFNS80Q0VBRT0iLCJwcm9qZWN0VGVhbSI6eyJwcm9qZWN0TnVtYmVyIjoiODkyOTQyNjM4MTI5IiwidGVhbSI6InZpZXdlcnMifX0seyJraW5kIjoic3RvcmFnZSNvYmplY3RBY2Nlc3NDb250cm9sIiwib2JqZWN0IjoiYmxvYi1mb3ItdXBsb2FkLWRvd25sb2FkIiwiZ2VuZXJhdGlvbiI6IjE2ODM2MTEyODA1MDg2NDYiLCJpZCI6ImdvLWNsb3VkLWJsb2ItdGVzdC1idWNrZXQvYmxvYi1mb3ItdXBsb2FkLWRvd25sb2FkLzE2ODM2MTEyODA1MDg2NDYvdXNlci1ydmFuZ2VudEBnb29nbGUuY29tIiwic2VsZkxpbmsiOiJodHRwczovL3d3dy5nb29nbGVhcGlzLmNvbS9zdG9yYWdlL3YxL2IvZ28tY2xvdWQtYmxvYi10ZXN0LWJ1Y2tldC9vL2Jsb2ItZm9yLXVwbG9hZC1kb3dubG9hZC9hY2wvdXNlci1ydmFuZ2VudEBnb29nbGUuY29tIiwiYnVja2V0IjoiZ28tY2xvdWQtYmxvYi10ZXN0LWJ1Y2tldCIsImVudGl0eSI6InVzZXItcnZhbmdlbnRAZ29vZ2xlLmNvbSIsInJvbGUiOiJPV05FUiIsImVtYWlsIjoicnZhbmdlbnRAZ29vZ2xlLmNvbSIsImV0YWciOiJDT2FOd3VMRTUvNENFQUU9In1dLCJvd25lciI6eyJlbnRpdHkiOiJ1c2VyLXJ2YW5nZW50QGdvb2dsZS5jb20ifX0=" + } + }, + { + "ID": "8488b6e39b32771b", + "Request": { + "Method": "GET", + "URL": "https://storage.googleapis.com/go-cloud-blob-test-bucket/blob-for-upload-download", + "Header": { + "Accept-Encoding": [ + "gzip" + ], + "User-Agent": [ + "go-cloud/blob/0.1.0" + ] + }, + "MediaType": "", + "BodyParts": [ + "" + ] + }, + "Response": { + "StatusCode": 200, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Accept-Ranges": [ + "bytes" + ], + "Alt-Svc": [ + "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + ], + "Cache-Control": [ + "private, max-age=0" + ], + "Content-Length": [ + "11" + ], + "Content-Type": [ + "text" + ], + "Date": [ + "Tue, 09 May 2023 05:48:00 GMT" + ], + "Etag": [ + "\"1b75533ed5a869ff6f3ae0336d0c3320\"" + ], + "Expires": [ + "CLEARED" + ], + "Last-Modified": [ + "Tue, 09 May 2023 05:48:00 GMT" + ], + "Server": [ + "UploadServer" + ], + "X-Goog-Generation": [ + "1683611280508646" + ], + "X-Goog-Hash": [ + "crc32c=35c6rg==", + "md5=G3VTPtWoaf9vOuAzbQwzIA==" + ], + "X-Goog-Metageneration": [ + "1" + ], + "X-Goog-Storage-Class": [ + "REGIONAL" + ], + "X-Goog-Stored-Content-Encoding": [ + "identity" + ], + "X-Goog-Stored-Content-Length": [ + "11" + ], + "X-Guploader-Customer": [ + "cloud-storage" + ], + "X-Guploader-Request-Class": [ + "LATENCY_SENSITIVE" + ], + "X-Guploader-Request-Result": [ + "success" + ], + "X-Guploader-Upload-Result": [ + "success" + ], + "X-Guploader-Uploadid": [ + "ADPycdtVE2zncuMfY8raPzVo-JwnRBR3x9ZIKUXsJJUhl4xyNWahK0X_7khtNpqS-GPoEkLy_aBXeda-OSmoQgDPqDpCLn25MhwO" + ] + }, + "Body": "dXAgYW5kIGRvd24=" + } + }, + { + "ID": "544efbd276b41ac1", + "Request": { + "Method": "DELETE", + "URL": "https://storage.googleapis.com/storage/v1/b/go-cloud-blob-test-bucket/o/blob-for-upload-download?alt=json\u0026prettyPrint=false", + "Header": { + "Accept-Encoding": [ + "gzip" + ], + "User-Agent": [ + "google-api-go-client/0.5 go-cloud/blob/0.1.0" + ] + }, + "MediaType": "", + "BodyParts": [ + "" + ] + }, + "Response": { + "StatusCode": 204, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Alt-Svc": [ + "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + ], + "Cache-Control": [ + "no-cache, no-store, max-age=0, must-revalidate" + ], + "Content-Length": [ + "0" + ], + "Content-Type": [ + "application/json" + ], + "Date": [ + "Tue, 09 May 2023 05:48:00 GMT" + ], + "Expires": [ + "CLEARED" + ], + "Pragma": [ + "no-cache" + ], + "Server": [ + "UploadServer" + ], + "Vary": [ + "Origin", + "X-Origin" + ], + "X-Guploader-Customer": [ + "apiary_cloudstorage_metadata" + ], + "X-Guploader-Request-Class": [ + "LATENCY_SENSITIVE" + ], + "X-Guploader-Request-Result": [ + "success" + ], + "X-Guploader-Upload-Result": [ + "success" + ], + "X-Guploader-Uploadid": [ + "ADPycdsKyMlx-crC2XzJsOnUWASfWru8qepzCbKCjcBzMtYst3RnirIGm2BpiMEfffdSsLzhEJyg1YtvweO2dlbtZnBMcQ" + ] + }, + "Body": "" + } + } + ] +} \ No newline at end of file diff --git a/blob/memblob/memblob.go b/blob/memblob/memblob.go index 85540d3792..f45b557d80 100644 --- a/blob/memblob/memblob.go +++ b/blob/memblob/memblob.go @@ -256,6 +256,14 @@ func (r *reader) Read(p []byte) (int, error) { return r.r.Read(p) } +func (r *reader) Download(w io.Writer) error { + // This should always work because r.r was created from a bytes.Reader. + // It's only not a WriterTo when we wrap it with a LimitReader, + // which is guaranteed not to happen by the driver interface. + _, err := r.r.(io.WriterTo).WriteTo(w) + return err +} + func (r *reader) Close() error { return nil } @@ -314,6 +322,11 @@ func (w *writer) Write(p []byte) (n int, err error) { return w.buf.Write(p) } +func (w *writer) Upload(r io.Reader) error { + _, err := w.buf.ReadFrom(r) + return err +} + func (w *writer) Close() error { // Check if the write was cancelled. if err := w.ctx.Err(); err != nil { diff --git a/blob/s3blob/s3blob.go b/blob/s3blob/s3blob.go index b697f2377d..7a2e89b67b 100644 --- a/blob/s3blob/s3blob.go +++ b/blob/s3blob/s3blob.go @@ -264,7 +264,13 @@ func (r *reader) Attributes() *driver.ReaderAttributes { // writer writes an S3 object, it implements io.WriteCloser. type writer struct { - w *io.PipeWriter // created when the first byte is written + // Ends of an io.Pipe, created when the first byte is written. + pw *io.PipeWriter + pr *io.PipeReader + + // Alternatively, upload is set to true when Upload was + // used to upload data. + upload bool ctx context.Context useV2 bool @@ -280,69 +286,74 @@ type writer struct { err error } -// Write appends p to w. User must call Close to close the w after done writing. +// Write appends p to w.pw. User must call Close to close the w after done writing. func (w *writer) Write(p []byte) (int, error) { // Avoid opening the pipe for a zero-length write; // the concrete can do these for empty blobs. if len(p) == 0 { return 0, nil } - if w.w == nil { + if w.pw == nil { // We'll write into pw and use pr as an io.Reader for the // Upload call to S3. - pr, pw := io.Pipe() - w.w = pw - if err := w.open(pr); err != nil { - return 0, err - } - } - select { - case <-w.donec: - return 0, w.err - default: + w.pr, w.pw = io.Pipe() + w.open(w.pr, true) } - return w.w.Write(p) + return w.pw.Write(p) } -// pr may be nil if we're Closing and no data was written. -func (w *writer) open(pr *io.PipeReader) error { +// Upload reads from r. Per the driver, it is guaranteed to be the only +// write call for this writer. +func (w *writer) Upload(r io.Reader) error { + w.upload = true + w.open(r, false) + return nil +} +// r may be nil if we're Closing and no data was written. +// If closePipeOnError is true, w.pr will be closed if there's an +// error uploading to S3. +func (w *writer) open(r io.Reader, closePipeOnError bool) { + // This goroutine will keep running until Close, unless there's an error. go func() { defer close(w.donec) - body := io.Reader(pr) - if pr == nil { + if r == nil { // AWS doesn't like a nil Body. - body = http.NoBody + r = http.NoBody } var err error if w.useV2 { - w.reqV2.Body = body + w.reqV2.Body = r _, err = w.uploaderV2.Upload(w.ctx, w.reqV2) } else { - w.req.Body = body + w.req.Body = r _, err = w.uploader.UploadWithContext(w.ctx, w.req) } if err != nil { - w.err = err - if pr != nil { - pr.CloseWithError(err) + if closePipeOnError { + w.pr.CloseWithError(err) + w.pr = nil } - return + w.err = err } }() - return nil } // Close completes the writer and closes it. Any error occurring during write // will be returned. If a writer is closed before any Write is called, Close // will create an empty file at the given key. func (w *writer) Close() error { - if w.w == nil { - // We never got any bytes written. We'll write an http.NoBody. - w.open(nil) - } else if err := w.w.Close(); err != nil { - return err + if !w.upload { + if w.pr != nil { + defer w.pr.Close() + } + if w.pw == nil { + // We never got any bytes written. We'll write an http.NoBody. + w.open(nil, false) + } else if err := w.pw.Close(); err != nil { + return err + } } <-w.donec return w.err diff --git a/blob/s3blob/testdata/TestConformance/TestUploadDownload.replay b/blob/s3blob/testdata/TestConformance/TestUploadDownload.replay new file mode 100644 index 0000000000..567004ca6e --- /dev/null +++ b/blob/s3blob/testdata/TestConformance/TestUploadDownload.replay @@ -0,0 +1,276 @@ +{ + "Initial": "AQAAAA7b69IEElG5Y/5c", + "Version": "0.2", + "Converter": { + "ScrubBody": null, + "ClearHeaders": [ + "^X-Goog-.*Encryption-Key$", + "^X-Amz-Date$", + "^User-Agent$" + ], + "RemoveRequestHeaders": [ + "^Authorization$", + "^Proxy-Authorization$", + "^Connection$", + "^Content-Type$", + "^Date$", + "^Host$", + "^Transfer-Encoding$", + "^Via$", + "^X-Forwarded-.*$", + "^X-Cloud-Trace-Context$", + "^X-Goog-Api-Client$", + "^X-Google-.*$", + "^X-Gfe-.*$", + "^Authorization$", + "^Duration$", + "^X-Amz-Security-Token$" + ], + "RemoveResponseHeaders": [ + "^X-Google-.*$", + "^X-Gfe-.*$" + ], + "ClearParams": [ + "^X-Amz-Date$" + ], + "RemoveParams": [ + "^X-Amz-Credential$", + "^X-Amz-Signature$", + "^X-Amz-Security-Token$" + ] + }, + "Entries": [ + { + "ID": "dd6b97872330631b", + "Request": { + "Method": "PUT", + "URL": "https://go-cloud-testing.s3.us-west-1.amazonaws.com/blob-for-upload-download", + "Header": { + "Accept-Encoding": [ + "gzip" + ], + "Content-Length": [ + "11" + ], + "Content-Md5": [ + "G3VTPtWoaf9vOuAzbQwzIA==" + ], + "User-Agent": [ + "CLEARED" + ], + "X-Amz-Content-Sha256": [ + "b382216a0e762fbfadcc9ed316fad1b2d68d1fda1803b953466656af9248c2b1" + ], + "X-Amz-Date": [ + "CLEARED" + ] + }, + "MediaType": "text", + "BodyParts": [ + "dXAgYW5kIGRvd24=" + ] + }, + "Response": { + "StatusCode": 200, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Content-Length": [ + "0" + ], + "Date": [ + "Tue, 09 May 2023 05:32:53 GMT" + ], + "Etag": [ + "\"1b75533ed5a869ff6f3ae0336d0c3320\"" + ], + "Server": [ + "AmazonS3" + ], + "X-Amz-Id-2": [ + "I2qPyy2SZKIFWX1plcheoIw+DT9VpW0s08eIjeHcg0o2zfUXf7P+/Mh84k4TISxAvw9HkChDsC1KYyynSFBv5A==" + ], + "X-Amz-Request-Id": [ + "SJRAENGY0VSX0K2V" + ], + "X-Amz-Server-Side-Encryption": [ + "AES256" + ] + }, + "Body": "" + } + }, + { + "ID": "ee1f8f13e160303c", + "Request": { + "Method": "GET", + "URL": "https://go-cloud-testing.s3.us-west-1.amazonaws.com/blob-for-upload-download", + "Header": { + "Accept-Encoding": [ + "gzip" + ], + "User-Agent": [ + "CLEARED" + ], + "X-Amz-Content-Sha256": [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ], + "X-Amz-Date": [ + "CLEARED" + ] + }, + "MediaType": "", + "BodyParts": [ + "" + ] + }, + "Response": { + "StatusCode": 200, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Accept-Ranges": [ + "bytes" + ], + "Content-Length": [ + "11" + ], + "Content-Type": [ + "text" + ], + "Date": [ + "Tue, 09 May 2023 05:32:53 GMT" + ], + "Etag": [ + "\"1b75533ed5a869ff6f3ae0336d0c3320\"" + ], + "Last-Modified": [ + "Tue, 09 May 2023 05:32:53 GMT" + ], + "Server": [ + "AmazonS3" + ], + "X-Amz-Id-2": [ + "4HQraBaU60CzXdc4hpRusn1ujAhv5yKyYSaLS+Jra1ocno0Eqbe781MhhfHfwbxcooef0y0Uu6dXjKuVXP7hRw==" + ], + "X-Amz-Request-Id": [ + "SJR9H78BTWTC58PY" + ], + "X-Amz-Server-Side-Encryption": [ + "AES256" + ] + }, + "Body": "dXAgYW5kIGRvd24=" + } + }, + { + "ID": "8d48f07a282a2a3a", + "Request": { + "Method": "HEAD", + "URL": "https://go-cloud-testing.s3.us-west-1.amazonaws.com/blob-for-upload-download", + "Header": { + "User-Agent": [ + "CLEARED" + ], + "X-Amz-Content-Sha256": [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ], + "X-Amz-Date": [ + "CLEARED" + ] + }, + "MediaType": "", + "BodyParts": [ + "" + ] + }, + "Response": { + "StatusCode": 200, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Accept-Ranges": [ + "bytes" + ], + "Content-Length": [ + "11" + ], + "Content-Type": [ + "text" + ], + "Date": [ + "Tue, 09 May 2023 05:32:53 GMT" + ], + "Etag": [ + "\"1b75533ed5a869ff6f3ae0336d0c3320\"" + ], + "Last-Modified": [ + "Tue, 09 May 2023 05:32:53 GMT" + ], + "Server": [ + "AmazonS3" + ], + "X-Amz-Id-2": [ + "evHBH1QQ5ThOps7WC5zGp3B2cIimSsWC36xo4/+EvayiylYBgBL6VdXdE3xB6atiIe1Qca6bjdgWjdnmvrf01Q==" + ], + "X-Amz-Request-Id": [ + "SJR7RZ10RV2HR61Z" + ], + "X-Amz-Server-Side-Encryption": [ + "AES256" + ] + }, + "Body": "" + } + }, + { + "ID": "149177a7f0ef5ee3", + "Request": { + "Method": "DELETE", + "URL": "https://go-cloud-testing.s3.us-west-1.amazonaws.com/blob-for-upload-download", + "Header": { + "Accept-Encoding": [ + "gzip" + ], + "User-Agent": [ + "CLEARED" + ], + "X-Amz-Content-Sha256": [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ], + "X-Amz-Date": [ + "CLEARED" + ] + }, + "MediaType": "", + "BodyParts": [ + "" + ] + }, + "Response": { + "StatusCode": 204, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Date": [ + "Tue, 09 May 2023 05:32:53 GMT" + ], + "Server": [ + "AmazonS3" + ], + "X-Amz-Id-2": [ + "OHKdJZD32Gk4iLGl+KW++wRNawTN6R4Y0kUsyu07O6o+W7b5wdh7loubwl0RC+OOUbM6NPkSy40xPuVuYi+how==" + ], + "X-Amz-Request-Id": [ + "SJRA3BV8Y9MMHCTC" + ] + }, + "Body": "" + } + } + ] +} \ No newline at end of file diff --git a/blob/s3blob/testdata/TestConformanceUsingLegacyList/TestUploadDownload.replay b/blob/s3blob/testdata/TestConformanceUsingLegacyList/TestUploadDownload.replay new file mode 100644 index 0000000000..775b103c99 --- /dev/null +++ b/blob/s3blob/testdata/TestConformanceUsingLegacyList/TestUploadDownload.replay @@ -0,0 +1,276 @@ +{ + "Initial": "AQAAAA7b69IEM2HXWv5c", + "Version": "0.2", + "Converter": { + "ScrubBody": null, + "ClearHeaders": [ + "^X-Goog-.*Encryption-Key$", + "^X-Amz-Date$", + "^User-Agent$" + ], + "RemoveRequestHeaders": [ + "^Authorization$", + "^Proxy-Authorization$", + "^Connection$", + "^Content-Type$", + "^Date$", + "^Host$", + "^Transfer-Encoding$", + "^Via$", + "^X-Forwarded-.*$", + "^X-Cloud-Trace-Context$", + "^X-Goog-Api-Client$", + "^X-Google-.*$", + "^X-Gfe-.*$", + "^Authorization$", + "^Duration$", + "^X-Amz-Security-Token$" + ], + "RemoveResponseHeaders": [ + "^X-Google-.*$", + "^X-Gfe-.*$" + ], + "ClearParams": [ + "^X-Amz-Date$" + ], + "RemoveParams": [ + "^X-Amz-Credential$", + "^X-Amz-Signature$", + "^X-Amz-Security-Token$" + ] + }, + "Entries": [ + { + "ID": "a0162e47bbe2afac", + "Request": { + "Method": "PUT", + "URL": "https://go-cloud-testing.s3.us-west-1.amazonaws.com/blob-for-upload-download", + "Header": { + "Accept-Encoding": [ + "gzip" + ], + "Content-Length": [ + "11" + ], + "Content-Md5": [ + "G3VTPtWoaf9vOuAzbQwzIA==" + ], + "User-Agent": [ + "CLEARED" + ], + "X-Amz-Content-Sha256": [ + "b382216a0e762fbfadcc9ed316fad1b2d68d1fda1803b953466656af9248c2b1" + ], + "X-Amz-Date": [ + "CLEARED" + ] + }, + "MediaType": "text", + "BodyParts": [ + "dXAgYW5kIGRvd24=" + ] + }, + "Response": { + "StatusCode": 200, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Content-Length": [ + "0" + ], + "Date": [ + "Tue, 09 May 2023 05:32:53 GMT" + ], + "Etag": [ + "\"1b75533ed5a869ff6f3ae0336d0c3320\"" + ], + "Server": [ + "AmazonS3" + ], + "X-Amz-Id-2": [ + "U4KhpI2gzhhNG8KAmkMMrC5ey1hxUb9cG3Z8WSvRK33UgpfR9l3yQt0++Z2qSlOTAPrjFvBD6dpCQs9PIPnvLg==" + ], + "X-Amz-Request-Id": [ + "SJR1G65AE17BET6B" + ], + "X-Amz-Server-Side-Encryption": [ + "AES256" + ] + }, + "Body": "" + } + }, + { + "ID": "3b71dbb32eb867c7", + "Request": { + "Method": "GET", + "URL": "https://go-cloud-testing.s3.us-west-1.amazonaws.com/blob-for-upload-download", + "Header": { + "Accept-Encoding": [ + "gzip" + ], + "User-Agent": [ + "CLEARED" + ], + "X-Amz-Content-Sha256": [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ], + "X-Amz-Date": [ + "CLEARED" + ] + }, + "MediaType": "", + "BodyParts": [ + "" + ] + }, + "Response": { + "StatusCode": 200, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Accept-Ranges": [ + "bytes" + ], + "Content-Length": [ + "11" + ], + "Content-Type": [ + "text" + ], + "Date": [ + "Tue, 09 May 2023 05:32:53 GMT" + ], + "Etag": [ + "\"1b75533ed5a869ff6f3ae0336d0c3320\"" + ], + "Last-Modified": [ + "Tue, 09 May 2023 05:32:53 GMT" + ], + "Server": [ + "AmazonS3" + ], + "X-Amz-Id-2": [ + "cndBRVkLqM3ZOJ1FnWzmCcfBbqh5XKhgZqPstvUbaBI89ZQUEET2hygZbAEK9krUzPJMoP1kEJfKujncv9veMg==" + ], + "X-Amz-Request-Id": [ + "SJRDJC94EDS05GSY" + ], + "X-Amz-Server-Side-Encryption": [ + "AES256" + ] + }, + "Body": "dXAgYW5kIGRvd24=" + } + }, + { + "ID": "5698097e2f92a480", + "Request": { + "Method": "HEAD", + "URL": "https://go-cloud-testing.s3.us-west-1.amazonaws.com/blob-for-upload-download", + "Header": { + "User-Agent": [ + "CLEARED" + ], + "X-Amz-Content-Sha256": [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ], + "X-Amz-Date": [ + "CLEARED" + ] + }, + "MediaType": "", + "BodyParts": [ + "" + ] + }, + "Response": { + "StatusCode": 200, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Accept-Ranges": [ + "bytes" + ], + "Content-Length": [ + "11" + ], + "Content-Type": [ + "text" + ], + "Date": [ + "Tue, 09 May 2023 05:32:53 GMT" + ], + "Etag": [ + "\"1b75533ed5a869ff6f3ae0336d0c3320\"" + ], + "Last-Modified": [ + "Tue, 09 May 2023 05:32:53 GMT" + ], + "Server": [ + "AmazonS3" + ], + "X-Amz-Id-2": [ + "XlG6+DMD5BUQ1iFMj6/Dz8x671E2au6xytESpeMoIj7mZin/cTNfGxWLVoGTuIUOeL9IYFKL5UX4pj6gAzSFxQ==" + ], + "X-Amz-Request-Id": [ + "SJR2VJEP3V0K0C0C" + ], + "X-Amz-Server-Side-Encryption": [ + "AES256" + ] + }, + "Body": "" + } + }, + { + "ID": "45970372a8e665f1", + "Request": { + "Method": "DELETE", + "URL": "https://go-cloud-testing.s3.us-west-1.amazonaws.com/blob-for-upload-download", + "Header": { + "Accept-Encoding": [ + "gzip" + ], + "User-Agent": [ + "CLEARED" + ], + "X-Amz-Content-Sha256": [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ], + "X-Amz-Date": [ + "CLEARED" + ] + }, + "MediaType": "", + "BodyParts": [ + "" + ] + }, + "Response": { + "StatusCode": 204, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Date": [ + "Tue, 09 May 2023 05:32:53 GMT" + ], + "Server": [ + "AmazonS3" + ], + "X-Amz-Id-2": [ + "DnufQKev+/2kKMWkFErcPIx3AxTD01jcMF408V6SdO5gl16n9KncSMNleScZoon1Xl7BPbokD/awvoGitGJq4w==" + ], + "X-Amz-Request-Id": [ + "SJR80KF5KR1F12KW" + ] + }, + "Body": "" + } + } + ] +} \ No newline at end of file diff --git a/blob/s3blob/testdata/TestConformanceUsingLegacyListV2/TestUploadDownload.replay b/blob/s3blob/testdata/TestConformanceUsingLegacyListV2/TestUploadDownload.replay new file mode 100644 index 0000000000..e56df1345d --- /dev/null +++ b/blob/s3blob/testdata/TestConformanceUsingLegacyListV2/TestUploadDownload.replay @@ -0,0 +1,301 @@ +{ + "Initial": "AQAAAA7b69IFBhVFDf5c", + "Version": "0.2", + "Converter": { + "ScrubBody": null, + "ClearHeaders": [ + "^X-Goog-.*Encryption-Key$", + "^Amz-Sdk-Invocation-Id$", + "^X-Amz-Date$", + "^User-Agent$" + ], + "RemoveRequestHeaders": [ + "^Authorization$", + "^Proxy-Authorization$", + "^Connection$", + "^Content-Type$", + "^Date$", + "^Host$", + "^Transfer-Encoding$", + "^Via$", + "^X-Forwarded-.*$", + "^X-Cloud-Trace-Context$", + "^X-Goog-Api-Client$", + "^X-Google-.*$", + "^X-Gfe-.*$", + "^Authorization$", + "^Duration$", + "^X-Amz-Security-Token$" + ], + "RemoveResponseHeaders": [ + "^X-Google-.*$", + "^X-Gfe-.*$" + ], + "ClearParams": [ + "^X-Amz-Date$" + ], + "RemoveParams": [ + "^X-Amz-Credential$", + "^X-Amz-Signature$", + "^X-Amz-Security-Token$" + ] + }, + "Entries": [ + { + "ID": "82171b2e0948e058", + "Request": { + "Method": "PUT", + "URL": "https://go-cloud-testing.s3.us-west-1.amazonaws.com/blob-for-upload-download?x-id=PutObject", + "Header": { + "Accept-Encoding": [ + "identity" + ], + "Amz-Sdk-Invocation-Id": [ + "CLEARED" + ], + "Amz-Sdk-Request": [ + "attempt=1; max=1" + ], + "Content-Length": [ + "11" + ], + "User-Agent": [ + "CLEARED" + ], + "X-Amz-Content-Sha256": [ + "UNSIGNED-PAYLOAD" + ], + "X-Amz-Date": [ + "CLEARED" + ] + }, + "MediaType": "text", + "BodyParts": [ + "dXAgYW5kIGRvd24=" + ] + }, + "Response": { + "StatusCode": 200, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Content-Length": [ + "0" + ], + "Date": [ + "Tue, 09 May 2023 05:32:54 GMT" + ], + "Etag": [ + "\"1b75533ed5a869ff6f3ae0336d0c3320\"" + ], + "Server": [ + "AmazonS3" + ], + "X-Amz-Id-2": [ + "QLcL/NQjHeBMxYfXpZY5xfFBDxyuY2yjzrJFYaXBWmaEwXtPqpuLxiW9oyt51UUKqH0aFpN4Vl4njOu8a0mq3Q==" + ], + "X-Amz-Request-Id": [ + "JXS066BT16A1M5JY" + ], + "X-Amz-Server-Side-Encryption": [ + "AES256" + ] + }, + "Body": "" + } + }, + { + "ID": "74d3954409de25ab", + "Request": { + "Method": "GET", + "URL": "https://go-cloud-testing.s3.us-west-1.amazonaws.com/blob-for-upload-download?x-id=GetObject", + "Header": { + "Accept-Encoding": [ + "identity" + ], + "Amz-Sdk-Invocation-Id": [ + "CLEARED" + ], + "Amz-Sdk-Request": [ + "attempt=1; max=1" + ], + "User-Agent": [ + "CLEARED" + ], + "X-Amz-Content-Sha256": [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ], + "X-Amz-Date": [ + "CLEARED" + ] + }, + "MediaType": "", + "BodyParts": [ + "" + ] + }, + "Response": { + "StatusCode": 200, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Accept-Ranges": [ + "bytes" + ], + "Content-Length": [ + "11" + ], + "Content-Type": [ + "text" + ], + "Date": [ + "Tue, 09 May 2023 05:32:54 GMT" + ], + "Etag": [ + "\"1b75533ed5a869ff6f3ae0336d0c3320\"" + ], + "Last-Modified": [ + "Tue, 09 May 2023 05:32:54 GMT" + ], + "Server": [ + "AmazonS3" + ], + "X-Amz-Id-2": [ + "VtyHNAmK3++s6Ed3QrKldOCR2oISntuz/76fCS/6W86s7oizQoVNjcFwNYmn5QgwZh0FEhVuLgwIxEOHCw8FBA==" + ], + "X-Amz-Request-Id": [ + "JXSEEKF2E1SG2437" + ], + "X-Amz-Server-Side-Encryption": [ + "AES256" + ] + }, + "Body": "dXAgYW5kIGRvd24=" + } + }, + { + "ID": "855599a2173e236f", + "Request": { + "Method": "HEAD", + "URL": "https://go-cloud-testing.s3.us-west-1.amazonaws.com/blob-for-upload-download", + "Header": { + "Accept-Encoding": [ + "identity" + ], + "Amz-Sdk-Invocation-Id": [ + "CLEARED" + ], + "Amz-Sdk-Request": [ + "attempt=1; max=1" + ], + "User-Agent": [ + "CLEARED" + ], + "X-Amz-Content-Sha256": [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ], + "X-Amz-Date": [ + "CLEARED" + ] + }, + "MediaType": "", + "BodyParts": [ + "" + ] + }, + "Response": { + "StatusCode": 200, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Accept-Ranges": [ + "bytes" + ], + "Content-Length": [ + "11" + ], + "Content-Type": [ + "text" + ], + "Date": [ + "Tue, 09 May 2023 05:32:54 GMT" + ], + "Etag": [ + "\"1b75533ed5a869ff6f3ae0336d0c3320\"" + ], + "Last-Modified": [ + "Tue, 09 May 2023 05:32:54 GMT" + ], + "Server": [ + "AmazonS3" + ], + "X-Amz-Id-2": [ + "H5vYkDifqsGXyiJXe3Sr9deRpHvOQShU0CVRXEwq2jj4jrio2cs4q0erqEZcStXuvN9sr6Ct1EUcAGFDH0nx6g==" + ], + "X-Amz-Request-Id": [ + "JXS1GH82KQ96QQP1" + ], + "X-Amz-Server-Side-Encryption": [ + "AES256" + ] + }, + "Body": "" + } + }, + { + "ID": "e9714b50dd6e05de", + "Request": { + "Method": "DELETE", + "URL": "https://go-cloud-testing.s3.us-west-1.amazonaws.com/blob-for-upload-download?x-id=DeleteObject", + "Header": { + "Accept-Encoding": [ + "identity" + ], + "Amz-Sdk-Invocation-Id": [ + "CLEARED" + ], + "Amz-Sdk-Request": [ + "attempt=1; max=1" + ], + "User-Agent": [ + "CLEARED" + ], + "X-Amz-Content-Sha256": [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ], + "X-Amz-Date": [ + "CLEARED" + ] + }, + "MediaType": "", + "BodyParts": [ + "" + ] + }, + "Response": { + "StatusCode": 204, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Date": [ + "Tue, 09 May 2023 05:32:54 GMT" + ], + "Server": [ + "AmazonS3" + ], + "X-Amz-Id-2": [ + "OIx+3wF1qGZDs7Y05dJIY8WL+FDQxDpJ4funEx0n0DVIz9/LD6C8rzsY3BB+Exdg30jWWqBDm56D0ZEQIyEaiw==" + ], + "X-Amz-Request-Id": [ + "JXS9DGBNP797PACX" + ] + }, + "Body": "" + } + } + ] +} \ No newline at end of file diff --git a/blob/s3blob/testdata/TestConformanceV2/TestUploadDownload.replay b/blob/s3blob/testdata/TestConformanceV2/TestUploadDownload.replay new file mode 100644 index 0000000000..11aaeb9c93 --- /dev/null +++ b/blob/s3blob/testdata/TestConformanceV2/TestUploadDownload.replay @@ -0,0 +1,301 @@ +{ + "Initial": "AQAAAA7b69IEOnaeT/5c", + "Version": "0.2", + "Converter": { + "ScrubBody": null, + "ClearHeaders": [ + "^X-Goog-.*Encryption-Key$", + "^Amz-Sdk-Invocation-Id$", + "^X-Amz-Date$", + "^User-Agent$" + ], + "RemoveRequestHeaders": [ + "^Authorization$", + "^Proxy-Authorization$", + "^Connection$", + "^Content-Type$", + "^Date$", + "^Host$", + "^Transfer-Encoding$", + "^Via$", + "^X-Forwarded-.*$", + "^X-Cloud-Trace-Context$", + "^X-Goog-Api-Client$", + "^X-Google-.*$", + "^X-Gfe-.*$", + "^Authorization$", + "^Duration$", + "^X-Amz-Security-Token$" + ], + "RemoveResponseHeaders": [ + "^X-Google-.*$", + "^X-Gfe-.*$" + ], + "ClearParams": [ + "^X-Amz-Date$" + ], + "RemoveParams": [ + "^X-Amz-Credential$", + "^X-Amz-Signature$", + "^X-Amz-Security-Token$" + ] + }, + "Entries": [ + { + "ID": "e3c208df14575ffc", + "Request": { + "Method": "PUT", + "URL": "https://go-cloud-testing.s3.us-west-1.amazonaws.com/blob-for-upload-download?x-id=PutObject", + "Header": { + "Accept-Encoding": [ + "identity" + ], + "Amz-Sdk-Invocation-Id": [ + "CLEARED" + ], + "Amz-Sdk-Request": [ + "attempt=1; max=1" + ], + "Content-Length": [ + "11" + ], + "User-Agent": [ + "CLEARED" + ], + "X-Amz-Content-Sha256": [ + "UNSIGNED-PAYLOAD" + ], + "X-Amz-Date": [ + "CLEARED" + ] + }, + "MediaType": "text", + "BodyParts": [ + "dXAgYW5kIGRvd24=" + ] + }, + "Response": { + "StatusCode": 200, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Content-Length": [ + "0" + ], + "Date": [ + "Tue, 09 May 2023 05:32:54 GMT" + ], + "Etag": [ + "\"1b75533ed5a869ff6f3ae0336d0c3320\"" + ], + "Server": [ + "AmazonS3" + ], + "X-Amz-Id-2": [ + "IOHwaad93pXaQ7NOPRLmJUuRnQmiIvaS9ODlAagCxDnUqk32symdYfauk3aeTAoVAneS/nl3zcz6QlwjIQlZeg==" + ], + "X-Amz-Request-Id": [ + "JXS0WDN1J2ZXDK3E" + ], + "X-Amz-Server-Side-Encryption": [ + "AES256" + ] + }, + "Body": "" + } + }, + { + "ID": "6b8472f00686ce38", + "Request": { + "Method": "GET", + "URL": "https://go-cloud-testing.s3.us-west-1.amazonaws.com/blob-for-upload-download?x-id=GetObject", + "Header": { + "Accept-Encoding": [ + "identity" + ], + "Amz-Sdk-Invocation-Id": [ + "CLEARED" + ], + "Amz-Sdk-Request": [ + "attempt=1; max=1" + ], + "User-Agent": [ + "CLEARED" + ], + "X-Amz-Content-Sha256": [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ], + "X-Amz-Date": [ + "CLEARED" + ] + }, + "MediaType": "", + "BodyParts": [ + "" + ] + }, + "Response": { + "StatusCode": 200, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Accept-Ranges": [ + "bytes" + ], + "Content-Length": [ + "11" + ], + "Content-Type": [ + "text" + ], + "Date": [ + "Tue, 09 May 2023 05:32:54 GMT" + ], + "Etag": [ + "\"1b75533ed5a869ff6f3ae0336d0c3320\"" + ], + "Last-Modified": [ + "Tue, 09 May 2023 05:32:54 GMT" + ], + "Server": [ + "AmazonS3" + ], + "X-Amz-Id-2": [ + "zEnSKHePmq18xdiilHb0JgAh2o+A6h9D3NqGMyoz7S9ha8x3zmSa0BBt8Fm8RftnLDrnhizOcI0TtipVjwwAgg==" + ], + "X-Amz-Request-Id": [ + "JXS4Z3ZWHFS3A651" + ], + "X-Amz-Server-Side-Encryption": [ + "AES256" + ] + }, + "Body": "dXAgYW5kIGRvd24=" + } + }, + { + "ID": "0fd92ce902c3a1da", + "Request": { + "Method": "HEAD", + "URL": "https://go-cloud-testing.s3.us-west-1.amazonaws.com/blob-for-upload-download", + "Header": { + "Accept-Encoding": [ + "identity" + ], + "Amz-Sdk-Invocation-Id": [ + "CLEARED" + ], + "Amz-Sdk-Request": [ + "attempt=1; max=1" + ], + "User-Agent": [ + "CLEARED" + ], + "X-Amz-Content-Sha256": [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ], + "X-Amz-Date": [ + "CLEARED" + ] + }, + "MediaType": "", + "BodyParts": [ + "" + ] + }, + "Response": { + "StatusCode": 200, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Accept-Ranges": [ + "bytes" + ], + "Content-Length": [ + "11" + ], + "Content-Type": [ + "text" + ], + "Date": [ + "Tue, 09 May 2023 05:32:54 GMT" + ], + "Etag": [ + "\"1b75533ed5a869ff6f3ae0336d0c3320\"" + ], + "Last-Modified": [ + "Tue, 09 May 2023 05:32:54 GMT" + ], + "Server": [ + "AmazonS3" + ], + "X-Amz-Id-2": [ + "O2LVpW2yqgMriF98DSbMXvzUMW2iST/JLX1Wb6VFGJELG7R8IyB2pBYvaPxA8448rVV61maNVcAIcYwUIBxveA==" + ], + "X-Amz-Request-Id": [ + "JXS3PG0NSEQWGEM6" + ], + "X-Amz-Server-Side-Encryption": [ + "AES256" + ] + }, + "Body": "" + } + }, + { + "ID": "bff9b209f1f871c6", + "Request": { + "Method": "DELETE", + "URL": "https://go-cloud-testing.s3.us-west-1.amazonaws.com/blob-for-upload-download?x-id=DeleteObject", + "Header": { + "Accept-Encoding": [ + "identity" + ], + "Amz-Sdk-Invocation-Id": [ + "CLEARED" + ], + "Amz-Sdk-Request": [ + "attempt=1; max=1" + ], + "User-Agent": [ + "CLEARED" + ], + "X-Amz-Content-Sha256": [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ], + "X-Amz-Date": [ + "CLEARED" + ] + }, + "MediaType": "", + "BodyParts": [ + "" + ] + }, + "Response": { + "StatusCode": 204, + "Proto": "HTTP/1.1", + "ProtoMajor": 1, + "ProtoMinor": 1, + "Header": { + "Date": [ + "Tue, 09 May 2023 05:32:54 GMT" + ], + "Server": [ + "AmazonS3" + ], + "X-Amz-Id-2": [ + "6W7ftALe0P9KMQAa5Xq7KU/WN3+/GYg7LLV6qOLShfwD0+3qUOPTo4pYWxkkybXZMQX+MNjfa5DJvCMDxCskdQ==" + ], + "X-Amz-Request-Id": [ + "JXS5Z9R0J1VQX63J" + ] + }, + "Body": "" + } + } + ] +} \ No newline at end of file