feat: Provide metadata for the object that has been downloaded

Fixes #7897

apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs

@@ -50,6 +50,44 @@ public async Task SimpleDownload()
             }
         }
 
+        [Fact]
+        public async Task MetadataReturned_Simple()
+        {
+            var expected = await _fixture.Client.GetObjectAsync(_fixture.ReadBucket, _fixture.SmallObject);
+            var actual = await _fixture.Client.DownloadObjectAsync(_fixture.ReadBucket, _fixture.SmallObject, Stream.Null);
+
+            // Just test the values we expect to be populated
+            Assert.Equal(expected.Name, actual.Name);
+            Assert.Equal(expected.Bucket, actual.Bucket);
+            Assert.Equal(expected.Generation, actual.Generation);
+            Assert.Equal(expected.Metageneration, actual.Metageneration);
+            Assert.Equal(expected.ETag, actual.ETag);
+            Assert.Equal(expected.Crc32c, actual.Crc32c);
+            Assert.Equal(expected.Md5Hash, actual.Md5Hash);
+            Assert.Equal(expected.ContentType, actual.ContentType);
+        }
+
+        [Fact]
+        public void MetadataReturned_WithRange()
+        {
+            var expected = _fixture.Client.GetObject(_fixture.ReadBucket, _fixture.SmallObject);
+            var options = new DownloadObjectOptions { Range = new RangeHeaderValue(0, 3) };
+            var actual = _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, Stream.Null, options);
+
+            // The CRC32C hash will be just for the range downloaded
+            Assert.NotEqual(expected.Crc32c, actual.Crc32c);
+            // The MD5 won't be populated at all
+            Assert.Null(actual.Md5Hash);
+
+            // Simple values are still populated.
+            Assert.Equal(expected.Name, actual.Name);
+            Assert.Equal(expected.Bucket, actual.Bucket);
+            Assert.Equal(expected.Generation, actual.Generation);
+            Assert.Equal(expected.Metageneration, actual.Metageneration);
+            Assert.Equal(expected.ETag, actual.ETag);
+            Assert.Equal(expected.ContentType, actual.ContentType);
+        }
+
         [Fact]
         public void WrongObjectName() => ValidateNotFound(_fixture.ReadBucket, "doesntexist");
 

apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/CodeHealthTest.cs

@@ -28,7 +28,8 @@ public void PrivateFields()
         [Fact]
         public void SealedClasses()
         {
-            CodeHealthTester.AssertClassesAreSealedOrAbstract(typeof(StorageClient));
+            CodeHealthTester.AssertClassesAreSealedOrAbstract(
+                typeof(StorageClient), new[] { typeof(ContentMetadataRecordingMediaDownloader) });
         }
     }
 }

apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/HashValidatingDownloaderTest.cs

@@ -72,7 +72,8 @@ public void Invalid()
         private static HashValidatingDownloader CreateDownloader(Func<HttpRequestMessage, HttpResponseMessage> handler)
         {
             var service = new MockableService(handler);
-            return new HashValidatingDownloader(service);
+            var metadata = new Apis.Storage.v1.Data.Object();
+            return new HashValidatingDownloader(metadata, service);
         }
 
         class MockableService : BaseClientService

apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/HashValidatingDownloader.cs

@@ -15,7 +15,7 @@
 using Google.Apis.Download;
 using Google.Apis.Http;
 using Google.Apis.Services;
-using System;
+using Google.Apis.Storage.v1.Data;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
@@ -27,13 +27,13 @@ namespace Google.Cloud.Storage.V1
     /// Subclass of <see cref="MediaDownloader"/> which validates the data it receives
     /// against a CRC32c hash set in the header.
     /// </summary>
-    internal sealed class HashValidatingDownloader : MediaDownloader
+    internal sealed class HashValidatingDownloader : ContentMetadataRecordingMediaDownloader
     {
         private string crc32cHashBase64;
         private Crc32c hasher;
 
         /// <summary>Constructs a new downloader with the given client service.</summary>
-        internal HashValidatingDownloader(IClientService service) : base(service)
+        internal HashValidatingDownloader(Object metadata, IClientService service) : base(metadata, service)
         {
             ResponseStreamInterceptorProvider = CreateInterceptor;
         }
@@ -66,7 +66,7 @@ protected override void OnDownloadCompleted()
 
             if (crc32cHashBase64 != null)
             {
-                string actualHash = Convert.ToBase64String(hasher.GetHash());
+                string actualHash = System.Convert.ToBase64String(hasher.GetHash());
                 if (actualHash != crc32cHashBase64)
                 {
                     throw new IOException($"Incorrect hash: expected '{crc32cHashBase64}' (base64), was '{actualHash}' (base64)");

apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/MetadataRecordingMediaDownloader.cs

@@ -0,0 +1,83 @@
+// Copyright 2022 Google LLC
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     https://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using Google.Apis.Download;
+using Google.Apis.Services;
+using Google.Apis.Storage.v1.Data;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Net.Http;
+using System.Net.Http.Headers;
+
+namespace Google.Cloud.Storage.V1;
+
+/// <summary>
+/// MediaDownloader subclass which populates an <see cref="Object"/> instance
+/// from the headers it sees on a successful response. Note: if this ever becomes public,
+/// we should put more effort into naming it carefully.
+/// </summary>
+internal class ContentMetadataRecordingMediaDownloader : MediaDownloader
+{
+    private const string ETagHeader = "ETag";
+    private const string GenerationHeader = "X-Goog-Generation";
+    private const string MetagenerationHeader = "X-Goog-Metageneration";
+    private const string HashHeader = "X-Goog-Hash";
+    private const string ContentTypeHeader = "Content-Type";
+
+    // The hashes are comma-separated...
+    private static readonly char[] HashToElementSplitter = new[] { ',' };
+    // ... and each hash is a key=value pair
+    private static readonly char[] HashKeyValueSplitter = new[] { '=' };
+
+    private readonly Object metadata;
+
+    /// <summary>Constructs a new downloader with the given client service.</summary>
+    /// <param name="metadata">The object in which to record metadata.</param>
+    /// <param name="service">The client service.</param>
+    internal ContentMetadataRecordingMediaDownloader(Object metadata, IClientService service) : base(service)
+    {
+        this.metadata = metadata;
+    }
+
+    protected override void OnResponseReceived(HttpResponseMessage response)
+    {
+        base.OnResponseReceived(response);
+        ProcessMetadataHeaders(response.Headers, response.Content.Headers);
+    }
+
+    private void ProcessMetadataHeaders(HttpResponseHeaders headers, HttpContentHeaders contentHeaders)
+    {
+        metadata.Generation = MaybeParse(GetFirstHeaderOrNull(GenerationHeader));
+        metadata.Metageneration = MaybeParse(GetFirstHeaderOrNull(MetagenerationHeader));
+        metadata.ETag = GetFirstHeaderOrNull(ETagHeader);
+        var hashes = GetFirstHeaderOrNull(HashHeader) ?? "";
+        // The hash header returns multiple comma-separated hashes.
+        var hashesByKey = hashes.Split(HashToElementSplitter)
+            .Where(hash => hash.Contains('='))
+            .Select(hash => hash.Split(HashKeyValueSplitter, 2))
+            .ToDictionary(bits => bits[0], bits => bits[1]);
+        metadata.Crc32c = hashesByKey.TryGetValue("crc32c", out string crc32c) ? crc32c : null;
+        metadata.Md5Hash = hashesByKey.TryGetValue("md5", out string md5) ? md5 : null;
+        metadata.ContentType = contentHeaders.ContentType?.ToString();
+
+        string GetFirstHeaderOrNull(string headerName) =>
+            headers.TryGetValues(headerName, out var values) ? values.FirstOrDefault() : null;
+
+        long? MaybeParse(string text) =>
+            text is null || !long.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out var value)
+            ? (long?) null
+            : value;
+    }
+}

apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClient.DownloadObject.cs

@@ -32,7 +32,12 @@ public abstract partial class StorageClient
         /// <param name="options">Additional options for the download. May be null, in which case appropriate
         /// defaults will be used.</param>
         /// <param name="progress">Progress reporter for the download. May be null.</param>
-        public virtual void DownloadObject(
+        /// <returns>An <see cref="Object"/> representation of the metadata for the object that has been downloaded
+        /// into the stream. This metadata is not the complete metadata for the object; it's just the information
+        /// provided in headers while downloading. Additionally, the CRC32C hash is only the hash of the data downloaded;
+        /// if the options specify a range which does not encompass the whole object, this will not be the same
+        /// as the CRC32C hash of the complete object.</returns>
+        public virtual Object DownloadObject(
             string bucket,
             string objectName,
             Stream destination,
@@ -52,8 +57,13 @@ public abstract partial class StorageClient
         /// defaults will be used.</param>
         /// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
         /// <param name="progress">Progress reporter for the download. May be null.</param>
-        /// <returns>A task representing the asynchronous operation.</returns>
-        public virtual Task DownloadObjectAsync(
+        /// <returns>A task representing the asynchronous operation. The result of the task is
+        /// an <see cref="Object"/> representation of the metadata for the object that has been downloaded
+        /// into the stream. This metadata is not the complete metadata for the object; it's just the information
+        /// provided in headers while downloading. Additionally, the CRC32C hash is only the hash of the data downloaded;
+        /// if the options specify a range which does not encompass the whole object, this will not be the same
+        /// as the CRC32C hash of the complete object.</returns>
+        public virtual Task<Object> DownloadObjectAsync(
             string bucket,
             string objectName,
             Stream destination,
@@ -75,7 +85,12 @@ public abstract partial class StorageClient
         /// <param name="options">Additional options for the download. May be null, in which case appropriate
         /// defaults will be used.</param>
         /// <param name="progress">Progress reporter for the download. May be null.</param>
-        public virtual void DownloadObject(
+        /// <returns>An <see cref="Object"/> representation of the metadata for the object that has been downloaded
+        /// into the stream. This metadata is not the complete metadata for the object; it's just the information
+        /// provided in headers while downloading. Additionally, the CRC32C hash is only the hash of the data downloaded;
+        /// if the options specify a range which does not encompass the whole object, this will not be the same
+        /// as the CRC32C hash of the complete object.</returns>
+        public virtual Object DownloadObject(
             Object source,
             Stream destination,
             DownloadObjectOptions options = null,
@@ -96,8 +111,13 @@ public abstract partial class StorageClient
         /// defaults will be used.</param>
         /// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
         /// <param name="progress">Progress reporter for the download. May be null.</param>
-        /// <returns>A task representing the asynchronous operation.</returns>
-        public virtual Task DownloadObjectAsync(
+        /// <returns>A task representing the asynchronous operation. The result of the task is
+        /// an <see cref="Object"/> representation of the metadata for the object that has been downloaded
+        /// into the stream. This metadata is not the complete metadata for the object; it's just the information
+        /// provided in headers while downloading. Additionally, the CRC32C hash is only the hash of the data downloaded;
+        /// if the options specify a range which does not encompass the whole object, this will not be the same
+        /// as the CRC32C hash of the complete object.</returns>
+        public virtual Task<Object> DownloadObjectAsync(
             Object source,
             Stream destination,
             DownloadObjectOptions options = null,

apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.DownloadObject.cs

@@ -28,19 +28,20 @@ namespace Google.Cloud.Storage.V1
     public sealed partial class StorageClientImpl : StorageClient
     {
         /// <inheritdoc />
-        public override void DownloadObject(
+        public override Object DownloadObject(
             string bucket,
             string objectName,
             Stream destination,
             DownloadObjectOptions options = null,
             IProgress<IDownloadProgress> progress = null)
         {
             var builder = CreateRequestBuilder(bucket, objectName);
-            DownloadObjectImpl(builder, destination, options, progress);
+            var metadata = new Object { Bucket = bucket, Name = objectName };
+            return DownloadObjectImpl(metadata, builder, destination, options, progress);
         }
 
         /// <inheritdoc />
-        public override Task DownloadObjectAsync(
+        public override Task<Object> DownloadObjectAsync(
             string bucket,
             string objectName,
             Stream destination,
@@ -49,30 +50,33 @@ public sealed partial class StorageClientImpl : StorageClient
             IProgress<IDownloadProgress> progress = null)
         {
             var builder = CreateRequestBuilder(bucket, objectName);
-            return DownloadObjectAsyncImpl(builder, destination, options, cancellationToken, progress);
+            var metadata = new Object { Bucket = bucket, Name = objectName };
+            return DownloadObjectAsyncImpl(metadata, builder, destination, options, cancellationToken, progress);
         }
 
         /// <inheritdoc />
-        public override void DownloadObject(
+        public override Object DownloadObject(
             Object source,
             Stream destination,
             DownloadObjectOptions options = null,
             IProgress<IDownloadProgress> progress = null)
         {
             var builder = CreateRequestBuilder(source);
-            DownloadObjectImpl(builder, destination, options, progress);
+            var metadata = new Object { Bucket = source.Bucket, Name = source.Name };
+            return DownloadObjectImpl(metadata, builder, destination, options, progress);
         }
 
         /// <inheritdoc />
-        public override Task DownloadObjectAsync(
+        public override Task<Object> DownloadObjectAsync(
             Object source,
             Stream destination,
             DownloadObjectOptions options = null,
             CancellationToken cancellationToken = default,
             IProgress<IDownloadProgress> progress = null)
         {
             var builder = CreateRequestBuilder(source);
-            return DownloadObjectAsyncImpl(builder, destination, options, cancellationToken, progress);
+            var metadata = new Object { Bucket = source.Bucket, Name = source.Name };
+            return DownloadObjectAsyncImpl(metadata, builder, destination, options, cancellationToken, progress);
         }
 
         /// <summary>
@@ -109,15 +113,16 @@ private RequestBuilder CreateRequestBuilder(Object source)
             return CreateRequestBuilder(source.Bucket, source.Name);
         }
 
-        private void DownloadObjectImpl(
+        private Object DownloadObjectImpl(
+            Object metadata,
             RequestBuilder requestBuilder,
             Stream destination,
             DownloadObjectOptions options,
             IProgress<IDownloadProgress> progress)
         {
             // URI will definitely not be null; that's constructed internally.
             GaxPreconditions.CheckNotNull(destination, nameof(destination));
-            var downloader = CreateDownloader(options);
+            var downloader = CreateDownloader(metadata, options);
             options?.ModifyRequestBuilder(requestBuilder);
             string uri = requestBuilder.BuildUri().AbsoluteUri;
             if (progress != null)
@@ -130,9 +135,12 @@ private RequestBuilder CreateRequestBuilder(Object source)
             {
                 throw result.Exception;
             }
+            // This will have been populated by the downloader.
+            return metadata;
         }
 
-        private Task DownloadObjectAsyncImpl(
+        private Task<Object> DownloadObjectAsyncImpl(
+            Object metadata,
             RequestBuilder requestBuilder,
             Stream destination,
             DownloadObjectOptions options,
@@ -143,7 +151,7 @@ private RequestBuilder CreateRequestBuilder(Object source)
             Task.Run(async () =>
             {
                 GaxPreconditions.CheckNotNull(destination, nameof(destination));
-                var downloader = CreateDownloader(options);
+                var downloader = CreateDownloader(metadata, options);
                 options?.ModifyRequestBuilder(requestBuilder);
                 string uri = requestBuilder.BuildUri().AbsoluteUri;
                 if (progress != null)
@@ -156,15 +164,17 @@ private RequestBuilder CreateRequestBuilder(Object source)
                 {
                     throw result.Exception;
                 }
+                // This will have been populated by the downloader.
+                return metadata;
             });
 
-        private MediaDownloader CreateDownloader(DownloadObjectOptions options)
+        private ContentMetadataRecordingMediaDownloader CreateDownloader(Object metadata, DownloadObjectOptions options)
         {
             DownloadValidationMode mode = options?.DownloadValidationMode ?? DownloadValidationMode.Always;
             GaxPreconditions.CheckEnumValue(mode, nameof(DownloadObjectOptions.DownloadValidationMode));
 
-            MediaDownloader downloader = mode == DownloadValidationMode.Never
-                ? new MediaDownloader(Service) : new HashValidatingDownloader(Service);
+            var downloader = mode == DownloadValidationMode.Never
+                ? new ContentMetadataRecordingMediaDownloader(metadata, Service) : new HashValidatingDownloader(metadata, Service);
             options?.ModifyDownloader(downloader);
             ApplyEncryptionKey(options?.EncryptionKey, downloader);
             return downloader;