diff --git a/Docs/API.md b/Docs/API.md index 3ff7b599d..da6563ca9 100644 --- a/Docs/API.md +++ b/Docs/API.md @@ -611,9 +611,9 @@ catch (MinioException e) ## 3. Object operations -### GetObjectAsync(string bucketName, string objectName, Action callback) +### GetObjectAsync(string bucketName, string objectName, Action callback, ServerSideEncryption sse) -`Task GetObjectAsync(string bucketName, string objectName, Action callback, CancellationToken cancellationToken = default(CancellationToken))` +`Task GetObjectAsync(string bucketName, string objectName, Action callback, ServerSideEncryption sse = null, CancellationToken cancellationToken = default(CancellationToken))` Downloads an object as a stream. @@ -626,6 +626,7 @@ __Parameters__ | ``bucketName`` | _string_ | Name of the bucket | | ``objectName`` | _string_ | Object name in the bucket | | ``callback`` | _Action_ | Call back to process stream | +| ``sse`` | _ServerSideEncryption_ | Server-side encryption option | Optional parameter. Defaults to null | | ``cancellationToken``| _System.Threading.CancellationToken_ | Optional parameter. Defaults to default(CancellationToken) | @@ -663,9 +664,9 @@ try ``` -### GetObjectAsync(string bucketName, string objectName, long offset,long length, Action callback) +### GetObjectAsync(string bucketName, string objectName, long offset,long length, Action callback, ServerSideEncryption sse) -`Task GetObjectAsync(string bucketName, string objectName, long offset, long length, Action callback, CancellationToken cancellationToken = default(CancellationToken))` +`Task GetObjectAsync(string bucketName, string objectName, long offset, long length, Action callback, ServerSideEncryption sse = null, CancellationToken cancellationToken = default(CancellationToken))` Downloads the specified range bytes of an object as a stream.Both offset and length are required. @@ -680,6 +681,7 @@ __Parameters__ | ``offset``| _long_ | Offset of the object from where stream will start | | ``length``| _long_| Length of the object to read in from the stream | | ``callback`` | _Action_ | Call back to process stream | +| ``sse`` | _ServerSideEncryption_ | Server-side encryption option | Optional parameter. Defaults to null | | ``cancellationToken``| _System.Threading.CancellationToken_ | Optional parameter. Defaults to default(CancellationToken) | @@ -717,9 +719,9 @@ try ``` -### GetObjectAsync(String bucketName, String objectName, String fileName) +### GetObjectAsync(String bucketName, String objectName, String fileName, ServerSideEncryption sse) -`Task GetObjectAsync(string bucketName, string objectName, string fileName, CancellationToken cancellationToken = default(CancellationToken))` +`Task GetObjectAsync(string bucketName, string objectName, string fileName, ServerSideEncryption sse = null, CancellationToken cancellationToken = default(CancellationToken))` Downloads and saves the object as a file in the local filesystem. @@ -732,6 +734,7 @@ __Parameters__ | ``bucketName`` | _String_ | Name of the bucket | | ``objectName`` | _String_ | Object name in the bucket | | ``fileName`` | _String_ | File name | +| ``sse`` | _ServerSideEncryption_ | Server-side encryption option | Optional parameter. Defaults to null | | ``cancellationToken``| _System.Threading.CancellationToken_ | Optional parameter. Defaults to default(CancellationToken) | @@ -763,9 +766,9 @@ catch (MinioException e) } ``` -### PutObjectAsync(string bucketName, string objectName, Stream data, long size, string contentType) +### PutObjectAsync(string bucketName, string objectName, Stream data, long size, string contentType,ServerSideEncryption sse) -` Task PutObjectAsync(string bucketName, string objectName, Stream data, long size, string contentType,Dictionary metaData=null, CancellationToken cancellationToken = default(CancellationToken))` +` Task PutObjectAsync(string bucketName, string objectName, Stream data, long size, string contentType,Dictionary metaData=null,ServerSideEncryption sse = null,CancellationToken cancellationToken = default(CancellationToken))` Uploads contents from a stream to objectName. @@ -782,6 +785,7 @@ __Parameters__ | ``size`` | _long_ | size of stream | | ``contentType`` | _string_ | Content type of the file. Defaults to "application/octet-stream" | | ``metaData`` | _Dictionary_ | Dictionary of metadata headers. Defaults to null. | +| ``sse`` | _ServerSideEncryption_ | Server-side encryption option | Optional parameter. Defaults to null | | ``cancellationToken``| _System.Threading.CancellationToken_ | Optional parameter. Defaults to default(CancellationToken) | @@ -807,12 +811,16 @@ try { byte[] bs = File.ReadAllBytes(fileName); System.IO.MemoryStream filestream = new System.IO.MemoryStream(bs); - + // Specify SSE-C encryption options + Aes aesEncryption = Aes.Create(); + aesEncryption.KeySize = 256; + aesEncryption.GenerateKey(); + var ssec = new SSEC(aesEncryption.Key); await minio.PutObjectAsync("mybucket", "island.jpg", filestream, filestream.Length, - "application/octet-stream"); + "application/octet-stream",ssec); Console.Out.WriteLine("island.jpg is uploaded successfully"); } catch(MinioException e) @@ -822,9 +830,9 @@ catch(MinioException e) ``` -### PutObjectAsync(string bucketName, string objectName, string filePath, string contentType=null) +### PutObjectAsync(string bucketName, string objectName, string filePath, string contentType=null,ServerSideEncryption sse) -` Task PutObjectAsync(string bucketName, string objectName, string filePath, string contentType=null,Dictionary metaData=null, CancellationToken cancellationToken = default(CancellationToken))` +` Task PutObjectAsync(string bucketName, string objectName, string filePath, string contentType=null,Dictionary metaData=null, ServerSideEncryption sse=null,CancellationToken cancellationToken = default(CancellationToken))` Uploads contents from a file to objectName. @@ -840,6 +848,7 @@ __Parameters__ | ``fileName`` | _string_ | File to upload | | ``contentType`` | _string_ | Content type of the file. Defaults to " | | ``metadata`` | _Dictionary_ | Dictionary of meta data headers and their values.Defaults to null.| +| ``sse`` | _ServerSideEncryption_ | Server-side encryption option | Optional parameter. Defaults to null | | ``cancellationToken``| _System.Threading.CancellationToken_ | Optional parameter. Defaults to default(CancellationToken) | @@ -870,9 +879,9 @@ catch(MinioException e) } ``` -### StatObjectAsync(string bucketName, string objectName) +### StatObjectAsync(string bucketName, string objectName,ServerSideEncryption sse) -`Task StatObjectAsync(string bucketName, string objectName, CancellationToken cancellationToken = default(CancellationToken))` +`Task StatObjectAsync(string bucketName, string objectName,ServerSideEncryption sse = null, CancellationToken cancellationToken = default(CancellationToken))` Gets metadata of an object. @@ -884,6 +893,7 @@ __Parameters__ |:--- |:--- |:--- | | ``bucketName`` | _string_ | Name of the bucket | | ``objectName`` | _string_ | Object name in the bucket | +| ``sse`` | _ServerSideEncryption_ | Server-side encryption option | Optional parameter. Defaults to null | | ``cancellationToken``| _System.Threading.CancellationToken_ | Optional parameter. Defaults to default(CancellationToken) | @@ -913,9 +923,9 @@ catch(MinioException e) ``` -### CopyObjectAsync(string bucketName, string objectName, string destBucketName, string destObjectName = null, CopyConditions copyConditions = null,Dictionary metadata = null) +### CopyObjectAsync(string bucketName, string objectName, string destBucketName, string destObjectName = null, CopyConditions copyConditions = null,Dictionary metadata = null, ServerSideEncryption sseSrc = null, ServerSideEncryption sseDest = null) -*`Task CopyObjectAsync(string bucketName, string objectName, string destBucketName, string destObjectName = null, CopyConditions copyConditions = null, Dictionary metadata = null,CancellationToken cancellationToken = default(CancellationToken))`* +*`Task CopyObjectAsync(string bucketName, string objectName, string destBucketName, string destObjectName = null, CopyConditions copyConditions = null, Dictionary metadata = null,ServerSideEncryption sseSrc = null, ServerSideEncryption sseDest = null,CancellationToken cancellationToken = default(CancellationToken))`* Copies content from objectName to destObjectName. @@ -931,6 +941,8 @@ __Parameters__ | ``destObjectName`` | _string_ | Destination object name to be created, if not provided defaults to source object name| | ``copyConditions`` | _CopyConditions_ | Map of conditions useful for applying restrictions on copy operation| | ``metadata`` | _Dictionary_ | Dictionary of meta data headers and their values on the destination side.Defaults to null.| +| ``sseSrc`` | _ServerSideEncryption_ | Server-side encryption option for source object | Optional parameter. Defaults to null | +| ``sseDest`` | _ServerSideEncryption_ | Server-side encryption option for destination object| Optional parameter. Defaults to null | | ``cancellationToken``| _System.Threading.CancellationToken_ | Optional parameter. Defaults to default(CancellationToken) | @@ -945,15 +957,23 @@ __Parameters__ __Example__ -This API performs a server side copy operation from a given source object to destination object. +This API performs a Server-side copy operation from a given source object to destination object. ```cs try { CopyConditions copyConditions = new CopyConditions(); copyConditions.setMatchETagNone("TestETag"); - - await minioClient.CopyObjectAsync("mybucket", "island.jpg", "mydestbucket", "processed.png", copyConditions); + ServerSideEncryption sseSrc,sseDst; + // Uncomment to specify source and destination Server-side encryption options + /* + Aes aesEncryption = Aes.Create(); + aesEncryption.KeySize = 256; + aesEncryption.GenerateKey(); + sseSrc = new SSEC(aesEncryption.Key); + sseDst = new SSES3(); + */ + await minioClient.CopyObjectAsync("mybucket", "island.jpg", "mydestbucket", "processed.png", copyConditions,sseSrc:sseSrc, sseDest:sseDst); Console.Out.WriteLine("island.jpg is uploaded successfully"); } catch(MinioException e) diff --git a/Minio.Examples/Cases/CopyObject.cs b/Minio.Examples/Cases/CopyObject.cs index e5ab45244..c7f3766ad 100644 --- a/Minio.Examples/Cases/CopyObject.cs +++ b/Minio.Examples/Cases/CopyObject.cs @@ -17,6 +17,8 @@ using System; using System.Threading.Tasks; +using Minio.DataModel; + namespace Minio.Examples.Cases { class CopyObject @@ -26,7 +28,9 @@ public async static Task Run(Minio.MinioClient minio, string fromBucketName="from-bucket-name", string fromObjectName="from-object-name", string destBucketName="dest-bucket", - string destObjectName="to-object-name") + string destObjectName="to-object-name", + ServerSideEncryption sseSrc = null, + ServerSideEncryption sseDest = null) { try { @@ -36,7 +40,9 @@ await minio.CopyObjectAsync(fromBucketName, fromObjectName, destBucketName, destObjectName, - copyConditions:null); + copyConditions:null, + sseSrc: sseSrc, + sseDest: sseDest); Console.Out.WriteLine("Copied object {0} from bucket {1} to bucket {2}", fromObjectName, fromBucketName, destBucketName); Console.Out.WriteLine(); } diff --git a/Minio.Examples/Cases/FGetObject.cs b/Minio.Examples/Cases/FGetObject.cs index d439c9fe4..ce8491f7f 100644 --- a/Minio.Examples/Cases/FGetObject.cs +++ b/Minio.Examples/Cases/FGetObject.cs @@ -18,6 +18,8 @@ using System.IO; using System.Threading.Tasks; +using Minio.DataModel; + namespace Minio.Examples.Cases { class FGetObject @@ -26,13 +28,14 @@ class FGetObject public async static Task Run(Minio.MinioClient minio, string bucketName = "my-bucket-name", string objectName = "my-object-name", - string fileName="local-filename") + string fileName="local-filename", + ServerSideEncryption sse = null) { try { Console.Out.WriteLine("Running example for API: GetObjectAsync"); File.Delete(fileName); - await minio.GetObjectAsync(bucketName, objectName, fileName); + await minio.GetObjectAsync(bucketName, objectName, fileName, sse: sse); Console.WriteLine("Downloaded the file " + fileName + " from bucket " + bucketName); Console.Out.WriteLine(); } diff --git a/Minio.Examples/Cases/PutObject.cs b/Minio.Examples/Cases/PutObject.cs index f024f8531..51f57fe00 100644 --- a/Minio.Examples/Cases/PutObject.cs +++ b/Minio.Examples/Cases/PutObject.cs @@ -17,6 +17,9 @@ using System; using System.IO; using System.Threading.Tasks; +using Minio.DataModel; + +using System.Security.Cryptography; namespace Minio.Examples.Cases { @@ -28,7 +31,8 @@ class PutObject public async static Task Run(Minio.MinioClient minio, string bucketName = "my-bucket-name", string objectName = "my-object-name", - string fileName="location-of-file") + string fileName="location-of-file", + ServerSideEncryption sse = null) { try { @@ -43,12 +47,12 @@ public async static Task Run(Minio.MinioClient minio, { Console.Out.WriteLine("Running example for API: PutObjectAsync with Stream and MultiPartUpload"); } - await minio.PutObjectAsync(bucketName, objectName, filestream, filestream.Length, - "application/octet-stream"); + "application/octet-stream", + sse:sse); } diff --git a/Minio.Examples/Program.cs b/Minio.Examples/Program.cs index 30e8d3d86..c9b15a809 100644 --- a/Minio.Examples/Program.cs +++ b/Minio.Examples/Program.cs @@ -21,6 +21,9 @@ using Minio.DataModel; using Minio.Exceptions; +using System.Net; +using System.Security.Cryptography; + namespace Minio.Examples { public class Program @@ -73,7 +76,8 @@ public static void Main(string[] args) secretKey = "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"; enableHTTPS = true; } - + ServicePointManager.ServerCertificateValidationCallback += + (sender, certificate, chain, sslPolicyErrors) => true; // WithSSL() enables SSL support in Minio client MinioClient minioClient = null; if (enableHTTPS) @@ -101,7 +105,6 @@ public static void Main(string[] args) // Set HTTP Tracing On // minioClient.SetTraceOn(); - // Set HTTP Tracing Off // minioClient.SetTraceOff(); // Check if bucket exists @@ -146,6 +149,27 @@ public static void Main(string[] args) // Automatic Multipart Upload with object more than 5Mb Cases.PutObject.Run(minioClient, bucketName, objectName, bigFileName).Wait(); + // Specify SSE-C encryption options + Aes aesEncryption = Aes.Create(); + aesEncryption.KeySize = 256; + aesEncryption.GenerateKey(); + var ssec = new SSEC(aesEncryption.Key); + // Specify SSE-C source side encryption for Copy operations + var sseCpy = new SSECopy(aesEncryption.Key); + + // Uncommment to specify SSE-S3 encryption option + // var sses3 = new SSES3(); + + // Uncommment to specify SSE-KMS encryption option + // var sseKms = new SSEKMS("kms-key",new Dictionary{{ "kms-context", "somevalue"}}); + + // Upload encrypted object + Cases.PutObject.Run(minioClient, bucketName, objectName, smallFileName,sse:ssec).Wait(); + // Copy SSE-C encrypted object to unencrypted object + Cases.CopyObject.Run(minioClient, bucketName, objectName, destBucketName, objectName,sseSrc:sseCpy,sseDest:ssec).Wait(); + // Download SSE-C encrypted object + Cases.FGetObject.Run(minioClient, destBucketName, objectName, bigFileName,sse:ssec).Wait(); + // List the incomplete uploads Cases.ListIncompleteUploads.Run(minioClient, bucketName); @@ -190,12 +214,13 @@ public static void Main(string[] args) // Remove the buckets Cases.RemoveBucket.Run(minioClient, bucketName).Wait(); Cases.RemoveBucket.Run(minioClient, destBucketName).Wait(); - + // Remove the binary files created for test File.Delete(smallFileName); File.Delete(bigFileName); Console.ReadLine(); + } catch (MinioException ex) { diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index 070b3ab21..2a9072fab 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -21,6 +21,7 @@ using System.Net; using System.Net.Http; using System.Collections.Generic; +using System.Security.Cryptography; using System.Threading; using Microsoft.VisualStudio.TestTools.UnitTesting; using Minio.DataModel; @@ -118,6 +119,7 @@ public static void Main(string[] args) String accessKey = null; String secretKey = null; String enableHttps = "0"; + String kmsEnabled = "0"; bool useAWS = Environment.GetEnvironmentVariable("AWS_ENDPOINT") != null; if (Environment.GetEnvironmentVariable("SERVER_ENDPOINT") != null) @@ -126,6 +128,7 @@ public static void Main(string[] args) accessKey = Environment.GetEnvironmentVariable("ACCESS_KEY"); secretKey = Environment.GetEnvironmentVariable("SECRET_KEY"); enableHttps = Environment.GetEnvironmentVariable("ENABLE_HTTPS"); + kmsEnabled = Environment.GetEnvironmentVariable("ENABLE_KMS"); } else { @@ -243,8 +246,24 @@ public static void Main(string[] args) RemoveIncompleteUpload_Test(minioClient).Wait(); // Test GetBucket policy - GetBucketPolicy_Test1(minioClient).Wait(); + + // Test encryption + if (enableHttps.Equals("1")) + { + ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, sslPolicyErrors) => true; + PutGetStatEncryptedObject_Test1(minioClient).Wait(); + PutGetStatEncryptedObject_Test2(minioClient).Wait(); + + EncryptedCopyObject_Test1(minioClient).Wait(); + EncryptedCopyObject_Test2(minioClient).Wait(); + } + if (kmsEnabled.Equals("1")) + { + PutGetStatEncryptedObject_Test3(minioClient).Wait(); + EncryptedCopyObject_Test3(minioClient).Wait(); + EncryptedCopyObject_Test4(minioClient).Wait(); + } } private static void runCoreTests(MinioClient minioClient) { @@ -719,6 +738,180 @@ await minio.PutObjectAsync(bucketName, new MintLogger("PutObject_Test8",putObjectSignature1,"Tests whether PutObject with unknown stream-size passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(), args).Log(); } } + private async static Task PutGetStatEncryptedObject_Test1(MinioClient minio) + { + DateTime startTime = DateTime.Now; + string bucketName = GetRandomName(15); + string objectName = GetRandomName(10); + string contentType = "application/octet-stream"; + Dictionary args = new Dictionary + { + { "bucketName", bucketName}, + {"objectName",objectName}, + {"contentType", contentType}, + {"data","1MB"}, + {"size","1MB"}, + }; + try + { + // Putobject with SSE-C encryption. + await Setup_Test(minio, bucketName); + Aes aesEncryption = Aes.Create(); + aesEncryption.KeySize = 256; + aesEncryption.GenerateKey(); + var ssec = new SSEC(aesEncryption.Key); + + using (System.IO.MemoryStream filestream = rsg.GenerateStreamFromSeed(1 * MB)) + { + long file_write_size = filestream.Length; + string tempFileName = "tempFileName"; + long file_read_size = 0; + await minio.PutObjectAsync(bucketName, + objectName, + filestream, + filestream.Length, + contentType,sse: ssec); + + await minio.GetObjectAsync(bucketName, objectName, + (stream) => + { + var fileStream = File.Create(tempFileName); + stream.CopyTo(fileStream); + fileStream.Dispose(); + FileInfo writtenInfo = new FileInfo(tempFileName); + file_read_size = writtenInfo.Length; + + Assert.AreEqual(file_read_size, file_write_size); + File.Delete(tempFileName); + },sse:ssec); + await minio.StatObjectAsync(bucketName, objectName,sse:ssec); + await minio.RemoveObjectAsync(bucketName, objectName); + } + await TearDown(minio, bucketName); + + new MintLogger("PutGetStatEncryptedObject_Test1",putObjectSignature1,"Tests whether Put/Get/Stat Object with encryption passes",TestStatus.PASS,(DateTime.Now - startTime), args:args).Log(); + } + catch (Exception ex) + { + new MintLogger("PutGetStatEncryptedObject_Test1",putObjectSignature1,"Tests whether Put/Get/Stat Object with encryption passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(), args).Log(); + } + } + + private async static Task PutGetStatEncryptedObject_Test2(MinioClient minio) + { + DateTime startTime = DateTime.Now; + string bucketName = GetRandomName(15); + string objectName = GetRandomName(10); + string contentType = "application/octet-stream"; + Dictionary args = new Dictionary + { + { "bucketName", bucketName}, + {"objectName",objectName}, + {"contentType", contentType}, + {"data","6MB"}, + {"size","6MB"}, + }; + try + { + // Test multipart Put with SSE-C encryption + await Setup_Test(minio, bucketName); + Aes aesEncryption = Aes.Create(); + aesEncryption.KeySize = 256; + aesEncryption.GenerateKey(); + var ssec = new SSEC(aesEncryption.Key); + + using (System.IO.MemoryStream filestream = rsg.GenerateStreamFromSeed(6 * MB)) + { + long file_write_size = filestream.Length; + string tempFileName = "tempFileName"; + long file_read_size = 0; + await minio.PutObjectAsync(bucketName, + objectName, + filestream, + filestream.Length, + contentType,sse: ssec); + + await minio.GetObjectAsync(bucketName, objectName, + (stream) => + { + var fileStream = File.Create(tempFileName); + stream.CopyTo(fileStream); + fileStream.Dispose(); + FileInfo writtenInfo = new FileInfo(tempFileName); + file_read_size = writtenInfo.Length; + + Assert.AreEqual(file_read_size, file_write_size); + File.Delete(tempFileName); + },sse:ssec); + await minio.StatObjectAsync(bucketName, objectName,sse:ssec); + await minio.RemoveObjectAsync(bucketName, objectName); + } + await TearDown(minio, bucketName); + + new MintLogger("PutGetStatEncryptedObject_Test2",putObjectSignature1,"Tests whether Put/Get/Stat multipart upload with encryption passes",TestStatus.PASS,(DateTime.Now - startTime), args:args).Log(); + } + catch (Exception ex) + { + new MintLogger("PutGetStatEncryptedObject_Test2",putObjectSignature2,"Tests whether Put/Get/Stat multipart upload with encryption passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(), args).Log(); + } + } + + private async static Task PutGetStatEncryptedObject_Test3(MinioClient minio) + { + DateTime startTime = DateTime.Now; + string bucketName = GetRandomName(15); + string objectName = GetRandomName(10); + string contentType = "application/octet-stream"; + Dictionary args = new Dictionary + { + { "bucketName", bucketName}, + {"objectName",objectName}, + {"contentType", contentType}, + {"data","6MB"}, + {"size","6MB"}, + }; + try + { + // Test multipart Put/Get/Stat with SSE-S3 encryption + await Setup_Test(minio, bucketName); + Aes aesEncryption = Aes.Create(); + var sses3 = new SSES3(); + + using (System.IO.MemoryStream filestream = rsg.GenerateStreamFromSeed(6 * MB)) + { + long file_write_size = filestream.Length; + string tempFileName = "tempFileName"; + long file_read_size = 0; + await minio.PutObjectAsync(bucketName, + objectName, + filestream, + filestream.Length, + contentType,sse: sses3); + + await minio.GetObjectAsync(bucketName, objectName, + (stream) => + { + var fileStream = File.Create(tempFileName); + stream.CopyTo(fileStream); + fileStream.Dispose(); + FileInfo writtenInfo = new FileInfo(tempFileName); + file_read_size = writtenInfo.Length; + + Assert.AreEqual(file_read_size, file_write_size); + File.Delete(tempFileName); + }); + await minio.StatObjectAsync(bucketName, objectName); + await minio.RemoveObjectAsync(bucketName, objectName); + } + await TearDown(minio, bucketName); + + new MintLogger("PutGetStatEncryptedObject_Test3",putObjectSignature1,"Tests whether Put/Get/Stat multipart upload with encryption passes",TestStatus.PASS,(DateTime.Now - startTime), args:args).Log(); + } + catch (Exception ex) + { + new MintLogger("PutGetStatEncryptedObject_Test3",putObjectSignature2,"Tests whether Put/Get/Stat multipart upload with encryption passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(), args).Log(); + } + } private async static Task PutObject_Task(MinioClient minio, string bucketName, string objectName, string fileName = null, string contentType = "application/octet-stream", long size = 0, Dictionary metaData = null, MemoryStream mstream = null) { DateTime startTime = DateTime.Now; @@ -1256,6 +1449,219 @@ await minio.PutObjectAsync(bucketName, new MintLogger("CopyObject_Test8",copyObjectSignature,"Tests whether CopyObject with metadata replacement passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log(); } } + + private async static Task EncryptedCopyObject_Test1(MinioClient minio) + { + DateTime startTime = DateTime.Now; + string bucketName = GetRandomName(15); + string objectName = GetRandomName(10); + string destBucketName = GetRandomName(15); + string destObjectName = GetRandomName(10); + Dictionary args = new Dictionary + { + {"bucketName", bucketName}, + {"objectName",objectName}, + {"destBucketName", destBucketName}, + {"destObjectName", destObjectName}, + {"data","1MB"}, + {"size","1MB"}, + }; + try + { + // Test Copy with SSE-C -> SSE-C encryption + await Setup_Test(minio, bucketName); + await Setup_Test(minio, destBucketName); + Aes aesEncryption = Aes.Create(); + aesEncryption.KeySize = 256; + aesEncryption.GenerateKey(); + var ssec = new SSEC(aesEncryption.Key); + var sseCpy = new SSECopy(aesEncryption.Key); + Aes destAesEncryption = Aes.Create(); + destAesEncryption.KeySize = 256; + destAesEncryption.GenerateKey(); + var ssecDst = new SSEC(destAesEncryption.Key); + using (MemoryStream filestream = rsg.GenerateStreamFromSeed(1 * MB)) + { + await minio.PutObjectAsync(bucketName, + objectName, + filestream, filestream.Length, null, sse:ssec); + } + + await minio.CopyObjectAsync(bucketName, objectName, destBucketName, destObjectName,sseSrc:sseCpy,sseDest:ssecDst); + string outFileName = "outFileName"; + + await minio.GetObjectAsync(destBucketName, destObjectName, outFileName,sse:ssecDst); + File.Delete(outFileName); + await minio.RemoveObjectAsync(bucketName, objectName); + await minio.RemoveObjectAsync(destBucketName, destObjectName); + + + await TearDown(minio, bucketName); + await TearDown(minio, destBucketName); + new MintLogger("EncryptedCopyObject_Test1",copyObjectSignature,"Tests whether encrypted CopyObject passes",TestStatus.PASS,(DateTime.Now - startTime), args:args).Log(); + } + catch (MinioException ex) + { + new MintLogger("EncryptedCopyObject_Test1",copyObjectSignature,"Tests whether encrypted CopyObject passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log(); + } + } + + private async static Task EncryptedCopyObject_Test2(MinioClient minio) + { + DateTime startTime = DateTime.Now; + string bucketName = GetRandomName(15); + string objectName = GetRandomName(10); + string destBucketName = GetRandomName(15); + string destObjectName = GetRandomName(10); + Dictionary args = new Dictionary + { + {"bucketName", bucketName}, + {"objectName",objectName}, + {"destBucketName", destBucketName}, + {"destObjectName", destObjectName}, + {"data","1MB"}, + {"size","1MB"}, + }; + try + { + // Test Copy of SSE-C encrypted object to unencrypted on destination side + await Setup_Test(minio, bucketName); + await Setup_Test(minio, destBucketName); + Aes aesEncryption = Aes.Create(); + aesEncryption.KeySize = 256; + aesEncryption.GenerateKey(); + var ssec = new SSEC(aesEncryption.Key); + var sseCpy = new SSECopy(aesEncryption.Key); + + using (MemoryStream filestream = rsg.GenerateStreamFromSeed(1 * MB)) + { + await minio.PutObjectAsync(bucketName, + objectName, + filestream, filestream.Length, null, sse:ssec); + } + + await minio.CopyObjectAsync(bucketName, objectName, destBucketName, destObjectName,sseSrc:sseCpy,sseDest:null); + string outFileName = "outFileName"; + + await minio.GetObjectAsync(destBucketName, destObjectName, outFileName); + File.Delete(outFileName); + await minio.RemoveObjectAsync(bucketName, objectName); + await minio.RemoveObjectAsync(destBucketName, destObjectName); + + + await TearDown(minio, bucketName); + await TearDown(minio, destBucketName); + new MintLogger("EncryptedCopyObject_Test2",copyObjectSignature,"Tests whether encrypted CopyObject passes",TestStatus.PASS,(DateTime.Now - startTime), args:args).Log(); + } + catch (MinioException ex) + { + new MintLogger("EncryptedCopyObject_Test2",copyObjectSignature,"Tests whether encrypted CopyObject passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log(); + } + } + + private async static Task EncryptedCopyObject_Test3(MinioClient minio) + { + DateTime startTime = DateTime.Now; + string bucketName = GetRandomName(15); + string objectName = GetRandomName(10); + string destBucketName = GetRandomName(15); + string destObjectName = GetRandomName(10); + Dictionary args = new Dictionary + { + {"bucketName", bucketName}, + {"objectName",objectName}, + {"destBucketName", destBucketName}, + {"destObjectName", destObjectName}, + {"data","1MB"}, + {"size","1MB"}, + }; + try + { + // Test Copy of SSE-C encrypted object to unencrypted on destination side + await Setup_Test(minio, bucketName); + await Setup_Test(minio, destBucketName); + Aes aesEncryption = Aes.Create(); + aesEncryption.KeySize = 256; + aesEncryption.GenerateKey(); + var ssec = new SSEC(aesEncryption.Key); + var sseCpy = new SSECopy(aesEncryption.Key); + var sses3 = new SSES3(); + + using (MemoryStream filestream = rsg.GenerateStreamFromSeed(1 * MB)) + { + await minio.PutObjectAsync(bucketName, + objectName, + filestream, filestream.Length, null, sse:ssec); + } + + await minio.CopyObjectAsync(bucketName, objectName, destBucketName, destObjectName,sseSrc:sseCpy,sseDest:sses3); + string outFileName = "outFileName"; + + await minio.GetObjectAsync(destBucketName, destObjectName, outFileName); + File.Delete(outFileName); + await minio.RemoveObjectAsync(bucketName, objectName); + await minio.RemoveObjectAsync(destBucketName, destObjectName); + + + await TearDown(minio, bucketName); + await TearDown(minio, destBucketName); + new MintLogger("EncryptedCopyObject_Test3",copyObjectSignature,"Tests whether encrypted CopyObject passes",TestStatus.PASS,(DateTime.Now - startTime), args:args).Log(); + } + catch (MinioException ex) + { + new MintLogger("EncryptedCopyObject_Test3",copyObjectSignature,"Tests whether encrypted CopyObject passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log(); + } + } + + private async static Task EncryptedCopyObject_Test4(MinioClient minio) + { + DateTime startTime = DateTime.Now; + string bucketName = GetRandomName(15); + string objectName = GetRandomName(10); + string destBucketName = GetRandomName(15); + string destObjectName = GetRandomName(10); + Dictionary args = new Dictionary + { + {"bucketName", bucketName}, + {"objectName",objectName}, + {"destBucketName", destBucketName}, + {"destObjectName", destObjectName}, + {"data","1MB"}, + {"size","1MB"}, + }; + try + { + // Test Copy of SSE-S3 encrypted object to SSE-S3 on destination side + await Setup_Test(minio, bucketName); + await Setup_Test(minio, destBucketName); + + var sses3 = new SSES3(); + var sseDest = new SSES3(); + using (MemoryStream filestream = rsg.GenerateStreamFromSeed(1 * MB)) + { + await minio.PutObjectAsync(bucketName, + objectName, + filestream, filestream.Length, null, sse:sses3); + } + + await minio.CopyObjectAsync(bucketName, objectName, destBucketName, destObjectName,sseSrc:null,sseDest:sses3); + string outFileName = "outFileName"; + + await minio.GetObjectAsync(destBucketName, destObjectName, outFileName); + File.Delete(outFileName); + await minio.RemoveObjectAsync(bucketName, objectName); + await minio.RemoveObjectAsync(destBucketName, destObjectName); + + + await TearDown(minio, bucketName); + await TearDown(minio, destBucketName); + new MintLogger("EncryptedCopyObject_Test4",copyObjectSignature,"Tests whether encrypted CopyObject passes",TestStatus.PASS,(DateTime.Now - startTime), args:args).Log(); + } + catch (MinioException ex) + { + new MintLogger("EncryptedCopyObject_Test4",copyObjectSignature,"Tests whether encrypted CopyObject passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log(); + } + } private async static Task GetObject_Test1(MinioClient minio) { DateTime startTime = DateTime.Now; diff --git a/Minio/ApiEndpoints/IObjectOperations.cs b/Minio/ApiEndpoints/IObjectOperations.cs index be8927b93..8f98be1da 100644 --- a/Minio/ApiEndpoints/IObjectOperations.cs +++ b/Minio/ApiEndpoints/IObjectOperations.cs @@ -31,8 +31,9 @@ public interface IObjectOperations /// Bucket to retrieve object from /// Name of object to retrieve /// A stream will be passed to the callback + /// Optional Server-side encryption option. Defaults to null. /// Optional cancellation token to cancel the operation - Task GetObjectAsync(string bucketName, string objectName, Action callback, CancellationToken cancellationToken = default(CancellationToken)); + Task GetObjectAsync(string bucketName, string objectName, Action callback, ServerSideEncryption sse = null, CancellationToken cancellationToken = default(CancellationToken)); /// /// Get an object. The object will be streamed to the callback given by the user. @@ -42,8 +43,9 @@ public interface IObjectOperations /// offset of the object from where stream will start /// length of object to read in from the stream /// A stream will be passed to the callback + /// Optional Server-side encryption option. Defaults to null. /// Optional cancellation token to cancel the operation - Task GetObjectAsync(string bucketName, string objectName, long offset, long length, Action cb, CancellationToken cancellationToken = default(CancellationToken)); + Task GetObjectAsync(string bucketName, string objectName, long offset, long length, Action cb, ServerSideEncryption sse = null, CancellationToken cancellationToken = default(CancellationToken)); /// /// Creates an object from file input stream @@ -53,9 +55,11 @@ public interface IObjectOperations /// Stream of file to upload /// Size of stream /// Content type of the new object, null defaults to "application/octet-stream" - /// Optional cancellation token to cancel the operation /// Optional Object metadata to be stored. Defaults to null. - Task PutObjectAsync(string bucketName, string objectName, Stream data, long size, string contentType = null, Dictionary metaData = null,CancellationToken cancellationToken = default(CancellationToken)); + /// Optional Server-side encryption option. Defaults to null. + /// Optional cancellation token to cancel the operation + + Task PutObjectAsync(string bucketName, string objectName, Stream data, long size, string contentType = null, Dictionary metaData = null, ServerSideEncryption sse = null, CancellationToken cancellationToken = default(CancellationToken)); /// /// Removes an object with given name in specific bucket @@ -80,9 +84,10 @@ public interface IObjectOperations /// /// Bucket to test object in /// Name of the object to stat + /// Optional Server-side encryption option. Defaults to null. /// Optional cancellation token to cancel the operation /// Facts about the object - Task StatObjectAsync(string bucketName, string objectName, CancellationToken cancellationToken = default(CancellationToken)); + Task StatObjectAsync(string bucketName, string objectName, ServerSideEncryption sse = null, CancellationToken cancellationToken = default(CancellationToken)); /// /// Lists all incomplete uploads in a given bucket and prefix recursively @@ -112,10 +117,13 @@ public interface IObjectOperations /// Object name to be created, if not provided uses source object name as destination object name. /// optionally can take a key value CopyConditions as well for conditionally attempting copyObject. /// Optional Object metadata to be stored. Defaults to null. + /// Optional Server-side encryption option for source. Defaults to null. + /// Optional Server-side encryption option for destination. Defaults to null. + /// Optional cancellation token to cancel the operation /// - Task CopyObjectAsync(string bucketName, string objectName, string destBucketName, string destObjectName = null, CopyConditions copyConditions = null, Dictionary metadata = null,CancellationToken cancellationToken = default(CancellationToken)); + Task CopyObjectAsync(string bucketName, string objectName, string destBucketName, string destObjectName = null, CopyConditions copyConditions = null, Dictionary metadata = null, ServerSideEncryption sseSrc = null, ServerSideEncryption sseDest = null, CancellationToken cancellationToken = default(CancellationToken)); /// /// Creates an object from file @@ -124,9 +132,10 @@ public interface IObjectOperations /// Key of the new object /// Path of file to upload /// Content type of the new object, null defaults to "application/octet-stream" - /// Optional cancellation token to cancel the operation /// Optional Object metadata to be stored. Defaults to null. - Task PutObjectAsync(string bucketName, string objectName, string filePath, string contentType = null, Dictionary metaData = null, CancellationToken cancellationToken = default(CancellationToken)); + /// Optional Server-side encryption option. Defaults to null. + /// Optional cancellation token to cancel the operation + Task PutObjectAsync(string bucketName, string objectName, string filePath, string contentType = null, Dictionary metaData = null, ServerSideEncryption sse = null, CancellationToken cancellationToken = default(CancellationToken)); /// /// Get an object. The object will be streamed to the callback given by the user. @@ -134,9 +143,12 @@ public interface IObjectOperations /// Bucket to retrieve object from /// Name of object to retrieve /// string with file path + /// Optional Server-side encryption option. Defaults to null. /// Optional cancellation token to cancel the operation /// - Task GetObjectAsync(string bucketName, string objectName, string filePath, CancellationToken cancellationToken = default(CancellationToken)); + //Task GetObjectAsync(string bucketName, string objectName, string filePath,ServerSideEncryption sse = null, CancellationToken cancellationToken = default(CancellationToken)); + + Task GetObjectAsync(string bucketName, string objectName, string filePath, ServerSideEncryption sse = null ,CancellationToken cancellationToken = default(CancellationToken)); /// /// Presigned get url - returns a presigned url to access an object's data without credentials.URL can have a maximum expiry of diff --git a/Minio/ApiEndpoints/ObjectOperations.cs b/Minio/ApiEndpoints/ObjectOperations.cs index 9b9aedc0c..20b723693 100644 --- a/Minio/ApiEndpoints/ObjectOperations.cs +++ b/Minio/ApiEndpoints/ObjectOperations.cs @@ -44,14 +44,21 @@ public partial class MinioClient : IObjectOperations /// Bucket to retrieve object from /// Name of object to retrieve /// A stream will be passed to the callback + /// Server-side encryption option. Defaults to null. /// Optional cancellation token to cancel the operation - public async Task GetObjectAsync(string bucketName, string objectName, Action cb, CancellationToken cancellationToken = default(CancellationToken)) + public async Task GetObjectAsync(string bucketName, string objectName, Action cb, ServerSideEncryption sse = null, CancellationToken cancellationToken = default(CancellationToken)) { - await StatObjectAsync(bucketName, objectName, cancellationToken).ConfigureAwait(false); + await StatObjectAsync(bucketName, objectName, cancellationToken:cancellationToken).ConfigureAwait(false); + var headers = new Dictionary(); + if (sse != null && sse.GetType().Equals(EncryptionType.SSE_C)) + { + sse.Marshal(headers); + } var request = await this.CreateRequest(Method.GET, bucketName, - objectName: objectName) + objectName: objectName, + headerMap: headers) .ConfigureAwait(false); request.ResponseWriter = cb; @@ -67,21 +74,26 @@ public partial class MinioClient : IObjectOperations /// Name of object to retrieve /// Offset of the object from where stream will start /// length of the object that will be read in the stream - /// Optional cancellation token to cancel the operation /// A stream will be passed to the callback - public async Task GetObjectAsync(string bucketName, string objectName, long offset, long length, Action cb, CancellationToken cancellationToken = default(CancellationToken)) + /// Server-side encryption option. Defaults to null. + /// Optional cancellation token to cancel the operation + public async Task GetObjectAsync(string bucketName, string objectName, long offset, long length, Action cb, ServerSideEncryption sse = null, CancellationToken cancellationToken = default(CancellationToken)) { if (offset < 0) throw new ArgumentException("Offset should be zero or greater"); if (length < 0) throw new ArgumentException("Length should be greater than zero"); - await StatObjectAsync(bucketName, objectName, cancellationToken).ConfigureAwait(false); + await StatObjectAsync(bucketName, objectName, cancellationToken:cancellationToken).ConfigureAwait(false); Dictionary headerMap = new Dictionary(); if (length > 0) headerMap.Add("Range", "bytes=" + offset.ToString() + "-" + (offset + length - 1).ToString()); + if (sse != null && sse.GetType().Equals(EncryptionType.SSE_C)) + { + sse.Marshal(headerMap); + } var request = await this.CreateRequest(Method.GET, bucketName, objectName: objectName, @@ -98,15 +110,16 @@ public partial class MinioClient : IObjectOperations /// Bucket to retrieve object from /// Name of object to retrieve /// string with file path + /// Server-side encryption option. Defaults to null. /// Optional cancellation token to cancel the operation /// - public async Task GetObjectAsync(string bucketName, string objectName, string fileName, CancellationToken cancellationToken = default(CancellationToken)) + public async Task GetObjectAsync(string bucketName, string objectName, string fileName, ServerSideEncryption sse = null, CancellationToken cancellationToken = default(CancellationToken)) { bool fileExists = File.Exists(fileName); utils.ValidateFile(fileName); - ObjectStat objectStat = await StatObjectAsync(bucketName, objectName, cancellationToken).ConfigureAwait(false); + ObjectStat objectStat = await StatObjectAsync(bucketName, objectName, sse:sse,cancellationToken:cancellationToken).ConfigureAwait(false); long length = objectStat.Size; string etag = objectStat.ETag; @@ -164,7 +177,7 @@ await GetObjectAsync(bucketName, objectName, (stream) => + ", written = " + writtenSize); } utils.MoveWithReplace(tempFileName, fileName); - }, cancellationToken).ConfigureAwait(false); + }, sse, cancellationToken).ConfigureAwait(false); } /// @@ -174,18 +187,18 @@ await GetObjectAsync(bucketName, objectName, (stream) => /// Key of the new object /// Path of file to upload /// Content type of the new object, null defaults to "application/octet-stream" - /// Optional cancellation token to cancel the operation /// Object metadata to be stored. Defaults to null. - public async Task PutObjectAsync(string bucketName, string objectName, string fileName, string contentType = null, Dictionary metaData = null, CancellationToken cancellationToken = default(CancellationToken)) + /// Server-side encryption option. Defaults to null. + /// Optional cancellation token to cancel the operation + public async Task PutObjectAsync(string bucketName, string objectName, string fileName, string contentType = null, Dictionary metaData = null, ServerSideEncryption sse = null, CancellationToken cancellationToken = default(CancellationToken)) { utils.ValidateFile(fileName, contentType); FileInfo fileInfo = new FileInfo(fileName); long size = fileInfo.Length; using (FileStream file = new FileStream(fileName, FileMode.Open, FileAccess.Read)) { - await PutObjectAsync(bucketName, objectName, file, size, contentType, metaData, cancellationToken).ConfigureAwait(false); + await PutObjectAsync(bucketName, objectName, file, size, contentType, metaData,sse,cancellationToken).ConfigureAwait(false); } - } /// @@ -196,12 +209,15 @@ await GetObjectAsync(bucketName, objectName, (stream) => /// Total size of bytes to be written, must match with data's length /// Content type of the new object, null defaults to "application/octet-stream" /// Stream of bytes to send - /// Optional cancellation token to cancel the operation /// Object metadata to be stored. Defaults to null. - public async Task PutObjectAsync(string bucketName, string objectName, Stream data, long size, string contentType = null, Dictionary metaData = null, CancellationToken cancellationToken = default(CancellationToken)) + /// Server-side encryption option. Defaults to null. + /// Optional cancellation token to cancel the operation + public async Task PutObjectAsync(string bucketName, string objectName, Stream data, long size, string contentType = null, Dictionary metaData = null, ServerSideEncryption sse = null, CancellationToken cancellationToken = default(CancellationToken)) { utils.validateBucketName(bucketName); utils.validateObjectName(objectName); + var sseHeaders = new Dictionary(); + if (metaData == null) { metaData = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -210,6 +226,10 @@ await GetObjectAsync(bucketName, objectName, (stream) => { metaData = new Dictionary(metaData, StringComparer.OrdinalIgnoreCase); } + if (sse != null) + { + sse.Marshal(sseHeaders); + } if (string.IsNullOrWhiteSpace(contentType)) { contentType = "application/octet-stream"; @@ -231,7 +251,7 @@ await GetObjectAsync(bucketName, objectName, (stream) => { throw new UnexpectedShortReadException("Data read " + bytes.Length + " is shorter than the size " + size + " of input buffer."); } - await this.PutObjectAsync(bucketName, objectName, null, 0, bytes, metaData, cancellationToken).ConfigureAwait(false); + await this.PutObjectAsync(bucketName, objectName, null, 0, bytes, metaData, sseHeaders,cancellationToken).ConfigureAwait(false); return; } // For all sizes greater than 5MiB do multipart. @@ -248,13 +268,21 @@ await GetObjectAsync(bucketName, objectName, (stream) => if (uploadId == null) { - uploadId = await this.NewMultipartUploadAsync(bucketName, objectName, metaData, cancellationToken).ConfigureAwait(false); + uploadId = await this.NewMultipartUploadAsync(bucketName, objectName, metaData,sseHeaders,cancellationToken).ConfigureAwait(false); } else { existingParts = await this.ListParts(bucketName, objectName, uploadId, cancellationToken).ToArray(); } + if (sse != null && + (sse.GetType().Equals(EncryptionType.SSE_S3) || + sse.GetType().Equals(EncryptionType.SSE_KMS))) + { + sseHeaders.Remove(Constants.SSEGenericHeader); + sseHeaders.Remove(Constants.SSEKMSContext); + sseHeaders.Remove(Constants.SSEKMSKeyId); + } double expectedReadSize = partSize; int partNumber; int numPartsUploaded = 0; @@ -295,7 +323,7 @@ await GetObjectAsync(bucketName, objectName, (stream) => if (!skipUpload) { numPartsUploaded += 1; - string etag = await this.PutObjectAsync(bucketName, objectName, uploadId, partNumber, dataToCopy, metaData, cancellationToken).ConfigureAwait(false); + string etag = await this.PutObjectAsync(bucketName, objectName, uploadId, partNumber, dataToCopy, metaData,sseHeaders, cancellationToken).ConfigureAwait(false); totalParts[partNumber - 1] = new Part() { PartNumber = partNumber, ETag = etag, size = (long)expectedReadSize }; } @@ -424,7 +452,6 @@ private async Task>> GetListPartsAsync(string }); return new Tuple>(listPartsResult, uploads.ToList()); - } @@ -434,12 +461,17 @@ private async Task>> GetListPartsAsync(string /// /// /// + /// Server-side encryption options /// Optional cancellation token to cancel the operation /// - private async Task NewMultipartUploadAsync(string bucketName, string objectName, Dictionary metaData, CancellationToken cancellationToken) + private async Task NewMultipartUploadAsync(string bucketName, string objectName, Dictionary metaData ,Dictionary sseHeaders, CancellationToken cancellationToken = default(CancellationToken)) { var resource = "?uploads"; + foreach(KeyValuePair kv in sseHeaders) + { + metaData.Add(kv.Key, kv.Value); + } var request = await this.CreateRequest(Method.POST, bucketName, objectName: objectName, headerMap: metaData, resourcePath: resource).ConfigureAwait(false); @@ -462,9 +494,10 @@ private async Task NewMultipartUploadAsync(string bucketName, string obj /// /// /// + /// Server-side encryption headers if any /// Optional cancellation token to cancel the operation /// - private async Task PutObjectAsync(string bucketName, string objectName, string uploadId, int partNumber, byte[] data, Dictionary metaData, CancellationToken cancellationToken) + private async Task PutObjectAsync(string bucketName, string objectName, string uploadId, int partNumber, byte[] data, Dictionary metaData, Dictionary sseHeaders, CancellationToken cancellationToken) { var resource = ""; if (!string.IsNullOrEmpty(uploadId) && partNumber > 0) @@ -475,7 +508,12 @@ private async Task PutObjectAsync(string bucketName, string objectName, string contentType = metaData["Content-Type"]; if (uploadId != null) { - metaData = null; + metaData = new Dictionary(); + } + + foreach (KeyValuePair kv in sseHeaders) + { + metaData.Add(kv.Key,kv.Value); } var request = await this.CreateRequest(Method.PUT, bucketName, objectName: objectName, @@ -791,11 +829,18 @@ private async Task> removeObjectsAsync(string bucketName, List /// /// Bucket to test object in /// Name of the object to stat + /// Server-side encryption option.Defaults to null /// Optional cancellation token to cancel the operation /// Facts about the object - public async Task StatObjectAsync(string bucketName, string objectName, CancellationToken cancellationToken = default(CancellationToken)) + public async Task StatObjectAsync(string bucketName, string objectName, ServerSideEncryption sse = null, CancellationToken cancellationToken = default(CancellationToken)) { - var request = await this.CreateRequest(Method.HEAD, bucketName, objectName: objectName).ConfigureAwait(false); + var headerMap = new Dictionary(); + + if (sse != null && sse.GetType().Equals(EncryptionType.SSE_C)) + { + sse.Marshal(headerMap); + } + var request = await this.CreateRequest(Method.HEAD, bucketName, objectName: objectName, headerMap: headerMap).ConfigureAwait(false); var response = await this.ExecuteTaskAsync(this.NoErrorHandlers, request, cancellationToken).ConfigureAwait(false); @@ -885,9 +930,11 @@ internal byte[] ReadFull(Stream data, int currentPartSize) /// Object name to be created, if not provided uses source object name as destination object name. /// optionally can take a key value CopyConditions as well for conditionally attempting copyObject. /// Optional Object metadata to be stored. Defaults to null. + /// Optional source encryption options.Defaults to null. + /// Optional destination encryption options.Defaults to null. /// Optional cancellation token to cancel the operation /// - public async Task CopyObjectAsync(string bucketName, string objectName, string destBucketName, string destObjectName = null, CopyConditions copyConditions = null, Dictionary metadata = null, CancellationToken cancellationToken = default(CancellationToken)) + public async Task CopyObjectAsync(string bucketName, string objectName, string destBucketName, string destObjectName = null, CopyConditions copyConditions = null, Dictionary metadata = null, ServerSideEncryption sseSrc = null, ServerSideEncryption sseDest = null, CancellationToken cancellationToken = default(CancellationToken)) { if (bucketName == null) { @@ -909,9 +956,14 @@ internal byte[] ReadFull(Stream data, int currentPartSize) { destObjectName = objectName; } - + ServerSideEncryption sseGet = sseSrc; + if (sseSrc is SSECopy) + { + SSECopy sseCpy = (SSECopy)sseSrc; + sseGet = sseCpy.CloneToSSEC(); + } // Get Stats on the source object - ObjectStat srcStats = await this.StatObjectAsync(bucketName, objectName, cancellationToken).ConfigureAwait(false); + ObjectStat srcStats = await this.StatObjectAsync(bucketName, objectName, sse:sseGet, cancellationToken:cancellationToken).ConfigureAwait(false); // Copy metadata from the source object if no metadata replace directive Dictionary meta = new Dictionary(); Dictionary m = metadata; @@ -919,6 +971,7 @@ internal byte[] ReadFull(Stream data, int currentPartSize) { m = srcStats.metaData; } + if (m != null) { foreach (var item in m) @@ -941,9 +994,19 @@ internal byte[] ReadFull(Stream data, int currentPartSize) if ((copySize > Constants.MaxSingleCopyObjectSize) || (srcByteRangeSize > 0 && (srcByteRangeSize != srcStats.Size))) - await MultipartCopyUploadAsync(bucketName, objectName, destBucketName, destObjectName, copyConditions, copySize, meta, cancellationToken).ConfigureAwait(false); + await MultipartCopyUploadAsync(bucketName, objectName, destBucketName, destObjectName, copyConditions, copySize, meta,sseSrc, sseDest, cancellationToken).ConfigureAwait(false); else + { + if (sseSrc != null && sseSrc is SSECopy) + { + sseSrc.Marshal(meta); + } + if (sseDest != null) + { + sseDest.Marshal(meta); + } await this.CopyObjectRequestAsync(bucketName, objectName, destBucketName, destObjectName, copyConditions, meta, null, cancellationToken, typeof(CopyObjectResult)).ConfigureAwait(false); + } } /// /// Create the copy request,execute it and @@ -1012,9 +1075,12 @@ private async Task CopyObjectRequestAsync(string bucketName, string obje /// destiantion object name /// copyconditions /// size of copy upload + /// optional metadata on the destination side + /// optional Server-side encryption options + /// optional Server-side encryption options /// optional cancellation token /// - private async Task MultipartCopyUploadAsync(string bucketName, string objectName, string destBucketName, string destObjectName, CopyConditions copyConditions, long copySize,Dictionary metadata = null, CancellationToken cancellationToken=default(CancellationToken)) + private async Task MultipartCopyUploadAsync(string bucketName, string objectName, string destBucketName, string destObjectName, CopyConditions copyConditions, long copySize,Dictionary metadata = null, ServerSideEncryption sseSrc = null, ServerSideEncryption sseDest = null, CancellationToken cancellationToken=default(CancellationToken)) { // For all sizes greater than 5GB or if Copy byte range specified in conditions and byte range larger // than minimum part size (5 MB) do multipart. @@ -1025,8 +1091,14 @@ private async Task MultipartCopyUploadAsync(string bucketName, string objectName double lastPartSize = multiPartInfo.lastPartSize; Part[] totalParts = new Part[(int)partCount]; - // No need to resume upload since this is a server side copy. Just initiate a new upload. - string uploadId = await this.NewMultipartUploadAsync(destBucketName, destObjectName, metadata, cancellationToken).ConfigureAwait(false); + var sseHeaders = new Dictionary(); + if (sseDest != null) + { + sseDest.Marshal(sseHeaders); + } + + // No need to resume upload since this is a Server-side copy. Just initiate a new upload. + string uploadId = await this.NewMultipartUploadAsync(destBucketName, destObjectName, metadata, sseHeaders,cancellationToken).ConfigureAwait(false); // Upload each part double expectedReadSize = partSize; @@ -1046,6 +1118,14 @@ private async Task MultipartCopyUploadAsync(string bucketName, string objectName } Dictionary customHeader = new Dictionary(); customHeader.Add("x-amz-copy-source-range", "bytes=" + partCondition.byteRangeStart.ToString() + "-" + partCondition.byteRangeEnd.ToString()); + if (sseSrc != null && sseSrc is SSECopy) + { + sseSrc.Marshal(customHeader); + } + if (sseDest != null) + { + sseDest.Marshal(customHeader); + } CopyPartResult cpPartResult = (CopyPartResult)await this.CopyObjectRequestAsync(bucketName, objectName, destBucketName, destObjectName, copyConditions, customHeader, resource, cancellationToken, typeof(CopyPartResult)).ConfigureAwait(false); totalParts[partNumber - 1] = new Part() { PartNumber = partNumber, ETag = cpPartResult.ETag, size = (long)expectedReadSize }; diff --git a/Minio/DataModel/ServerSideEncryption.cs b/Minio/DataModel/ServerSideEncryption.cs new file mode 100644 index 000000000..34dd9d32c --- /dev/null +++ b/Minio/DataModel/ServerSideEncryption.cs @@ -0,0 +1,166 @@ +/* + * Minio .NET Library for Amazon S3 Compatible Cloud Storage, (C) 2019 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System; +using System.Text; +using System.Collections.Generic; +using System.Security.Cryptography; + +using Minio.Helper; + +namespace Minio.DataModel +{ + // Type of Server-side encryption + public enum EncryptionType { + SSE_C, + SSE_S3, + SSE_KMS + } + + // ServerSideEncryption interface + public interface ServerSideEncryption + { + // GetType() needs to return the type of Server-side encryption + EncryptionType GetType() ; + // Marshals the Server-side encryption headers into dictionary + void Marshal(Dictionary headers) ; + } + + // Server-side encryption with customer provided keys (SSE-C) + public class SSEC : ServerSideEncryption { + + // secret AES-256 Key + protected byte[] key; + public new EncryptionType GetType() + { + return EncryptionType.SSE_C; + } + + public virtual void Marshal(Dictionary headers) + { + var md5SumStr = utils.getMD5SumStr(this.key); + headers.Add("X-Amz-Server-Side-Encryption-Customer-Algorithm", "AES256"); + headers.Add("X-Amz-Server-Side-Encryption-Customer-Key",Convert.ToBase64String(this.key)) ; + headers.Add("X-Amz-Server-Side-Encryption-Customer-Key-Md5", md5SumStr); + return; + } + + public SSEC(byte[] key) + { + if (key == null || key.Length != 32) + { + throw new ArgumentException("Secret key needs to be a 256 bit AES Key"); + } + this.key = key; + } + } + + // Server-side encryption option for source side SSE-C + // copy operation + public class SSECopy : SSEC + { + public override void Marshal(Dictionary headers) + { + var md5SumStr = utils.getMD5SumStr(this.key); + headers.Add("X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm", "AES256"); + headers.Add("X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key",Convert.ToBase64String(key)) ; + headers.Add("X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5", md5SumStr); + return; + } + public SSECopy(byte[] key) : base(key) + { + } + + public SSEC CloneToSSEC() + { + return new SSEC(this.key); + } + } + + // Server-side encryption with S3 managed encryption keys (SSE-S3) + public class SSES3 : ServerSideEncryption + { + public new EncryptionType GetType() + { + return EncryptionType.SSE_S3; + } + + public virtual void Marshal(Dictionary headers) + { + headers.Add(Constants.SSEGenericHeader, "AES256"); + return; + } + } + + // Server-side encryption with AWS KMS managed keys + public class SSEKMS : ServerSideEncryption + { + // Specifies the customer master key(CMK).Cannot be null + protected string key; + protected Dictionary context; + public SSEKMS(string key, Dictionary context = null) + { + + if (key == "") + { + throw new ArgumentException("KMS Key cannot be empty"); + } + this.key = key; + this.context = context; + } + public new EncryptionType GetType() + { + return EncryptionType.SSE_KMS; + } + + public void Marshal(Dictionary headers) + { + headers.Add(Constants.SSEKMSKeyId, this.key); + headers.Add(Constants.SSEGenericHeader, "aws:kms"); + if (context != null) + { + headers.Add(Constants.SSEKMSContext, this.marshalContext()); + } + return; + } + /// + /// Serialize context into JSON string. + /// + /// Serialized JSON context + private string marshalContext() + { + StringBuilder sb = new StringBuilder(); + + sb.Append("{"); + int i = 0; + int len = this.context.Count; + foreach (KeyValuePair pair in this.context) + { + sb.Append("\"").Append(pair.Key).Append("\""); + sb.Append(":"); + sb.Append("\"").Append(pair.Value).Append("\""); + i += 1; + if (i != len) + { + sb.Append(":"); + } + } + sb.Append("}"); + byte[] contextBytes = System.Text.Encoding.UTF8.GetBytes(sb.ToString() as string); + return Convert.ToBase64String(contextBytes); + } + } + +} diff --git a/Minio/Helper/Constants.cs b/Minio/Helper/Constants.cs index 124107ddb..e995632a9 100644 --- a/Minio/Helper/Constants.cs +++ b/Minio/Helper/Constants.cs @@ -41,5 +41,14 @@ internal static class Constants public static long OptimalReadBufferSize = 1024L * 1024L * 5; public static int DefaultExpiryTime = 7 * 24 * 3600; + + // SSEGenericHeader is the AWS SSE header used for SSE-S3 and SSE-KMS. + public static string SSEGenericHeader = "X-Amz-Server-Side-Encryption"; + + // SSEKMSKeyId is the AWS SSE KMS Key-Id. + public static string SSEKMSKeyId = "X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"; + + // SSEKMSContext is the AWS SSE KMS Context. + public static string SSEKMSContext = "X-Amz-Server-Side-Encryption-Context"; } -} \ No newline at end of file +} diff --git a/Minio/Helper/utils.cs b/Minio/Helper/utils.cs index 08f459309..40cd37d09 100644 --- a/Minio/Helper/utils.cs +++ b/Minio/Helper/utils.cs @@ -259,6 +259,14 @@ public static bool IsValidExpiry(int expiryInt) return (expiryInt > 0) && (expiryInt <= Constants.DefaultExpiryTime); } + internal static string getMD5SumStr(byte[] key) + { + var hashedBytes = System.Security.Cryptography.MD5 + .Create() + .ComputeHash(key); + + return Convert.ToBase64String(hashedBytes); + } private static readonly Lazy> _contentTypeMap = new Lazy>(AddContentTypeMappings); private static IDictionary AddContentTypeMappings() diff --git a/Minio/Minio.csproj b/Minio/Minio.csproj index 9d0f8aa94..bb50ae83b 100644 --- a/Minio/Minio.csproj +++ b/Minio/Minio.csproj @@ -9,7 +9,7 @@ Minio - 2.0.8 + 3.0.0 Minio .NET SDK for Amazon S3 Compatible Cloud Storage. Copyright (c) 2018. All rights reserved. Minio, Inc.