From a3fd0cbb5fa8282df447bf926725b2567e425624 Mon Sep 17 00:00:00 2001 From: Ersan Date: Tue, 31 May 2022 04:13:00 -0700 Subject: [PATCH 1/4] Fixes #629 GetObject offset=0 problem --- Minio.Functional.Tests/FunctionalTest.cs | 143 +++++++++++++++-------- Minio.Functional.Tests/Program.cs | 6 +- Minio/DataModel/ObjectOperationsArgs.cs | 28 +++-- Minio/Helper/OperationsHelper.cs | 2 +- 4 files changed, 119 insertions(+), 60 deletions(-) diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index 4462e7248..25e88750b 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -459,10 +459,11 @@ internal static async Task ListBuckets_Test(MinioClient minio) internal static async Task Setup_Test(MinioClient minio, string bucketName) { - var mbArgs = new MakeBucketArgs() - .WithBucket(bucketName); var beArgs = new BucketExistsArgs() .WithBucket(bucketName); + if (await minio.BucketExistsAsync(beArgs)) return; + var mbArgs = new MakeBucketArgs() + .WithBucket(bucketName); await minio.MakeBucketAsync(mbArgs); var found = await minio.BucketExistsAsync(beArgs); Assert.IsTrue(found); @@ -4273,69 +4274,111 @@ internal static async Task GetObject_Test2(MinioClient minio) } } - internal static async Task GetObject_Test3(MinioClient minio) + internal static async Task GetObject_3_OffsetLength_Tests(MinioClient minio) + // 3 tests will run to check different values of offset and length parameters + // when GetObject api returns part of the object as defined by the offset + // and length parameters. Tests will be reported as GetObject_Test3, + // GetObject_Test4 and GetObject_Test5. { var startTime = DateTime.Now; var bucketName = GetRandomName(15); var objectName = GetRandomObjectName(10); string contentType = null; var tempFileName = "tempFileName"; - var args = new Dictionary + var offsetLengthTests = new Dictionary> { - { "bucketName", bucketName }, - { "objectName", objectName }, - { "contentType", contentType }, - { "size", "1024L" }, - { "length", "10L" } + // list is {offset, length} values + { "GetObject_Test3", new List { 14, 20 } }, + { "GetObject_Test4", new List { 30, 0 } }, + { "GetObject_Test5", new List { 0, 25 } } }; - try + foreach (var test in offsetLengthTests) { - await Setup_Test(minio, bucketName); - using (var filestream = rsg.GenerateStreamFromSeed(10 * KB)) + var testName = test.Key; + var offsetToStartFrom = test.Value[0]; + var lengthToBeRead = test.Value[1]; + var args = new Dictionary { - var file_write_size = 10L; - long file_read_size = 0; - var putObjectArgs = new PutObjectArgs() - .WithBucket(bucketName) - .WithObject(objectName) - .WithStreamData(filestream) - .WithObjectSize(filestream.Length) - .WithContentType(contentType); - await minio.PutObjectAsync(putObjectArgs); + { "bucketName", bucketName }, + { "objectName", objectName }, + { "contentType", contentType }, + { "offset", offsetToStartFrom.ToString() }, + { "length", lengthToBeRead.ToString() } + }; + try + { + await Setup_Test(minio, bucketName); - var getObjectArgs = new GetObjectArgs() - .WithBucket(bucketName) - .WithObject(objectName) - .WithOffsetAndLength(1024L, file_write_size) - .WithCallbackStream(stream => + // Create a file with distintc byte characters to test partial + // get object. + var tempSource = "tempSourceFile"; + var line = new[] { "abcdefghijklmnopqrstuvwxyz0123456789" }; + // abcdefghijklmnopqrstuvwxyz0123456789 + // 012345678911234567892123456789312345 + // ^1stChr, ^10thChr, ^20thChr, ^30th ^35thChr => characters' sequence + // Example: offset 10 and length 4, the expected size and content + // getObjectAsync will return are 4 and "klmn" respectively. + await File.WriteAllLinesAsync(tempSource, line); + + using (var filestream = File.OpenRead(tempSource)) + { + var objectSize = (int)filestream.Length; + var expectedFileSize = lengthToBeRead; + var expectedContent = string.Join("", line).Substring(offsetToStartFrom, expectedFileSize); + if (lengthToBeRead == 0) { - var fileStream = File.Create(tempFileName); - stream.CopyTo(fileStream); - fileStream.Dispose(); - var writtenInfo = new FileInfo(tempFileName); - file_read_size = writtenInfo.Length; + expectedFileSize = objectSize - offsetToStartFrom; + expectedContent = string.Join("", line).Substring(offsetToStartFrom, expectedFileSize - 1); + } - Assert.AreEqual(file_write_size, file_read_size); - File.Delete(tempFileName); - }); + long actualFileSize; + var putObjectArgs = new PutObjectArgs() + .WithBucket(bucketName) + .WithObject(objectName) + .WithStreamData(filestream) + .WithObjectSize(objectSize) + .WithContentType(contentType); + await minio.PutObjectAsync(putObjectArgs); - await minio.GetObjectAsync(getObjectArgs); - } + var getObjectArgs = new GetObjectArgs() + .WithBucket(bucketName) + .WithObject(objectName) + .WithOffsetAndLength(offsetToStartFrom, lengthToBeRead) + .WithCallbackStream(stream => + { + var fileStream = File.Create(tempFileName); + stream.CopyTo(fileStream); + fileStream.Dispose(); + var writtenInfo = new FileInfo(tempFileName); + actualFileSize = writtenInfo.Length; + + Assert.AreEqual(expectedFileSize, actualFileSize); + + // Checking the content + var actualContent = File.ReadAllText(tempFileName).Replace("\n", "").Replace("\r", ""); + Assert.AreEqual(actualContent, expectedContent); + File.Delete(tempFileName); + File.Delete(tempSource); + }); + + await minio.GetObjectAsync(getObjectArgs); + } - new MintLogger("GetObject_Test3", getObjectSignature, "Tests whether GetObject returns all the data", - TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); - } - catch (Exception ex) - { - new MintLogger("GetObject_Test3", getObjectSignature, "Tests whether GetObject returns all the data", - TestStatus.FAIL, DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log(); - throw; - } - finally - { - if (File.Exists(tempFileName)) - File.Delete(tempFileName); - await TearDown(minio, bucketName); + new MintLogger(testName, getObjectSignature, "Tests whether GetObject returns all the data", + TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); + } + catch (Exception ex) + { + new MintLogger(testName, getObjectSignature, "Tests whether GetObject returns all the data", + TestStatus.FAIL, DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log(); + throw; + } + finally + { + if (File.Exists(tempFileName)) + File.Delete(tempFileName); + await TearDown(minio, bucketName); + } } } diff --git a/Minio.Functional.Tests/Program.cs b/Minio.Functional.Tests/Program.cs index 16b1bc885..04f31e973 100644 --- a/Minio.Functional.Tests/Program.cs +++ b/Minio.Functional.Tests/Program.cs @@ -134,7 +134,11 @@ public static void Main(string[] args) // Test GetObjectAsync function FunctionalTest.GetObject_Test1(minioClient).Wait(); FunctionalTest.GetObject_Test2(minioClient).Wait(); - FunctionalTest.GetObject_Test3(minioClient).Wait(); + // 3 tests will run to check different values of offset and length parameters + // when GetObject api returns part of the object as defined by the offset + // and length parameters. Tests will be reported as GetObject_Test3, + // GetObject_Test4 and GetObject_Test5. + FunctionalTest.GetObject_3_OffsetLength_Tests(minioClient).Wait(); // Test File GetObject and PutObject functions FunctionalTest.FGetObject_Test1(minioClient).Wait(); diff --git a/Minio/DataModel/ObjectOperationsArgs.cs b/Minio/DataModel/ObjectOperationsArgs.cs index 80d53664e..2886c3792 100644 --- a/Minio/DataModel/ObjectOperationsArgs.cs +++ b/Minio/DataModel/ObjectOperationsArgs.cs @@ -248,8 +248,8 @@ internal override void Validate() throw new ArgumentException(nameof(ObjectOffset) + " and " + nameof(ObjectLength) + "cannot be less than 0."); if (ObjectOffset == 0 && ObjectLength == 0) - throw new ArgumentException("One of " + nameof(ObjectOffset) + " or " + nameof(ObjectLength) + - "must be greater than 0."); + throw new ArgumentException("Either " + nameof(ObjectOffset) + " or " + nameof(ObjectLength) + + " must be greater than 0."); } Populate(); @@ -259,7 +259,15 @@ private void Populate() { Headers = new Dictionary(); if (SSE != null && SSE.GetType().Equals(EncryptionType.SSE_C)) SSE.Marshal(Headers); - if (OffsetLengthSet) Headers["Range"] = "bytes=" + ObjectOffset + "-" + (ObjectOffset + ObjectLength - 1); + if (OffsetLengthSet) + { + // "Range" header accepts byte start index and end index + if (ObjectLength > 0 && ObjectOffset > 0) + Headers["Range"] = "bytes=" + ObjectOffset + "-" + (ObjectOffset + ObjectLength - 1); + else if (ObjectLength == 0 && ObjectOffset > 0) + Headers["Range"] = "bytes=" + ObjectOffset + "-"; + else if (ObjectLength > 0 && ObjectOffset == 0) Headers["Range"] = "bytes=0-" + (ObjectLength - 1); + } } public StatObjectArgs WithOffsetAndLength(long offset, long length) @@ -525,11 +533,15 @@ private void Populate() Headers = new Dictionary(); if (SSE != null && SSE.GetType().Equals(EncryptionType.SSE_C)) SSE.Marshal(Headers); - if (ObjectLength > 0 && ObjectOffset > 0) - Headers["Range"] = "bytes=" + ObjectOffset + "-" + (ObjectOffset + ObjectLength - 1); - else if (ObjectLength == 0 && ObjectOffset > 0) - Headers["Range"] = "bytes=" + ObjectOffset + "-"; - else if (ObjectLength > 0 && ObjectOffset == 0) Headers["Range"] = "bytes=-" + (ObjectLength - 1); + if (OffsetLengthSet) + { + // "Range" header accepts byte start index and end index + if (ObjectLength > 0 && ObjectOffset > 0) + Headers["Range"] = "bytes=" + ObjectOffset + "-" + (ObjectOffset + ObjectLength - 1); + else if (ObjectLength == 0 && ObjectOffset > 0) + Headers["Range"] = "bytes=" + ObjectOffset + "-"; + else if (ObjectLength > 0 && ObjectOffset == 0) Headers["Range"] = "bytes=0-" + (ObjectLength - 1); + } } internal override HttpRequestMessageBuilder BuildRequest(HttpRequestMessageBuilder requestMessageBuilder) diff --git a/Minio/Helper/OperationsHelper.cs b/Minio/Helper/OperationsHelper.cs index f03bb3d24..e6b598a4e 100644 --- a/Minio/Helper/OperationsHelper.cs +++ b/Minio/Helper/OperationsHelper.cs @@ -95,7 +95,7 @@ private async Task getObjectHelper(GetObjectArgs args, CancellationT /// - /// private helper method to remove list of objects from bucket + /// private helper method. It returns the specified portion or full object from the bucket /// /// GetObjectArgs Arguments Object encapsulates information like - bucket name, object name etc /// From d9eb5bbb9327d79c65c5ee391f6608b113467341 Mon Sep 17 00:00:00 2001 From: Ersan Date: Tue, 31 May 2022 10:52:25 -0700 Subject: [PATCH 2/4] To resolve build issue on Windows --- Minio.Functional.Tests/FunctionalTest.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index 25e88750b..540c5d080 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -4284,7 +4284,7 @@ internal static async Task GetObject_3_OffsetLength_Tests(MinioClient minio) var bucketName = GetRandomName(15); var objectName = GetRandomObjectName(10); string contentType = null; - var tempFileName = "tempFileName"; + var tempFileName = "tempFile-" + GetRandomName(5); var offsetLengthTests = new Dictionary> { // list is {offset, length} values @@ -4311,7 +4311,7 @@ internal static async Task GetObject_3_OffsetLength_Tests(MinioClient minio) // Create a file with distintc byte characters to test partial // get object. - var tempSource = "tempSourceFile"; + var tempSource = "tempSourceFile-" + GetRandomName(5); var line = new[] { "abcdefghijklmnopqrstuvwxyz0123456789" }; // abcdefghijklmnopqrstuvwxyz0123456789 // 012345678911234567892123456789312345 From 93b14ea15378936f15f0f3bae728c7a65cb0e9ed Mon Sep 17 00:00:00 2001 From: Ersan Date: Fri, 3 Jun 2022 03:20:24 -0700 Subject: [PATCH 3/4] fix for the windows build issue --- Minio.Functional.Tests/FunctionalTest.cs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index 540c5d080..9fe5e00d4 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -32,6 +32,7 @@ using System.Threading.Tasks; using System.Web; using System.Xml; +using System.Runtime.InteropServices; using Microsoft.VisualStudio.TestTools.UnitTesting; using Minio.DataModel; using Minio.DataModel.ILM; @@ -4285,6 +4286,7 @@ internal static async Task GetObject_3_OffsetLength_Tests(MinioClient minio) var objectName = GetRandomObjectName(10); string contentType = null; var tempFileName = "tempFile-" + GetRandomName(5); + var tempSource = "tempSourceFile-" + GetRandomName(5); var offsetLengthTests = new Dictionary> { // list is {offset, length} values @@ -4311,7 +4313,6 @@ internal static async Task GetObject_3_OffsetLength_Tests(MinioClient minio) // Create a file with distintc byte characters to test partial // get object. - var tempSource = "tempSourceFile-" + GetRandomName(5); var line = new[] { "abcdefghijklmnopqrstuvwxyz0123456789" }; // abcdefghijklmnopqrstuvwxyz0123456789 // 012345678911234567892123456789312345 @@ -4320,7 +4321,7 @@ internal static async Task GetObject_3_OffsetLength_Tests(MinioClient minio) // getObjectAsync will return are 4 and "klmn" respectively. await File.WriteAllLinesAsync(tempSource, line); - using (var filestream = File.OpenRead(tempSource)) + using (var filestream = File.Open(tempSource, FileMode.Open, FileAccess.Read, FileShare.Read)) { var objectSize = (int)filestream.Length; var expectedFileSize = lengthToBeRead; @@ -4328,7 +4329,10 @@ internal static async Task GetObject_3_OffsetLength_Tests(MinioClient minio) if (lengthToBeRead == 0) { expectedFileSize = objectSize - offsetToStartFrom; - expectedContent = string.Join("", line).Substring(offsetToStartFrom, expectedFileSize - 1); + var noOfCtrlChars = 1; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) noOfCtrlChars = 2; + + expectedContent = string.Join("", line).Substring(offsetToStartFrom, expectedFileSize - noOfCtrlChars); } long actualFileSize; @@ -4357,8 +4361,6 @@ internal static async Task GetObject_3_OffsetLength_Tests(MinioClient minio) // Checking the content var actualContent = File.ReadAllText(tempFileName).Replace("\n", "").Replace("\r", ""); Assert.AreEqual(actualContent, expectedContent); - File.Delete(tempFileName); - File.Delete(tempSource); }); await minio.GetObjectAsync(getObjectArgs); @@ -4375,8 +4377,8 @@ internal static async Task GetObject_3_OffsetLength_Tests(MinioClient minio) } finally { - if (File.Exists(tempFileName)) - File.Delete(tempFileName); + if (File.Exists(tempFileName)) File.Delete(tempFileName); + if (File.Exists(tempSource)) File.Delete(tempSource); await TearDown(minio, bucketName); } } From cb100516d8e70aa179bc9492049b74bcea26403b Mon Sep 17 00:00:00 2001 From: Ersan Date: Fri, 3 Jun 2022 03:35:38 -0700 Subject: [PATCH 4/4] Forgotten regitlint format fixes --- Minio.Functional.Tests/FunctionalTest.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index 9fe5e00d4..b287ef562 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -25,6 +25,7 @@ using System.Net; using System.Net.Http; using System.Reflection; +using System.Runtime.InteropServices; using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; @@ -32,7 +33,6 @@ using System.Threading.Tasks; using System.Web; using System.Xml; -using System.Runtime.InteropServices; using Microsoft.VisualStudio.TestTools.UnitTesting; using Minio.DataModel; using Minio.DataModel.ILM; @@ -4285,8 +4285,8 @@ internal static async Task GetObject_3_OffsetLength_Tests(MinioClient minio) var bucketName = GetRandomName(15); var objectName = GetRandomObjectName(10); string contentType = null; - var tempFileName = "tempFile-" + GetRandomName(5); - var tempSource = "tempSourceFile-" + GetRandomName(5); + var tempFileName = "tempFile-" + GetRandomName(); + var tempSource = "tempSourceFile-" + GetRandomName(); var offsetLengthTests = new Dictionary> { // list is {offset, length} values @@ -4332,7 +4332,8 @@ internal static async Task GetObject_3_OffsetLength_Tests(MinioClient minio) var noOfCtrlChars = 1; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) noOfCtrlChars = 2; - expectedContent = string.Join("", line).Substring(offsetToStartFrom, expectedFileSize - noOfCtrlChars); + expectedContent = string.Join("", line) + .Substring(offsetToStartFrom, expectedFileSize - noOfCtrlChars); } long actualFileSize;