Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fixes GetObject offset=0 problem #646

Merged
merged 4 commits into from
Jun 3, 2022
Merged

Fixes GetObject offset=0 problem #646

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a3fd0cb
Fixes #629 GetObject offset=0 problem
ebozduman May 31, 2022
d9eb5bb
To resolve build issue on Windows
ebozduman May 31, 2022
93b14ea
fix for the windows build issue
ebozduman Jun 3, 2022
cb10051
Forgotten regitlint format fixes
ebozduman Jun 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
150 changes: 98 additions & 52 deletions Minio.Functional.Tests/FunctionalTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
using System.Net;
using System.Net.Http;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
Expand Down Expand Up @@ -459,10 +460,11 @@ internal static async Task ListBuckets_Test(MinioClient minio)

internal static async Task Setup_Test(MinioClient minio, string bucketName)
{
var mbArgs = new MakeBucketArgs()
.WithBucket(bucketName);
var beArgs = new BucketExistsArgs()
.WithBucket(bucketName);
if (await minio.BucketExistsAsync(beArgs)) return;
var mbArgs = new MakeBucketArgs()
.WithBucket(bucketName);
await minio.MakeBucketAsync(mbArgs);
var found = await minio.BucketExistsAsync(beArgs);
Assert.IsTrue(found);
Expand Down Expand Up @@ -4273,69 +4275,113 @@ internal static async Task GetObject_Test2(MinioClient minio)
}
}

internal static async Task GetObject_Test3(MinioClient minio)
internal static async Task GetObject_3_OffsetLength_Tests(MinioClient minio)
// 3 tests will run to check different values of offset and length parameters
// when GetObject api returns part of the object as defined by the offset
// and length parameters. Tests will be reported as GetObject_Test3,
// GetObject_Test4 and GetObject_Test5.
{
var startTime = DateTime.Now;
var bucketName = GetRandomName(15);
var objectName = GetRandomObjectName(10);
string contentType = null;
var tempFileName = "tempFileName";
var args = new Dictionary<string, string>
{
{ "bucketName", bucketName },
{ "objectName", objectName },
{ "contentType", contentType },
{ "size", "1024L" },
{ "length", "10L" }
var tempFileName = "tempFile-" + GetRandomName();
var tempSource = "tempSourceFile-" + GetRandomName();
var offsetLengthTests = new Dictionary<string, List<int>>
{
// list is {offset, length} values
{ "GetObject_Test3", new List<int> { 14, 20 } },
{ "GetObject_Test4", new List<int> { 30, 0 } },
{ "GetObject_Test5", new List<int> { 0, 25 } }
};
try
foreach (var test in offsetLengthTests)
{
await Setup_Test(minio, bucketName);
using (var filestream = rsg.GenerateStreamFromSeed(10 * KB))
var testName = test.Key;
var offsetToStartFrom = test.Value[0];
var lengthToBeRead = test.Value[1];
var args = new Dictionary<string, string>
{
var file_write_size = 10L;
long file_read_size = 0;
var putObjectArgs = new PutObjectArgs()
.WithBucket(bucketName)
.WithObject(objectName)
.WithStreamData(filestream)
.WithObjectSize(filestream.Length)
.WithContentType(contentType);
await minio.PutObjectAsync(putObjectArgs);
{ "bucketName", bucketName },
{ "objectName", objectName },
{ "contentType", contentType },
{ "offset", offsetToStartFrom.ToString() },
{ "length", lengthToBeRead.ToString() }
};
try
{
await Setup_Test(minio, bucketName);

var getObjectArgs = new GetObjectArgs()
.WithBucket(bucketName)
.WithObject(objectName)
.WithOffsetAndLength(1024L, file_write_size)
.WithCallbackStream(stream =>
// Create a file with distintc byte characters to test partial
// get object.
var line = new[] { "abcdefghijklmnopqrstuvwxyz0123456789" };
// abcdefghijklmnopqrstuvwxyz0123456789
// 012345678911234567892123456789312345
// ^1stChr, ^10thChr, ^20thChr, ^30th ^35thChr => characters' sequence
// Example: offset 10 and length 4, the expected size and content
// getObjectAsync will return are 4 and "klmn" respectively.
await File.WriteAllLinesAsync(tempSource, line);

using (var filestream = File.Open(tempSource, FileMode.Open, FileAccess.Read, FileShare.Read))
{
var objectSize = (int)filestream.Length;
var expectedFileSize = lengthToBeRead;
var expectedContent = string.Join("", line).Substring(offsetToStartFrom, expectedFileSize);
if (lengthToBeRead == 0)
{
var fileStream = File.Create(tempFileName);
stream.CopyTo(fileStream);
fileStream.Dispose();
var writtenInfo = new FileInfo(tempFileName);
file_read_size = writtenInfo.Length;
expectedFileSize = objectSize - offsetToStartFrom;
var noOfCtrlChars = 1;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) noOfCtrlChars = 2;

Assert.AreEqual(file_write_size, file_read_size);
File.Delete(tempFileName);
});
expectedContent = string.Join("", line)
.Substring(offsetToStartFrom, expectedFileSize - noOfCtrlChars);
}

await minio.GetObjectAsync(getObjectArgs);
}
long actualFileSize;
var putObjectArgs = new PutObjectArgs()
.WithBucket(bucketName)
.WithObject(objectName)
.WithStreamData(filestream)
.WithObjectSize(objectSize)
.WithContentType(contentType);
await minio.PutObjectAsync(putObjectArgs);

new MintLogger("GetObject_Test3", getObjectSignature, "Tests whether GetObject returns all the data",
TestStatus.PASS, DateTime.Now - startTime, args: args).Log();
}
catch (Exception ex)
{
new MintLogger("GetObject_Test3", getObjectSignature, "Tests whether GetObject returns all the data",
TestStatus.FAIL, DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log();
throw;
}
finally
{
if (File.Exists(tempFileName))
File.Delete(tempFileName);
await TearDown(minio, bucketName);
var getObjectArgs = new GetObjectArgs()
.WithBucket(bucketName)
.WithObject(objectName)
.WithOffsetAndLength(offsetToStartFrom, lengthToBeRead)
.WithCallbackStream(stream =>
{
var fileStream = File.Create(tempFileName);
stream.CopyTo(fileStream);
fileStream.Dispose();
var writtenInfo = new FileInfo(tempFileName);
actualFileSize = writtenInfo.Length;

Assert.AreEqual(expectedFileSize, actualFileSize);

// Checking the content
var actualContent = File.ReadAllText(tempFileName).Replace("\n", "").Replace("\r", "");
Assert.AreEqual(actualContent, expectedContent);
});

await minio.GetObjectAsync(getObjectArgs);
}

new MintLogger(testName, getObjectSignature, "Tests whether GetObject returns all the data",
TestStatus.PASS, DateTime.Now - startTime, args: args).Log();
}
catch (Exception ex)
{
new MintLogger(testName, getObjectSignature, "Tests whether GetObject returns all the data",
TestStatus.FAIL, DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log();
throw;
}
finally
{
if (File.Exists(tempFileName)) File.Delete(tempFileName);
if (File.Exists(tempSource)) File.Delete(tempSource);
await TearDown(minio, bucketName);
}
}
}

Expand Down
6 changes: 5 additions & 1 deletion Minio.Functional.Tests/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,11 @@ public static void Main(string[] args)
// Test GetObjectAsync function
FunctionalTest.GetObject_Test1(minioClient).Wait();
FunctionalTest.GetObject_Test2(minioClient).Wait();
FunctionalTest.GetObject_Test3(minioClient).Wait();
// 3 tests will run to check different values of offset and length parameters
// when GetObject api returns part of the object as defined by the offset
// and length parameters. Tests will be reported as GetObject_Test3,
// GetObject_Test4 and GetObject_Test5.
FunctionalTest.GetObject_3_OffsetLength_Tests(minioClient).Wait();

// Test File GetObject and PutObject functions
FunctionalTest.FGetObject_Test1(minioClient).Wait();
Expand Down
28 changes: 20 additions & 8 deletions Minio/DataModel/ObjectOperationsArgs.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,8 +248,8 @@ internal override void Validate()
throw new ArgumentException(nameof(ObjectOffset) + " and " + nameof(ObjectLength) +
"cannot be less than 0.");
if (ObjectOffset == 0 && ObjectLength == 0)
throw new ArgumentException("One of " + nameof(ObjectOffset) + " or " + nameof(ObjectLength) +
"must be greater than 0.");
throw new ArgumentException("Either " + nameof(ObjectOffset) + " or " + nameof(ObjectLength) +
" must be greater than 0.");
}

Populate();
Expand All @@ -259,7 +259,15 @@ private void Populate()
{
Headers = new Dictionary<string, string>();
if (SSE != null && SSE.GetType().Equals(EncryptionType.SSE_C)) SSE.Marshal(Headers);
if (OffsetLengthSet) Headers["Range"] = "bytes=" + ObjectOffset + "-" + (ObjectOffset + ObjectLength - 1);
if (OffsetLengthSet)
{
// "Range" header accepts byte start index and end index
if (ObjectLength > 0 && ObjectOffset > 0)
Headers["Range"] = "bytes=" + ObjectOffset + "-" + (ObjectOffset + ObjectLength - 1);
else if (ObjectLength == 0 && ObjectOffset > 0)
Headers["Range"] = "bytes=" + ObjectOffset + "-";
else if (ObjectLength > 0 && ObjectOffset == 0) Headers["Range"] = "bytes=0-" + (ObjectLength - 1);
}
}

public StatObjectArgs WithOffsetAndLength(long offset, long length)
Expand Down Expand Up @@ -525,11 +533,15 @@ private void Populate()
Headers = new Dictionary<string, string>();
if (SSE != null && SSE.GetType().Equals(EncryptionType.SSE_C)) SSE.Marshal(Headers);

if (ObjectLength > 0 && ObjectOffset > 0)
Headers["Range"] = "bytes=" + ObjectOffset + "-" + (ObjectOffset + ObjectLength - 1);
else if (ObjectLength == 0 && ObjectOffset > 0)
Headers["Range"] = "bytes=" + ObjectOffset + "-";
else if (ObjectLength > 0 && ObjectOffset == 0) Headers["Range"] = "bytes=-" + (ObjectLength - 1);
if (OffsetLengthSet)
{
// "Range" header accepts byte start index and end index
if (ObjectLength > 0 && ObjectOffset > 0)
Headers["Range"] = "bytes=" + ObjectOffset + "-" + (ObjectOffset + ObjectLength - 1);
else if (ObjectLength == 0 && ObjectOffset > 0)
Headers["Range"] = "bytes=" + ObjectOffset + "-";
else if (ObjectLength > 0 && ObjectOffset == 0) Headers["Range"] = "bytes=0-" + (ObjectLength - 1);
}
}

internal override HttpRequestMessageBuilder BuildRequest(HttpRequestMessageBuilder requestMessageBuilder)
Expand Down
2 changes: 1 addition & 1 deletion Minio/Helper/OperationsHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ private async Task<ObjectStat> getObjectHelper(GetObjectArgs args, CancellationT


/// <summary>
/// private helper method to remove list of objects from bucket
/// private helper method. It returns the specified portion or full object from the bucket
/// </summary>
/// <param name="args">GetObjectArgs Arguments Object encapsulates information like - bucket name, object name etc </param>
/// <param name="objectStat">
Expand Down