Skip to content

Commit

Permalink
Merge branch 'master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
@Lanz86
Lanz86 committed Oct 19, 2022
2 parents c9334cf + 42ff388 commit 9c7907f
Show file tree
Hide file tree
Showing 8 changed files with 192 additions and 50 deletions.
218 changes: 180 additions & 38 deletions Minio.Functional.Tests/FunctionalTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@
using System.Threading.Tasks;
using System.Web;
using System.Xml;
using ICSharpCode.SharpZipLib.Core;
using ICSharpCode.SharpZipLib.Zip;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Minio.DataModel;
using Minio.DataModel.ILM;
Expand Down Expand Up @@ -2393,6 +2395,129 @@ internal static async Task ObjectRetentionAsync_Test1(MinioClient minio)

#endregion

internal static MemoryStream CreateZipFile(string prefix, int nFiles)
{
var outputMemStream = new MemoryStream();
var zipStream = new ZipOutputStream(outputMemStream);

zipStream.SetLevel(3); //0-9, 9 being the highest level of compression
byte[] bytes = null;

for (var i = 0; i <= nFiles; i++)
{
// Make one large, compressible file.
if (i == nFiles) i = 1000000;

var fileName = prefix + "file-" + i + ".bin";
Directory.CreateDirectory(prefix);
var newEntry = new ZipEntry(fileName);
newEntry.DateTime = DateTime.Now;
zipStream.PutNextEntry(newEntry);

bytes = rsg.GenerateStreamFromSeed(i).ToArray();
var inStream = new MemoryStream(bytes);
if (i == 0) StreamUtils.Copy(inStream, zipStream, new byte[128]);
else StreamUtils.Copy(inStream, zipStream, new byte[i * 128]);

inStream.Close();
zipStream.CloseEntry();
}

zipStream.IsStreamOwner = false; // False stops the Close also Closing the underlying stream.
zipStream.Close(); // Must finish the ZipOutputStream before using outputMemStream.

outputMemStream.Position = 0;
outputMemStream.Seek(0, SeekOrigin.Begin);

return outputMemStream;
}

internal static async Task GetObjectS3Zip_Test1(MinioClient minio)
{
var path = "test/small/";
var startTime = DateTime.Now;
var bucketName = GetRandomName(15);
var randomFileName = GetRandomName(15) + ".zip";
var objectName = GetRandomObjectName(15) + ".zip";

var args = new Dictionary<string, string>
{
{ "bucketName", bucketName },
{ "objectName", objectName }
};
try
{
await Setup_Test(minio, bucketName);
const int nFiles = 500;
var memStream = CreateZipFile(path, nFiles);

var putObjectArgs = new PutObjectArgs()
.WithBucket(bucketName)
.WithObject(objectName)
.WithStreamData(memStream)
.WithObjectSize(memStream.Length);
await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false);

var getObjectArgs = new GetObjectArgs()
.WithBucket(bucketName)
.WithFile(randomFileName)
.WithObject(objectName);

var resp = await minio.GetObjectAsync(getObjectArgs).ConfigureAwait(false);

var statArgs = new StatObjectArgs()
.WithBucket(bucketName)
.WithObject(objectName);
var stat = await minio.StatObjectAsync(statArgs).ConfigureAwait(false);

var lOpts = new Dictionary<string, string>();
lOpts.Add("x-minio-extract", "true");

// Test with different prefix values
// prefix value="", expected number of files listed=1
var prefix = "";
ListObjects_Test(minio, bucketName, prefix, 1, true, headers: lOpts);

// prefix value="/", expected number of files listed=nFiles+1
prefix = objectName + "/";
ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts);

// prefix value="/test", expected number of files listed=nFiles + 1
prefix = objectName + "/test";
ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts);

// prefix value="/test/", expected number of files listed=nFiles+1
prefix = objectName + "/test/";
ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts);

// prefix value="/test", expected number of files listed=nFiles+1
prefix = objectName + "/test/small";
ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts);

// prefix value="/test", expected number of files listed=nFiles+1
prefix = objectName + "/test/small/";
ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts);

// prefix value="/test", expected number of files listed=1
prefix = objectName + "/test/small/" + "file-1.bin";
ListObjects_Test(minio, bucketName, prefix, 1, true, headers: lOpts);

new MintLogger("GetObjectS3Zip_Test1", getObjectSignature, "Tests s3Zip files", TestStatus.PASS,
DateTime.Now - startTime, args: args).Log();
}
catch (Exception ex)
{
new MintLogger("GetObjectS3Zip_Test1", getObjectSignature, "Tests s3Zip files", TestStatus.FAIL,
DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log();
throw;
}
finally
{
File.Delete(randomFileName);
Directory.Delete(path.Split("/")[0], true);
await TearDown(minio, bucketName);
}
}

#region Bucket Notifications

Expand Down Expand Up @@ -2543,7 +2668,8 @@ internal static async Task ListenBucketNotificationsAsync_Test2(MinioClient mini
var startTime = DateTime.Now;
var events = new List<EventType>();
events.Add(EventType.ObjectCreatedAll);
var rxEvents = new List<NotificationEvent>();
var rxEventData = new MinioNotificationRaw("");
var rxEventsList = new List<NotificationEvent>();
IDisposable subscription = null;
var bucketName = GetRandomName(15);
var contentType = "application/json";
Expand Down Expand Up @@ -2582,7 +2708,7 @@ void Notify(MinioNotificationRaw data)
var notification = JsonConvert.DeserializeObject<MinioNotification>(data.json);
if (notification is not { Records: { } }) return;

foreach (var @event in notification.Records) rxEvents.Add(@event);
foreach (var @event in notification.Records) rxEventsList.Add(@event);
}

var listenArgs = new ListenBucketNotificationsArgs()
Expand All @@ -2591,10 +2717,18 @@ void Notify(MinioNotificationRaw data)
var observable = minio.ListenBucketNotificationsAsync(listenArgs);

subscription = observable.Subscribe(
ev => Notify(ev),
ev =>
{
rxEventData = ev;
Notify(rxEventData);
},
ex => throw new Exception($"OnError: {ex.Message}"),
() => throw new Exception("STOPPED LISTENING FOR BUCKET NOTIFICATIONS\n"));

// Sleep to give enough time for the subscriber to be ready
var sleepTime = 1000; // Milliseconds
Thread.Sleep(sleepTime);

var modelJson = "{\"test\": \"test\"}";
await using var stream = ToStream(modelJson);
var putObjectArgs = new PutObjectArgs()
Expand All @@ -2608,17 +2742,17 @@ void Notify(MinioNotificationRaw data)

// Waits until the Put event is detected
// Times out if the event is not caught in 3 seconds
var timeoutDuration = 3; // seconds
var timeout = 3000; // Milliseconds
var waitTime = 25; // Milliseconds
var stTime = DateTime.UtcNow;
var timeout = TimeSpan.FromSeconds(timeoutDuration);
while (rxEvents.Count < 1)
while (string.IsNullOrEmpty(rxEventData.json))
{
await Task.Delay(25);
if (DateTime.UtcNow - stTime >= timeout)
await Task.Delay(waitTime);
if ((DateTime.UtcNow - stTime).TotalMilliseconds >= timeout)
throw new Exception("Timeout: while waiting for events");
}

foreach (var ev in rxEvents) Assert.AreEqual("s3:ObjectCreated:Put", ev.eventName);
foreach (var ev in rxEventsList) Assert.AreEqual("s3:ObjectCreated:Put", ev.eventName);

new MintLogger(nameof(ListenBucketNotificationsAsync_Test2),
listenBucketNotificationsSignature,
Expand Down Expand Up @@ -2648,7 +2782,7 @@ internal static async Task ListenBucketNotificationsAsync_Test3(MinioClient mini
var startTime = DateTime.Now;
var events = new List<EventType>();
events.Add(EventType.ObjectCreatedAll);
var rxEventsData = new MinioNotificationRaw("");
var rxEventData = new MinioNotificationRaw("");
IDisposable disposable = null;
var bucketName = GetRandomName(15);
var suffix = ".json";
Expand Down Expand Up @@ -2677,23 +2811,6 @@ internal static async Task ListenBucketNotificationsAsync_Test3(MinioClient mini
.WithSuffix(suffix)
.WithEvents(events);

var notifications = minio.ListenBucketNotificationsAsync(notificationsArgs);

var testState = "fail";
Exception exception = null;
disposable = notifications.Subscribe(
x =>
{
rxEventsData = x;
testState = "pass";
},
ex =>
{
exception = ex;
testState = "fail";
},
() => { testState = "completed"; });

var modelJson = "{\"test\": \"test\"}";
var stream = new MemoryStream();
var writer = new StreamWriter(stream);
Expand All @@ -2708,24 +2825,45 @@ internal static async Task ListenBucketNotificationsAsync_Test3(MinioClient mini
.WithStreamData(stream)
.WithObjectSize(stream.Length);

Exception exception = null;
var notifications = minio.ListenBucketNotificationsAsync(notificationsArgs);
disposable = notifications.Subscribe(
x => { rxEventData = x; },
ex => { exception = ex; },
() => { });

// Sleep to give enough time for the subscriber to be ready
var sleepTime = 1000; // Milliseconds
Thread.Sleep(sleepTime);

await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false);
Thread.Sleep(1000);

if (testState == "pass")
var stTime = DateTime.UtcNow;
var waitTime = 25; // Milliseconds
var timeout = 3000; // Milliseconds
while (string.IsNullOrEmpty(rxEventData.json))
{
await Task.Delay(waitTime);
if ((DateTime.UtcNow - stTime).TotalMilliseconds >= timeout)
throw new Exception("Timeout: while waiting for events");
}

if (!string.IsNullOrEmpty(rxEventData.json))
{
Assert.IsTrue(rxEventsData.json.Contains("\"eventName\":\"s3:ObjectCreated:Put\""));
var notification = JsonConvert.DeserializeObject<MinioNotification>(rxEventData.json);
Assert.IsTrue(notification.Records[0].eventName.Equals("s3:ObjectCreated:Put"));
new MintLogger(nameof(ListenBucketNotificationsAsync_Test3),
listenBucketNotificationsSignature,
"Tests whether ListenBucketNotifications passes for no event processing",
TestStatus.PASS, DateTime.Now - startTime, args: args).Log();
}
else if (testState == "fail")
else if (exception != null)
{
throw exception;
}
else if (testState == "completed")
else
{
throw new Exception("Bucket notification completed without catching the event");
throw new Exception("Missed Event: Bucket notification failed.");
}
}
catch (Exception ex)
Expand Down Expand Up @@ -4963,13 +5101,14 @@ internal static async Task ListObjectVersions_Test1(MinioClient minio)


internal static void ListObjects_Test(MinioClient minio, string bucketName, string prefix, int numObjects,
bool recursive = true, bool versions = false)
bool recursive = true, bool versions = false, Dictionary<string, string> headers = null)
{
var startTime = DateTime.Now;
var count = 0;
var args = new ListObjectsArgs()
.WithBucket(bucketName)
.WithPrefix(prefix)
.WithHeaders(headers)
.WithRecursive(recursive)
.WithVersions(versions);
if (!versions)
Expand All @@ -4978,11 +5117,11 @@ internal static async Task ListObjectVersions_Test1(MinioClient minio)
var subscription = observable.Subscribe(
item =>
{
Assert.IsTrue(item.Key.StartsWith(prefix));
count += 1;
if (!string.IsNullOrEmpty(prefix)) Assert.IsTrue(item.Key.StartsWith(prefix));
count++;
},
ex => throw ex,
() => { Assert.AreEqual(count, numObjects); });
() => { ; });
}
else
{
Expand All @@ -4994,8 +5133,11 @@ internal static async Task ListObjectVersions_Test1(MinioClient minio)
count += 1;
},
ex => throw ex,
() => { Assert.AreEqual(count, numObjects); });
() => { ; });
}

Thread.Sleep(1000);
Assert.AreEqual(numObjects, count);
}

#endregion
Expand Down
1 change: 1 addition & 0 deletions Minio.Functional.Tests/Minio.Functional.Tests.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="MSTest.TestFramework" Version="1.3.2" />
<PackageReference Include="SharpZipLib" Version="1.3.3" />
<PackageReference Include="System.Net.Http" Version="4.3.4" />
<ProjectReference Include="..\Minio\Minio.csproj" />
</ItemGroup>
Expand Down
3 changes: 3 additions & 0 deletions Minio.Functional.Tests/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,9 @@ public static void Main(string[] args)
// Test SetPolicyAsync function
FunctionalTest.SetBucketPolicy_Test1(minioClient).Wait();

// Test S3Zip function
FunctionalTest.GetObjectS3Zip_Test1(minioClient).Wait();

// Test Presigned Get/Put operations
FunctionalTest.PresignedGetObject_Test1(minioClient).Wait();
FunctionalTest.PresignedGetObject_Test2(minioClient).Wait();
Expand Down
1 change: 1 addition & 0 deletions Minio/ApiEndpoints/BucketOperations.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,7 @@ public IObservable<Item> ListObjectsAsync(ListObjectsArgs args, CancellationToke
.WithContinuationToken(nextContinuationToken)
.WithMarker(marker)
.WithListObjectsV1(!args.UseV2)
.WithHeaders(args.Headers)
.WithVersionIdMarker(versionIdMarker);
if (args.Versions)
{
Expand Down
2 changes: 0 additions & 2 deletions Minio/ApiEndpoints/IObjectOperations.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,6 @@ public interface IObjectOperations
/// <exception cref="InvalidBucketNameException">When bucket name is invalid</exception>
/// <exception cref="InvalidObjectNameException">When object name is invalid</exception>
/// <exception cref="BucketNotFoundException">When bucket is not found</exception>
/// <exception cref="ObjectNotFoundException">When object is not found</exception>
Task RemoveObjectAsync(RemoveObjectArgs args, CancellationToken cancellationToken = default);

/// <summary>
Expand All @@ -154,7 +153,6 @@ public interface IObjectOperations
/// <exception cref="InvalidObjectNameException">When object name is invalid</exception>
/// <exception cref="BucketNotFoundException">When bucket is not found</exception>
/// <exception cref="NotImplementedException">When a functionality or extension is not implemented</exception>
/// <exception cref="ObjectNotFoundException">When object is not found</exception>
Task<IObservable<DeleteError>> RemoveObjectsAsync(RemoveObjectsArgs args,
CancellationToken cancellationToken = default);

Expand Down
8 changes: 2 additions & 6 deletions Minio/DataModel/BucketOperationsArgs.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,12 +187,8 @@ public GetObjectListArgs WithListObjectsV1(bool useV1)

internal override HttpRequestMessageBuilder BuildRequest(HttpRequestMessageBuilder requestMessageBuilder)
{
// using System.Web; Not sure if we need to add query parameters like this vs. requestMessageBuilder.AddQueryParameter()
// var query = HttpUtility.ParseQueryString(string.Empty);
// query["foo"] = "bar<>&-baz";
// query["bar"] = "bazinga";
// string queryString = query.ToString(); {

foreach (var h in Headers)
requestMessageBuilder.AddOrUpdateHeaderParameter(h.Key, h.Value);

requestMessageBuilder.AddQueryParameter("delimiter", Delimiter);
requestMessageBuilder.AddQueryParameter("max-keys", "1000");
Expand Down
Loading

0 comments on commit 9c7907f

Please sign in to comment.