Skip to content

Sto 255 admin module error #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Jul 22, 2022
Merged

Sto 255 admin module error #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
e81c4dc
Fixs to StorageAdminService
lillie-dae Jul 18, 2022
b44f8ab
fix up user credential creation
neildsouth Jul 19, 2022
d408d27
added optional payloadIds to CreateUserAsync
neildsouth Jul 20, 2022
cd18feb
fixup of the policy to restrict access to payloadId only
neildsouth Jul 20, 2022
2d70f88
fix for removed package
neildsouth Jul 20, 2022
789cbc5
fix policy to allow writes
neildsouth Jul 21, 2022
a0a2519
change interface to remove generic read/write permissions
neildsouth Jul 21, 2022
dbad738
removed references to mc.exe
neildsouth Jul 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/AssemblyInfo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* limitations under the License.
*/

//------------------------------------------------------------------------------
//------------------------------------------------------------------------------
// <auto-generated>
// This code was generated by GitVersion.
//
Expand Down
2 changes: 2 additions & 0 deletions src/Plugins/AWSS3/StorageAdminService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,13 @@
using System.Threading.Tasks;
using Amazon.SecurityToken.Model;
using Monai.Deploy.Storage.API;
using Monai.Deploy.Storage.S3Policy.Policies;

namespace Monai.Deploy.Storage.AWSS3
{
public class StorageAdminService : IStorageAdminService
{
public Task<Credentials> CreateUserAsync(string username, AccessPermissions permissions, string[] bucketNames) => throw new NotImplementedException();
public Task<Credentials> CreateUserAsync(string username, PolicyRequest[] policyRequests) => throw new NotImplementedException();
}
}
Binary file added src/Plugins/MinIO/Mc/mc.exe
View file
Open in desktop
Binary file not shown.
1 change: 1 addition & 0 deletions src/Plugins/MinIO/Monai.Deploy.Storage.MinIO.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,4 +63,5 @@
<ItemGroup>
<Compile Include="..\..\AssemblyInfo.cs" Link="AssemblyInfo.cs" />
</ItemGroup>

</Project>
252 changes: 144 additions & 108 deletions src/Plugins/MinIO/StorageAdminService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,23 @@
using Microsoft.Extensions.Options;
using Monai.Deploy.Storage.API;
using Monai.Deploy.Storage.Configuration;
using Monai.Deploy.Storage.Minio.Extensions;
using Monai.Deploy.Storage.S3Policy;
using Monai.Deploy.Storage.S3Policy.Policies;

namespace Monai.Deploy.Storage.MinIO
{
public class StorageAdminService : IStorageAdminService
{
private const string UserCommand = "admin user list minio";
private readonly string _executableLocation;
private readonly string _serviceName;
private readonly string _temporaryFilePath;
private readonly string _endpoint;
private readonly string _accessKey;
private readonly string _secretKey;
private readonly IFileSystem _fileSystem;
private readonly string _set_connection_cmd;
private readonly string _get_connections_cmd;
private readonly string _get_users_cmd;

public StorageAdminService(IOptions<StorageServiceConfiguration> options, ILogger<MinIoStorageService> logger, IFileSystem fileSystem)
{
Expand All @@ -48,56 +52,34 @@ public StorageAdminService(IOptions<StorageServiceConfiguration> options, ILogge
_executableLocation = options.Value.Settings[ConfigurationKeys.McExecutablePath];
_serviceName = options.Value.Settings[ConfigurationKeys.McServiceName];
_temporaryFilePath = _fileSystem.Path.GetTempPath();
_endpoint = options.Value.Settings[ConfigurationKeys.EndPoint];
_accessKey = options.Value.Settings[ConfigurationKeys.AccessKey];
_secretKey = options.Value.Settings[ConfigurationKeys.AccessToken];
_set_connection_cmd = $"alias set {_serviceName} http://{_endpoint} {_accessKey} {_secretKey}";
_get_connections_cmd = "alias list";
_get_users_cmd = $"admin user list {_serviceName}";
}

public async Task<Credentials> CreateUserAsync(string username, AccessPermissions permissions, string[] bucketNames)
private static void ValidateConfiguration(StorageServiceConfiguration configuration)
{
Guard.Against.NullOrWhiteSpace(username, nameof(username));
Guard.Against.NullOrEmpty(bucketNames, nameof(bucketNames));

if (await UserAlreadyExistsAsync(username).ConfigureAwait(false))
{
throw new InvalidOperationException("User already exists");
}

var credentials = new Credentials();
var userSecretKey = Convert.ToBase64String(Guid.NewGuid().ToByteArray());
credentials.SecretAccessKey = userSecretKey;
credentials.AccessKeyId = username;

var result = await Execute(CreateUserCmd(username, userSecretKey)).ConfigureAwait(false);

if (!result.Any(r => r.Contains($"Added user `{username}` successfully.")))
{
await RemoveUserAsync(username).ConfigureAwait(false);
throw new InvalidOperationException($"Unknown Output {result.SelectMany(e => e)}");
}

var minioPolicies = new List<string>()
{
permissions.GetString()
};

var policyRequests = bucketNames.Select(
bucket => new PolicyRequest(bucket, "")
).ToArray();
Guard.Against.Null(configuration, nameof(configuration));

await CreatePolicyAsync(policyRequests, username).ConfigureAwait(false);
var setPolicyResult = await SetPolicyAsync(IdentityType.User, minioPolicies, credentials.AccessKeyId).ConfigureAwait(false);
if (!setPolicyResult)
foreach (var key in ConfigurationKeys.McRequiredKeys)
{
await RemoveUserAsync(username).ConfigureAwait(false);
throw new InvalidOperationException("Failed to set policy, user has been removed");
if (!configuration.Settings.ContainsKey(key))
{
throw new ConfigurationException($"IMinioAdmin Shell is missing configuration for {key}.");
}
}

return credentials;
}

private async Task<bool> SetPolicyAsync(IdentityType policyType, List<string> policies, string itemName)
private string CreateUserCmd(string username, string secretKey) => $"admin user add {_serviceName} {username} {secretKey}";

public async Task<bool> SetPolicyAsync(IdentityType policyType, List<string> policies, string itemName)
{
var policiesStr = string.Join(',', policies);
var setPolicyCmd = $"admin policy set {_serviceName} {policiesStr} {policyType.ToString().ToLower()}={itemName}";
var result = await Execute(setPolicyCmd).ConfigureAwait(false);
var result = await ExecuteAsync(setPolicyCmd).ConfigureAwait(false);

var expectedResult = $"Policy `{policiesStr}` is set on {policyType.ToString().ToLower()} `{itemName}`";
if (!result.Any(r => r.Contains(expectedResult)))
Expand All @@ -107,17 +89,7 @@ private async Task<bool> SetPolicyAsync(IdentityType policyType, List<string> po
return true;
}

private async Task RemoveUserAsync(string username)
{
var result = await Execute($"admin user remove {_serviceName} {username}").ConfigureAwait(false);

if (!result.Any(r => r.Contains($"Removed user `{username}` successfully.")))
{
throw new InvalidOperationException("Unable to remove user");
}
}

private async Task<List<string>> Execute(string cmd)
private async Task<List<string>> ExecuteAsync(string cmd)
{
if (cmd.StartsWith("mc"))
{
Expand All @@ -126,19 +98,41 @@ private async Task<List<string>> Execute(string cmd)

using (var process = CreateProcess(cmd))
{
var (lines, errors) = await RunProcess(process).ConfigureAwait(false);
var (lines, errors) = await RunProcessAsync(process);
if (errors.Any())
{
throw new InvalidOperationException($"Unknown Error {errors.SelectMany(e => e)}");
throw new InvalidOperationException($"Unknown Error {string.Join("\n", errors)}");
}

return lines;
}
}

private static async Task<(List<string> Output, List<string> Errors)> RunProcessAsync(Process process)
{
var output = new List<string>();
var errors = new List<string>();
process.Start();
while (!process.StandardOutput.EndOfStream)
{
var line = process.StandardOutput.ReadLine();
if (line == null) continue;
output.Add(line);
}
while (!process.StandardError.EndOfStream)
{
var line = process.StandardError.ReadLine();
if (line == null) continue;
errors.Add(line);
}

await process.WaitForExitAsync().ConfigureAwait(false);
return (output, errors);
}

private Process CreateProcess(string cmd)
{
var startinfo = new ProcessStartInfo()
ProcessStartInfo startinfo = new()
{
FileName = _executableLocation,
Arguments = cmd,
Expand All @@ -148,14 +142,101 @@ private Process CreateProcess(string cmd)
RedirectStandardError = true
};

var process = new Process()
Process process = new()
{
StartInfo = startinfo
};

return process;
}

public async Task<bool> HasConnectionAsync()
{
var result = await ExecuteAsync(_get_connections_cmd).ConfigureAwait(false);
return result.Any(r => r.Equals(_serviceName));
}

public async Task<bool> SetConnectionAsync()
{
if (await HasConnectionAsync())
{
return true;
}
var result = await ExecuteAsync(_set_connection_cmd).ConfigureAwait(false);
if (result.Any(r => r.Contains($"Added `{_serviceName}` successfully.")))
{
return true;
}
return false;
}

public async Task<bool> UserAlreadyExistsAsync(string username)
{
var result = await ExecuteAsync(_get_users_cmd).ConfigureAwait(false);
return result.Any(r => r.Contains(username));
}

public async Task RemoveUserAsync(string username)
{
var result = await ExecuteAsync($"admin user remove {_serviceName} {username}").ConfigureAwait(false);

if (!result.Any(r => r.Contains($"Removed user `{username}` successfully.")))
{
throw new InvalidOperationException("Unable to remove user");
}
}

[Obsolete("CreateUserAsync with bucketNames is deprecated, please use CreateUserAsync with an array of PolicyRequest instead.")]
public async Task<Credentials> CreateUserAsync(string username, AccessPermissions permissions, string[] bucketNames)
{
var policyRequests = new List<PolicyRequest>();

for (var i = 0; i < bucketNames.Length; i++)
{
policyRequests.Add(new PolicyRequest(bucketNames[i], "/*"));
}

return await CreateUserAsync(username, policyRequests.ToArray()).ConfigureAwait(false);
}

public async Task<Credentials> CreateUserAsync(string username, PolicyRequest[] policyRequests)
{
if (!await SetConnectionAsync())
{
throw new InvalidOperationException("Unable to set connection for more information, attempt mc alias set {_serviceName} http://{_endpoint} {_accessKey} {_secretKey}");
}
if (await UserAlreadyExistsAsync(username))
{
throw new InvalidOperationException("User already exists");
}

Credentials credentials = new();
var userSecretKey = Convert.ToBase64String(Guid.NewGuid().ToByteArray());
credentials.SecretAccessKey = userSecretKey;
credentials.AccessKeyId = username;

var result = await ExecuteAsync(CreateUserCmd(username, userSecretKey)).ConfigureAwait(false);

if (result.Any(r => r.Contains($"Added user `{username}` successfully.")) is false)
{
await RemoveUserAsync(username);
throw new InvalidOperationException($"Unknown Output {string.Join("\n", result)}");
}


var policyName = await CreatePolicyAsync(policyRequests.ToArray(), username).ConfigureAwait(false);
var minioPolicies = new List<string> { policyName };

var setPolicyResult = await SetPolicyAsync(IdentityType.User, minioPolicies, credentials.AccessKeyId).ConfigureAwait(false);
if (setPolicyResult is false)
{
await RemoveUserAsync(username).ConfigureAwait(false);
throw new InvalidOperationException("Failed to set policy, user has been removed");
}

return credentials;
}

/// <summary>
/// Admin policy command requires json file for policy so we create file
/// and remove it after setting the admin policy for the user.
Expand All @@ -164,20 +245,18 @@ private Process CreateProcess(string cmd)
/// <param name="username"></param>
/// <returns></returns>
/// <exception cref="InvalidOperationException"></exception>
private async Task CreatePolicyAsync(PolicyRequest[] policyRequests, string username)
private async Task<string> CreatePolicyAsync(PolicyRequest[] policyRequests, string username)
{
Guard.Against.NullOrEmpty(policyRequests, nameof(policyRequests));
Guard.Against.NullOrWhiteSpace(username, nameof(username));

var userFile = await CreatePolicyFile(policyRequests, username).ConfigureAwait(false);
var result = await Execute($"admin policy {_serviceName} pol_{username} {username}.json").ConfigureAwait(false);
if (!result.Any(r => r.Contains($"Added policy `pol_{username}` successfully.")))
var policyFileName = await CreatePolicyFile(policyRequests, username).ConfigureAwait(false);
var result = await ExecuteAsync($"admin policy add {_serviceName} pol_{username} {policyFileName}").ConfigureAwait(false);
if (result.Any(r => r.Contains($"Added policy `pol_{username}` successfully.")) is false)
{
await RemoveUserAsync(username).ConfigureAwait(false);
await RemoveUserAsync(username);
File.Delete($"{username}.json");
throw new InvalidOperationException("Failed to create policy, user has been removed");
}
File.Delete(userFile);
File.Delete($"{username}.json");
return $"pol_{username}";
}

private async Task<string> CreatePolicyFile(PolicyRequest[] policyRequests, string username)
Expand All @@ -192,48 +271,5 @@ private async Task<string> CreatePolicyFile(PolicyRequest[] policyRequests, stri
await _fileSystem.File.WriteAllLinesAsync(filename, lines).ConfigureAwait(false);
return filename;
}

private async Task<bool> UserAlreadyExistsAsync(string username)
{
var result = await Execute(UserCommand).ConfigureAwait(false);
return result.Any(r => r.Contains(username));
}

private static void ValidateConfiguration(StorageServiceConfiguration configuration)
{
Guard.Against.Null(configuration, nameof(configuration));

foreach (var key in ConfigurationKeys.McRequiredKeys)
{
if (!configuration.Settings.ContainsKey(key))
{
throw new ConfigurationException($"IMinioAdmin Shell is missing configuration for {key}.");
}
}
}

private string CreateUserCmd(string username, string secretKey) => $"admin user add {_serviceName} {username} {secretKey}";

private static async Task<(List<string> Output, List<string> Errors)> RunProcess(Process process)
{
var output = new List<string>();
var errors = new List<string>();
process.Start();
while (!process.StandardOutput.EndOfStream)
{
var line = process.StandardOutput.ReadLine();
if (line == null) continue;
output.Add(line);
}
while (!process.StandardError.EndOfStream)
{
var line = process.StandardError.ReadLine();
if (line == null) continue;
errors.Add(line);
}

await process.WaitForExitAsync().ConfigureAwait(false);
return (output, errors);
}
}
}
Loading