Skip to content

Commit

Permalink
Use temporary permanent user accounts
Browse files Browse the repository at this point in the history
Signed-off-by: Victor Chang <vicchang@nvidia.com>
  • Loading branch information
mocsharp committed Sep 3, 2022
1 parent 044b3bd commit e7be031
Show file tree
Hide file tree
Showing 20 changed files with 154 additions and 130 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
<PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
<PackageReference Include="Karambolo.Extensions.Logging.File" Version="3.3.1" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
<PackageReference Include="Monai.Deploy.Storage" Version="0.2.5" />
<PackageReference Include="Monai.Deploy.Storage" Version="0.2.6-rc0005" />
<PackageReference Include="System.IO.Abstractions" Version="17.1.1" />
</ItemGroup>

Expand All @@ -42,8 +42,8 @@
<None Remove="Test\**" />
</ItemGroup>

<ItemGroup>
<AdditionalFiles Include="..\..\.sonarlint\project-monai_monai-deploy-workflow-manager\CSharp\SonarLint.xml" Link="SonarLint.xml" />
<ItemGroup>
<AdditionalFiles Include="..\..\.sonarlint\project-monai_monai-deploy-workflow-manager\CSharp\SonarLint.xml" Link="SonarLint.xml" />
</ItemGroup>

<PropertyGroup>
Expand Down
14 changes: 7 additions & 7 deletions src/Shared/Configuration/packages.lock.json
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
"type": "Direct",
"requested": "[0.1.4, )",
"resolved": "0.1.4",
"contentHash": "Y0+QmoUZUw79C7LKhKHNmi5bgeMiPSNjPQzEz4vnagJaEFtdugzQ90ofYMMMAMeOr0FjUgxyPBgvJ1ZG/r76Dw==",
"contentHash": "/fFC/OoidveajNeFj6r6aTGLNMY+Xl7iR/CjZortArBtAxzMgzTRzmjGltWn7Jae8+/Vv8w+pKgT0TZbK9JvxA==",
"dependencies": {
"Ardalis.GuardClauses": "4.0.1",
"Microsoft.Extensions.Configuration": "6.0.1",
Expand All @@ -65,16 +65,16 @@
},
"Monai.Deploy.Storage": {
"type": "Direct",
"requested": "[0.2.5, )",
"resolved": "0.2.5",
"contentHash": "EkctSwMUPm+omInxEGAbPPXz6dyd8nY4BAzrdGKt9MHsL1OesFXsgUXsmlCx+ltauohiiTSfjgd9eeVkUUD8MA==",
"requested": "[0.2.6-rc0005, )",
"resolved": "0.2.6-rc0005",
"contentHash": "XB6bXf4Z524d/45Rn1t+36tEK3O4oMdC3pa0W63LrifvidbI6jM9FKN8Lxzh/FXV5OBkfjFv5fhIskAlJCv75g==",
"dependencies": {
"AWSSDK.SecurityToken": "3.7.1.190",
"Ardalis.GuardClauses": "4.0.1",
"Microsoft.Extensions.Configuration": "6.0.1",
"Microsoft.Extensions.Diagnostics.HealthChecks": "6.0.8",
"Microsoft.Extensions.Logging": "6.0.0",
"Monai.Deploy.Storage.S3Policy": "0.2.5",
"Monai.Deploy.Storage.S3Policy": "0.2.6-rc0005",
"System.IO.Abstractions": "17.1.1"
}
},
Expand Down Expand Up @@ -229,8 +229,8 @@
},
"Monai.Deploy.Storage.S3Policy": {
"type": "Transitive",
"resolved": "0.2.5",
"contentHash": "wtvoeDf7MsuUE48Sggs6FYNOkLFUfQ5mPZer75jb63b9SKQjK2S34IdS8SnHVovNes1xhWInTB9YT2HUNgtzig==",
"resolved": "0.2.6-rc0005",
"contentHash": "W5UC+iR9H/CKOvib5tr1vOjvGr08E1BjRHvD4GZivxmaVt4KsWwl0jd3dKeHRLyGQO06CLVGqsz6d3zw0ew+LA==",
"dependencies": {
"Ardalis.GuardClauses": "4.0.1",
"Newtonsoft.Json": "13.0.1"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,41 +1,41 @@
<!--
~ Copyright 2022 MONAI Consortium
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->

<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

<PropertyGroup>
<RestorePackagesWithLockFile>true</RestorePackagesWithLockFile>
<DisableImplicitNuGetFallbackFolder>true</DisableImplicitNuGetFallbackFolder>
<CodeAnalysisRuleSet>..\..\.sonarlint\project-monai_monai-deploy-workflow-managercsharp.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>

<ItemGroup>
<AdditionalFiles Include="..\..\.sonarlint\project-monai_monai-deploy-workflow-manager\CSharp\SonarLint.xml" Link="SonarLint.xml" />
</ItemGroup>

<ItemGroup>
<PackageReference Include="Monai.Deploy.Messaging" Version="0.1.4" />
<PackageReference Include="Monai.Deploy.Storage" Version="0.2.5" />
<PackageReference Include="MongoDB.Bson" Version="2.17.1" />
</ItemGroup>

</Project>
<!--
~ Copyright 2022 MONAI Consortium
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->

<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

<PropertyGroup>
<RestorePackagesWithLockFile>true</RestorePackagesWithLockFile>
<DisableImplicitNuGetFallbackFolder>true</DisableImplicitNuGetFallbackFolder>
<CodeAnalysisRuleSet>..\..\.sonarlint\project-monai_monai-deploy-workflow-managercsharp.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>

<ItemGroup>
<AdditionalFiles Include="..\..\.sonarlint\project-monai_monai-deploy-workflow-manager\CSharp\SonarLint.xml" Link="SonarLint.xml" />
</ItemGroup>

<ItemGroup>
<PackageReference Include="Monai.Deploy.Messaging" Version="0.1.4" />
<PackageReference Include="Monai.Deploy.Storage" Version="0.2.6-rc0005" />
<PackageReference Include="MongoDB.Bson" Version="2.17.1" />
</ItemGroup>

</Project>
14 changes: 7 additions & 7 deletions src/TaskManager/API/packages.lock.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"type": "Direct",
"requested": "[0.1.4, )",
"resolved": "0.1.4",
"contentHash": "Y0+QmoUZUw79C7LKhKHNmi5bgeMiPSNjPQzEz4vnagJaEFtdugzQ90ofYMMMAMeOr0FjUgxyPBgvJ1ZG/r76Dw==",
"contentHash": "/fFC/OoidveajNeFj6r6aTGLNMY+Xl7iR/CjZortArBtAxzMgzTRzmjGltWn7Jae8+/Vv8w+pKgT0TZbK9JvxA==",
"dependencies": {
"Ardalis.GuardClauses": "4.0.1",
"Microsoft.Extensions.Configuration": "6.0.1",
Expand All @@ -20,16 +20,16 @@
},
"Monai.Deploy.Storage": {
"type": "Direct",
"requested": "[0.2.5, )",
"resolved": "0.2.5",
"contentHash": "EkctSwMUPm+omInxEGAbPPXz6dyd8nY4BAzrdGKt9MHsL1OesFXsgUXsmlCx+ltauohiiTSfjgd9eeVkUUD8MA==",
"requested": "[0.2.6-rc0005, )",
"resolved": "0.2.6-rc0005",
"contentHash": "XB6bXf4Z524d/45Rn1t+36tEK3O4oMdC3pa0W63LrifvidbI6jM9FKN8Lxzh/FXV5OBkfjFv5fhIskAlJCv75g==",
"dependencies": {
"AWSSDK.SecurityToken": "3.7.1.190",
"Ardalis.GuardClauses": "4.0.1",
"Microsoft.Extensions.Configuration": "6.0.1",
"Microsoft.Extensions.Diagnostics.HealthChecks": "6.0.8",
"Microsoft.Extensions.Logging": "6.0.0",
"Monai.Deploy.Storage.S3Policy": "0.2.5",
"Monai.Deploy.Storage.S3Policy": "0.2.6-rc0005",
"System.IO.Abstractions": "17.1.1"
}
},
Expand Down Expand Up @@ -169,8 +169,8 @@
},
"Monai.Deploy.Storage.S3Policy": {
"type": "Transitive",
"resolved": "0.2.5",
"contentHash": "wtvoeDf7MsuUE48Sggs6FYNOkLFUfQ5mPZer75jb63b9SKQjK2S34IdS8SnHVovNes1xhWInTB9YT2HUNgtzig==",
"resolved": "0.2.6-rc0005",
"contentHash": "W5UC+iR9H/CKOvib5tr1vOjvGr08E1BjRHvD4GZivxmaVt4KsWwl0jd3dKeHRLyGQO06CLVGqsz6d3zw0ew+LA==",
"dependencies": {
"Ardalis.GuardClauses": "4.0.1",
"Newtonsoft.Json": "13.0.1"
Expand Down
14 changes: 7 additions & 7 deletions src/TaskManager/Database/packages.lock.json
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@
"Monai.Deploy.Messaging": {
"type": "Transitive",
"resolved": "0.1.4",
"contentHash": "Y0+QmoUZUw79C7LKhKHNmi5bgeMiPSNjPQzEz4vnagJaEFtdugzQ90ofYMMMAMeOr0FjUgxyPBgvJ1ZG/r76Dw==",
"contentHash": "/fFC/OoidveajNeFj6r6aTGLNMY+Xl7iR/CjZortArBtAxzMgzTRzmjGltWn7Jae8+/Vv8w+pKgT0TZbK9JvxA==",
"dependencies": {
"Ardalis.GuardClauses": "4.0.1",
"Microsoft.Extensions.Configuration": "6.0.1",
Expand All @@ -189,22 +189,22 @@
},
"Monai.Deploy.Storage": {
"type": "Transitive",
"resolved": "0.2.5",
"contentHash": "EkctSwMUPm+omInxEGAbPPXz6dyd8nY4BAzrdGKt9MHsL1OesFXsgUXsmlCx+ltauohiiTSfjgd9eeVkUUD8MA==",
"resolved": "0.2.6-rc0005",
"contentHash": "XB6bXf4Z524d/45Rn1t+36tEK3O4oMdC3pa0W63LrifvidbI6jM9FKN8Lxzh/FXV5OBkfjFv5fhIskAlJCv75g==",
"dependencies": {
"AWSSDK.SecurityToken": "3.7.1.190",
"Ardalis.GuardClauses": "4.0.1",
"Microsoft.Extensions.Configuration": "6.0.1",
"Microsoft.Extensions.Diagnostics.HealthChecks": "6.0.8",
"Microsoft.Extensions.Logging": "6.0.0",
"Monai.Deploy.Storage.S3Policy": "0.2.5",
"Monai.Deploy.Storage.S3Policy": "0.2.6-rc0005",
"System.IO.Abstractions": "17.1.1"
}
},
"Monai.Deploy.Storage.S3Policy": {
"type": "Transitive",
"resolved": "0.2.5",
"contentHash": "wtvoeDf7MsuUE48Sggs6FYNOkLFUfQ5mPZer75jb63b9SKQjK2S34IdS8SnHVovNes1xhWInTB9YT2HUNgtzig==",
"resolved": "0.2.6-rc0005",
"contentHash": "W5UC+iR9H/CKOvib5tr1vOjvGr08E1BjRHvD4GZivxmaVt4KsWwl0jd3dKeHRLyGQO06CLVGqsz6d3zw0ew+LA==",
"dependencies": {
"Ardalis.GuardClauses": "4.0.1",
"Newtonsoft.Json": "13.0.1"
Expand Down Expand Up @@ -302,7 +302,7 @@
"type": "Project",
"dependencies": {
"Monai.Deploy.Messaging": "[0.1.4, )",
"Monai.Deploy.Storage": "[0.2.5, )",
"Monai.Deploy.Storage": "[0.2.6-rc0005, )",
"MongoDB.Bson": "[2.17.1, )"
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
<Nullable>enable</Nullable>
</PropertyGroup>

<ItemGroup>
<AdditionalFiles Include="..\..\..\.sonarlint\project-monai_monai-deploy-workflow-manager\CSharp\SonarLint.xml" Link="SonarLint.xml" />
<ItemGroup>
<AdditionalFiles Include="..\..\..\.sonarlint\project-monai_monai-deploy-workflow-manager\CSharp\SonarLint.xml" Link="SonarLint.xml" />
</ItemGroup>

<ItemGroup>
Expand All @@ -34,7 +34,7 @@
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Monai.Deploy.Messaging" Version="0.1.4" />
<PackageReference Include="Monai.Deploy.Storage" Version="0.2.5" />
<PackageReference Include="Monai.Deploy.Storage" Version="0.2.6-rc0005" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
</ItemGroup>

Expand Down
18 changes: 9 additions & 9 deletions src/TaskManager/Plug-ins/Argo/packages.lock.json
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
"type": "Direct",
"requested": "[0.1.4, )",
"resolved": "0.1.4",
"contentHash": "Y0+QmoUZUw79C7LKhKHNmi5bgeMiPSNjPQzEz4vnagJaEFtdugzQ90ofYMMMAMeOr0FjUgxyPBgvJ1ZG/r76Dw==",
"contentHash": "/fFC/OoidveajNeFj6r6aTGLNMY+Xl7iR/CjZortArBtAxzMgzTRzmjGltWn7Jae8+/Vv8w+pKgT0TZbK9JvxA==",
"dependencies": {
"Ardalis.GuardClauses": "4.0.1",
"Microsoft.Extensions.Configuration": "6.0.1",
Expand All @@ -50,16 +50,16 @@
},
"Monai.Deploy.Storage": {
"type": "Direct",
"requested": "[0.2.5, )",
"resolved": "0.2.5",
"contentHash": "EkctSwMUPm+omInxEGAbPPXz6dyd8nY4BAzrdGKt9MHsL1OesFXsgUXsmlCx+ltauohiiTSfjgd9eeVkUUD8MA==",
"requested": "[0.2.6-rc0005, )",
"resolved": "0.2.6-rc0005",
"contentHash": "XB6bXf4Z524d/45Rn1t+36tEK3O4oMdC3pa0W63LrifvidbI6jM9FKN8Lxzh/FXV5OBkfjFv5fhIskAlJCv75g==",
"dependencies": {
"AWSSDK.SecurityToken": "3.7.1.190",
"Ardalis.GuardClauses": "4.0.1",
"Microsoft.Extensions.Configuration": "6.0.1",
"Microsoft.Extensions.Diagnostics.HealthChecks": "6.0.8",
"Microsoft.Extensions.Logging": "6.0.0",
"Monai.Deploy.Storage.S3Policy": "0.2.5",
"Monai.Deploy.Storage.S3Policy": "0.2.6-rc0005",
"System.IO.Abstractions": "17.1.1"
}
},
Expand Down Expand Up @@ -338,8 +338,8 @@
},
"Monai.Deploy.Storage.S3Policy": {
"type": "Transitive",
"resolved": "0.2.5",
"contentHash": "wtvoeDf7MsuUE48Sggs6FYNOkLFUfQ5mPZer75jb63b9SKQjK2S34IdS8SnHVovNes1xhWInTB9YT2HUNgtzig==",
"resolved": "0.2.6-rc0005",
"contentHash": "W5UC+iR9H/CKOvib5tr1vOjvGr08E1BjRHvD4GZivxmaVt4KsWwl0jd3dKeHRLyGQO06CLVGqsz6d3zw0ew+LA==",
"dependencies": {
"Ardalis.GuardClauses": "4.0.1",
"Newtonsoft.Json": "13.0.1"
Expand Down Expand Up @@ -558,7 +558,7 @@
"Microsoft.Extensions.Logging": "[6.0.0, )",
"Microsoft.Extensions.Options": "[6.0.0, )",
"Monai.Deploy.Messaging": "[0.1.4, )",
"Monai.Deploy.Storage": "[0.2.5, )",
"Monai.Deploy.Storage": "[0.2.6-rc0005, )",
"Newtonsoft.Json": "[13.0.1, )",
"System.IO.Abstractions": "[17.1.1, )"
}
Expand All @@ -567,7 +567,7 @@
"type": "Project",
"dependencies": {
"Monai.Deploy.Messaging": "[0.1.4, )",
"Monai.Deploy.Storage": "[0.2.5, )",
"Monai.Deploy.Storage": "[0.2.6-rc0005, )",
"MongoDB.Bson": "[2.17.1, )"
}
}
Expand Down
18 changes: 6 additions & 12 deletions src/TaskManager/Plug-ins/Docker/DockerPlugin.cs
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ private async Task<List<ContainerVolumeMount>> SetupInputs(CancellationToken can
SessionToken = input.Credentials?.SessionToken,
};

var objects = await storageService.ListObjectsWithCredentialsAsync(input.Bucket, credentials, input.RelativeRootPath, true, cancellationToken);
var objects = await storageService.ListObjectsAsync(input.Bucket, input.RelativeRootPath, true, cancellationToken);
var id = Guid.NewGuid();

// Host Path of the Directory for this Input File.
Expand All @@ -179,21 +179,15 @@ private async Task<List<ContainerVolumeMount>> SetupInputs(CancellationToken can
foreach (var obj in objects)
{
// Task Manager Container Path of the Input File.
var inputContainerPath = Path.Combine(payloadContainerPath, id.ToString(), obj.Filename.Replace(input.RelativeRootPath, ""));
var inputContainerPath = Path.Combine(payloadContainerPath, id.ToString(), obj.FilePath.Replace(input.RelativeRootPath, ""));

// Task Manager Container Path of the Directory for this Input File.
var parentPath = Directory.GetParent(inputContainerPath).Parent.FullName;
var parentPath = Path.GetDirectoryName(inputContainerPath);
Directory.CreateDirectory(parentPath);

var stream = await storageService.GetObjectWithCredentialsAsync(input.Bucket, obj.Filename, credentials, cancellationToken);
var streamWriter = new StreamWriter(inputContainerPath);
var bufferSize = 1 * 1024 * 1024;
byte[] buffer = new byte[bufferSize];

while (stream.Read(buffer, 0, bufferSize) != 0)
{
streamWriter.Write(buffer);
}
using var stream = await storageService.GetObjectAsync(input.Bucket, obj.FilePath, cancellationToken) as MemoryStream;
using var fileStream = new FileStream(inputContainerPath, FileMode.CreateNew, FileAccess.Write);
stream!.WriteTo(fileStream);
}
}

Expand Down
3 changes: 3 additions & 0 deletions src/TaskManager/TaskManager/Logging/Log.cs
Original file line number Diff line number Diff line change
Expand Up @@ -116,5 +116,8 @@ public static partial class Log

[LoggerMessage(EventId = 118, Level = LogLevel.Error, Message = "Error removing storage user account {username}.")]
public static partial void ErrorRemovingStorageUserAccount(this ILogger logger, string username, Exception exception);

[LoggerMessage(EventId = 119, Level = LogLevel.Debug, Message = "Generating temporary credentials for bucket={bucket}, path={path}.")]
public static partial void GenerateTemporaryCredentials(this ILogger logger, string bucket, string path);
}
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<!--
<!--
~ Copyright 2021-2022 MONAI Consortium
~
~ Licensed under the Apache License, Version 2.0 (the "License");
Expand Down Expand Up @@ -52,14 +52,15 @@
</ItemGroup>

<ItemGroup>
<PackageReference Include="Ardalis.GuardClauses" Version="4.0.1" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.1" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="6.0.0" />
<PackageReference Include="Karambolo.Extensions.Logging.File" Version="3.3.1" />
<PackageReference Include="Monai.Deploy.Storage" Version="0.2.5" />
<PackageReference Include="Monai.Deploy.Storage" Version="0.2.6-rc0005" />
<PackageReference Include="Monai.Deploy.Messaging" Version="0.1.4" />
<PackageReference Include="Monai.Deploy.Storage.MinIO" Version="0.2.5" />
<PackageReference Include="Monai.Deploy.Storage.MinIO" Version="0.2.6-rc0005" />
<PackageReference Include="Monai.Deploy.Messaging.RabbitMQ" Version="0.1.4" />
</ItemGroup>

Expand All @@ -69,6 +70,7 @@
<ProjectReference Include="..\API\Monai.Deploy.WorkflowManager.TaskManager.API.csproj" />
<ProjectReference Include="..\Database\Monai.Deploy.WorkflowManager.TaskManager.Database.csproj" />
<ProjectReference Include="..\Plug-ins\Argo\Monai.Deploy.WorkflowManager.TaskManager.Argo.csproj" />
<ProjectReference Include="..\Plug-ins\Docker\Monai.Deploy.WorkflowManager.TaskManager.Docker.csproj" />
</ItemGroup>

<ItemGroup>
Expand Down
1 change: 1 addition & 0 deletions src/TaskManager/TaskManager/PluginStrings.cs
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,6 @@ public static class PluginStrings
{
// note not to be confused with the ApplicationId Monai.Deploy.WorkflowManager.TaskManager.Argo
public const string Argo = "argo";
public const string Docker = "docker";
}
}
Loading

0 comments on commit e7be031

Please sign in to comment.