Stops accepting/retrieving data when disk space is low.

doc/dependency_decisions.yml

@@ -46,7 +46,7 @@
   - :who: mocsharp
     :why: MIT (https://github.com/Cysharp/ConsoleAppFramework/raw/master/LICENSE)
     :versions:
-    - 4.2.2
+    - 4.2.3
     :when: 2022-08-16 23:05:31.110052610 Z
 - - :approve
   - Crayon

docs/api/rest/dicomweb-stow.md

@@ -56,8 +56,9 @@ Response Content Type: `JSON`
 | 202  | [DicomDataset](https://github.com/fo-dicom/fo-dicom/blob/development/FO-DICOM.Core/DicomDataset.cs) | All instances are received and stored with warnings (e.g. for a mismatched StudyInstanceUID. |
 | 204  | `none`                                                                                              | No data is provided.                                                                         |
 | 400  | [Problem details](https://datatracker.ietf.org/doc/html/rfc7807)                                    | Request contains invalid values.                                                             |
-| 415  | `none`                                                                                              | Unsupported media type                                                                       |
-| 500  | [Problem details](https://datatracker.ietf.org/doc/html/rfc7807)                                    | Server error                                                                                 |
+| 415  | `none`                                                                                              | Unsupported media typ.                                                                       |
+| 500  | [Problem details](https://datatracker.ietf.org/doc/html/rfc7807)                                    | Server error.                                                                                |
+| 507  | [Problem details](https://datatracker.ietf.org/doc/html/rfc7807)                                    | Insufficient storage.                                                                        |
 
 ---
 
@@ -97,3 +98,4 @@ Response Content Type: `JSON`
 | 400  | [Problem details](https://datatracker.ietf.org/doc/html/rfc7807)                                    | Request contains invalid values.                                                             |
 | 415  | `none`                                                                                              | Unsupported media type                                                                       |
 | 500  | [Problem details](https://datatracker.ietf.org/doc/html/rfc7807)                                    | Server error                                                                                 |
+| 507  | [Problem details](https://datatracker.ietf.org/doc/html/rfc7807)                                    | Insufficient storage.                                                                        |

docs/api/rest/fhir.md

@@ -55,9 +55,10 @@ Depending on the `Accept` header or the original document, the response supports
 
 If the `Accept` header is missing or a none supported value exists, the service will return the same type as the posted document.
 
-| Code | Data Type                                                     | Description                                                           |
-| ---- | ------------------------------------------------------------- | --------------------------------------------------------------------- |
-| 201  | Original JSON or XML document.                                | Resource created & stored successfully.                               |
-| 400  | [OperationOutcome](http://hl7.org/fhir/operationoutcome.html) | Unable to parse the resource or mismatching resource type specified.. |
-| 415  | `none`                                                        | Unsupported media type                                                |
-| 500  | [OperationOutcome](http://hl7.org/fhir/operationoutcome.html) | Server error.                                                         |
+| Code | Data Type                                                       | Description                                                           |
+| ---- | --------------------------------------------------------------- | --------------------------------------------------------------------- |
+| 201  | Original JSON or XML document.                                  | Resource created & stored successfully.                               |
+| 400  | [OperationOutcome](http://hl7.org/fhir/operationoutcome.html)   | Unable to parse the resource or mismatching resource type specified.. |
+| 415  | `none`                                                          | Unsupported media type                                                |
+| 500  | [OperationOutcome](http://hl7.org/fhir/operationoutcome.html)   | Server error.                                                         |
+| 507  | [Problem details](https://datatracker.ietf.org/doc/html/rfc7807)| Insufficient storage.                                                 |

docs/compliance/third-party-licenses.md

@@ -589,14 +589,14 @@ limitations under the License.
 
 
 <details>
-<summary>ConsoleAppFramework 4.2.2</summary>
+<summary>ConsoleAppFramework 4.2.3</summary>
 
 ## ConsoleAppFramework
 
-- Version: 4.2.2
+- Version: 4.2.3
 - Authors: Cysharp
 - Project URL: https://github.com/Cysharp/ConsoleAppFramework
-- Source: [NuGet](https://www.nuget.org/packages/ConsoleAppFramework/4.2.2)
+- Source: [NuGet](https://www.nuget.org/packages/ConsoleAppFramework/4.2.3)
 - License: [MIT](https://github.com/Cysharp/ConsoleAppFramework/raw/master/LICENSE)
 
 

docs/setup/schema.md

@@ -96,12 +96,14 @@ The `InformaticsGateway` configuration section contains the following sub-sectio
       }
     },
     "storage": {
-      "bufferRootPath": "./temp",
-      "tempStorageRootPath": "/incoming",
+      "localTemporaryStoragePath": "/payloads",
+      "remoteTemporaryStoragePath": "/incoming",
       "bucketName": "monaideploy",
       "storageRootPath": "/payloads",
       "temporaryBucketName": "monaideploy",
       "serviceAssemblyName": "Monai.Deploy.Storage.MinIO.MinIoStorageService, Monai.Deploy.Storage.MinIO",
+      "watermarkPercent": 75,
+      "reserveSpaceGB": 5,
       "settings": {
         "endpoint": "localhost:9000",
         "accessKey": "admin",

docs/setup/setup.md

@@ -101,9 +101,9 @@ The Informatics Gateway operates on two storage locations. In the first location
 
 ### Temporary Storage of Incoming Dataset
 
-By default, the temporary storage location is set to `/payloads` in the `appsettings.json` file.
+By default, the temporary storage location is set to use `Disk` and stores any incoming files inside `/payloads`.  This can be modified to user a different location, such as `Memory` or a different path.
 
-To change the temporary storage location, locate the `./InformaticsGateway/storage/temporary` property in the `appsettings.json` file and modify it.
+To change the temporary storage path, locate the `InformaticsGateway>storage>localTemporaryStoragePath` property in the `appsettings.json` file and modify it.
 
 > [!Note]
 > You will need to calculate the required temporary storage based on the number of studies and the size of each study.
@@ -116,7 +116,7 @@ To change the temporary storage location, locate the `./InformaticsGateway/stora
 > the expected number of studies and size of each study. The suggested value for `reserveSpaceGB` is 2x to 3x the
 > size of a single study multiplied by the number of configured AE Titles.
 
-### Shared Storage
+### Storage Service
 
 Informatics Gateway includes MinIO as the default storage service provider. To integrate with another storage service provider, please refer to the [Data Storage](https://github.com/Project-MONAI/monai-deploy-informatics-gateway/blob/main/guidelines/srs.md#data-storage) section of the SRS.
 
@@ -138,7 +138,6 @@ Locate the storage section of the configuration in `appsettings.json`:
         "accessToken": "password", # Access token or password
         "securedConnection": false, # Indicates if connection should be secured using HTTPS
         "region": "local", # Region
-        "executableLocation": "/bin/mc", # Path to minio client
         "serviceName": "MinIO" # Name of the service
       },
       "storageService": "Monai.Deploy.Storage.MinIO.MinIoStorageService, Monai.Deploy.Storage.MinIO", # Fully qualified type name of the storage service

src/Configuration/ConfigurationValidator.cs

@@ -81,7 +81,7 @@ private bool IsStorageValid(StorageConfiguration storage)
             var valid = true;
             valid &= IsValidBucketName("InformaticsGateway>storage>bucketName", storage.StorageServiceBucketName);
             valid &= IsValidBucketName("InformaticsGateway>storage>temporaryBucketName", storage.TemporaryStorageBucket);
-            valid &= IsNotNullOrWhiteSpace("InformaticsGateway>storage>temporary", storage.TemporaryStorageRootPath);
+            valid &= IsNotNullOrWhiteSpace("InformaticsGateway>storage>temporary", storage.RemoteTemporaryStoragePath);
             valid &= IsValueInRange("InformaticsGateway>storage>watermark", 1, 100, storage.Watermark);
             valid &= IsValueInRange("InformaticsGateway>storage>reserveSpaceGB", 1, 999, storage.ReserveSpaceGB);
             valid &= IsValueInRange("InformaticsGateway>storage>payloadProcessThreads", 1, 128, storage.PayloadProcessThreads);
@@ -90,8 +90,8 @@ private bool IsStorageValid(StorageConfiguration storage)
 
             if (storage.TemporaryDataStorage == TemporaryDataStorageLocation.Disk)
             {
-                valid &= IsNotNullOrWhiteSpace("InformaticsGateway>storage>bufferRootPath", storage.BufferStorageRootPath);
-                valid &= IsValidDirectory("InformaticsGateway>storage>bufferRootPath", storage.BufferStorageRootPath);
+                valid &= IsNotNullOrWhiteSpace("InformaticsGateway>storage>localTemporaryStoragePath", storage.LocalTemporaryStoragePath);
+                valid &= IsValidDirectory("InformaticsGateway>storage>localTemporaryStoragePath", storage.LocalTemporaryStoragePath);
                 valid &= IsValueInRange("InformaticsGateway>storage>bufferSize", 1, int.MaxValue, storage.BufferSize);
             }
 
@@ -105,13 +105,9 @@ private bool IsValidDirectory(string source, string directory)
             {
                 if (!_fileSystem.Directory.Exists(directory))
                 {
-                    valid = false;
-                    _validationErrors.Add($"Directory `{directory}` specified in `{source}` does not exist.");
-                }
-                else
-                {
-                    using var _ = _fileSystem.File.Create(Path.Combine(directory, Path.GetRandomFileName()), 1, FileOptions.DeleteOnClose);
+                    _fileSystem.Directory.CreateDirectory(directory);
                 }
+                using var _ = _fileSystem.File.Create(Path.Combine(directory, Path.GetRandomFileName()), 1, FileOptions.DeleteOnClose);
             }
             catch (Exception ex)
             {

src/Configuration/StorageConfiguration.cs

@@ -33,8 +33,8 @@ public class StorageConfiguration : StorageServiceConfiguration
         /// Gets or sets the path used for buffering incoming data.
         /// Defaults to <c>./temp</c>.
         /// </summary>
-        [ConfigurationKeyName("bufferRootPath")]
-        public string BufferStorageRootPath { get; set; } = "./temp";
+        [ConfigurationKeyName("localTemporaryStoragePath")]
+        public string LocalTemporaryStoragePath { get; set; } = "/payloads";
 
         /// <summary>
         /// Gets or sets the number of bytes buffered for reads and writes to the temporary file.
@@ -66,8 +66,8 @@ public class StorageConfiguration : StorageServiceConfiguration
         /// Gets or sets root directory path for storing incoming data in the <c>temporaryBucketName</c>.
         /// Defaults to <c>/incoming</c>.
         /// </summary>
-        [ConfigurationKeyName("tempStorageRootPath")]
-        public string TemporaryStorageRootPath { get; set; } = "/incoming";
+        [ConfigurationKeyName("remoteTemporaryStoragePath")]
+        public string RemoteTemporaryStoragePath { get; set; } = "/incoming";
 
         /// <summary>
         /// Gets or sets the watermark for disk usage with default value of 75%,

src/Configuration/Test/ConfigurationValidatorTest.cs

@@ -137,11 +137,11 @@ public void StorageWithInaccessbleDirectory()
 
             var config = MockValidConfiguration();
             config.Storage.TemporaryDataStorage = TemporaryDataStorageLocation.Disk;
-            config.Storage.BufferStorageRootPath = "/blabla";
+            config.Storage.LocalTemporaryStoragePath = "/blabla";
 
             var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config);
 
-            var validationMessages = new[] { $"Directory `/blabla` specified in `InformaticsGateway>storage>bufferRootPath` is not accessible: error." };
+            var validationMessages = new[] { $"Directory `/blabla` specified in `InformaticsGateway>storage>localTemporaryStoragePath` is not accessible: error." };
             Assert.Equal(string.Join(Environment.NewLine, validationMessages), valid.FailureMessage);
             foreach (var message in validationMessages)
             {

src/DicomWebClient/API/IDicomWebClient.cs

@@ -74,9 +74,9 @@ public interface IDicomWebClient
         /// </summary>
         /// <param name="serviceType"><c>ServiceType</c> to be configured</param>
         /// <param name="urlPrefix">Url prefix</param>
-        #pragma warning disable CA1054
+#pragma warning disable CA1054
         void ConfigureServicePrefix(DicomWebServiceType serviceType, string urlPrefix);
-        #pragma warning restore CA1054
+#pragma warning restore CA1054
         /// <summary>
         /// Configures the authentication header for the DICOMweb client.
         /// </summary>

src/DicomWebClient/API/IServiceBase.cs

@@ -19,8 +19,8 @@ namespace Monai.Deploy.InformaticsGateway.DicomWeb.Client.API
 {
     public interface IServiceBase
     {
-        #pragma warning disable CA1054
+#pragma warning disable CA1054
         bool TryConfigureServiceUriPrefix(string uriPrefix);
-        #pragma warning restore CA1054
+#pragma warning restore CA1054
     }
 }

src/DicomWebClient/CLI/Monai.Deploy.InformaticsGateway.DicomWeb.Client.CLI.csproj

@@ -34,7 +34,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="ConsoleAppFramework" Version="4.2.2" />
+    <PackageReference Include="ConsoleAppFramework" Version="4.2.3" />
     <PackageReference Include="fo-dicom" Version="5.0.3" />
     <PackageReference Include="Microsoft.Extensions.Http" Version="6.0.0" />
   </ItemGroup>

src/DicomWebClient/DicomWebClient.cs

@@ -78,9 +78,9 @@ public void ConfigureServiceUris(Uri uriRoot)
         }
 
         /// <inheritdoc/>
-        #pragma warning disable CA1054
+#pragma warning disable CA1054
         public void ConfigureServicePrefix(DicomWebServiceType serviceType, string urlPrefix)
-        #pragma warning restore CA1054
+#pragma warning restore CA1054
         {
             Guard.Against.NullOrWhiteSpace(urlPrefix, nameof(urlPrefix));
 

src/InformaticsGateway/Common/InsufficientStorageAvailableException.cs

@@ -0,0 +1,53 @@
+// Copyright 2021 MONAI Consortium
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//     http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/*
+ * Apache License, Version 2.0
+ * Copyright 2021 NVIDIA Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+namespace Monai.Deploy.InformaticsGateway.Common
+{
+    [Serializable]
+    public class InsufficientStorageAvailableException : Exception
+    {
+        public InsufficientStorageAvailableException()
+        {
+        }
+
+        public InsufficientStorageAvailableException(string message) : base(message)
+        {
+        }
+
+        public InsufficientStorageAvailableException(string message, Exception innerException) : base(message, innerException)
+        {
+        }
+
+        protected InsufficientStorageAvailableException(System.Runtime.Serialization.SerializationInfo serializationInfo, System.Runtime.Serialization.StreamingContext streamingContext)
+        {
+            throw new NotImplementedException();
+        }
+    }
+}

src/InformaticsGateway/Logging/Log.100.200.ScpService.cs

@@ -94,5 +94,8 @@ public static partial class Log
 
         [LoggerMessage(EventId = 211, Level = LogLevel.Warning, Message = "Verification service is disabled: rejecting association.")]
         public static partial void VerificationServiceDisabled(this ILogger logger);
+
+        [LoggerMessage(EventId = 212, Level = LogLevel.Error, Message = "Failed to process C-STORE request, out of storage space.")]
+        public static partial void CStoreFailedDueToLowStorageSpace(this ILogger logger, Exception ex);
     }
 }

src/InformaticsGateway/Logging/Log.300.StorageInfoProvider.cs

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2022 MONAI Consortium
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using Microsoft.Extensions.Logging;
+
+namespace Monai.Deploy.InformaticsGateway.Logging
+{
+    public static partial class Log
+    {
+        // StorageInfoProvider
+        [LoggerMessage(EventId = 300, Level = LogLevel.Information, Message = "Temporary Storage Path={path}. Storage Size: {totalSize:N0}. Reserved: {reservedSpace:N0}.")]
+        public static partial void StorageInfoProviderStartup(this ILogger logger, string path, long totalSize, long reservedSpace);
+
+        [LoggerMessage(EventId = 301, Level = LogLevel.Information, Message = "Storage Size: {totalSize:N0}. Reserved: {reservedSpace:N0}. Available: {freeSpace:N0}.")]
+        public static partial void CurrentStorageSize(this ILogger logger, long totalSize, long reservedSpace, long freeSpace);
+    }
+}

src/InformaticsGateway/Logging/Log.500.ExportService.cs

@@ -114,5 +114,8 @@ public static partial class Log
 
         [LoggerMessage(EventId = 530, Level = LogLevel.Error, Message = "{message}")]
         public static partial void ExportException(this ILogger logger, string message, Exception ex);
+
+        [LoggerMessage(EventId = 531, Level = LogLevel.Warning, Message = "Export service paused due to insufficient storage space.  Available storage space: {availableFreeSpace:D}")]
+        public static partial void ExportServiceStoppedDueToLowStorageSpace(this ILogger logger, long availableFreeSpace);
     }
 }

src/InformaticsGateway/Logging/Log.600.DataRetrievalService.cs

@@ -91,5 +91,8 @@ public static partial class Log
 
         [LoggerMessage(EventId = 622, Level = LogLevel.Warning, Message = "FHIR resource {type}/{id} contains no data.")]
         public static partial void FhirResourceContainsNoData(this ILogger logger, string type, string id);
+
+        [LoggerMessage(EventId = 623, Level = LogLevel.Warning, Message = "Data retrieval paused due to insufficient storage space.  Available storage space: {availableFreeSpace:D}.")]
+        public static partial void DataRetrievalServiceStoppedDueToLowStorageSpace(this ILogger logger, long availableFreeSpace);
     }
 }