Add file system body storage disk space custom check

src/ServiceControl.Persistence.EFCore.PostgreSql/persistence.manifest

@@ -18,7 +18,11 @@
       "Mandatory": true
     },
     {
-      "Name": "ServiceControl/MessageBody/StoragePath",
+      "Name": "ServiceControl/MessageBody/FileSystem/StoragePath",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/FileSystem/DataSpaceRemainingThreshold",
       "Mandatory": false
     },
     {

src/ServiceControl.Persistence.EFCore.SqlServer/persistence.manifest

@@ -18,7 +18,11 @@
       "Mandatory": true
     },
     {
-      "Name": "ServiceControl/MessageBody/StoragePath",
+      "Name": "ServiceControl/MessageBody/FileSystem/StoragePath",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/FileSystem/DataSpaceRemainingThreshold",
       "Mandatory": false
     },
     {

src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs

@@ -4,6 +4,7 @@ namespace ServiceControl.Persistence.EFCore.Abstractions;
 using Microsoft.Extensions.DependencyInjection.Extensions;
 using NServiceBus.Unicast.Subscriptions.MessageDrivenSubscriptions;
 using Particular.LicensingComponent.Persistence;
+using ServiceControl.CustomChecks;
 using ServiceControl.Operations.BodyStorage;
 using ServiceControl.Persistence.EFCore.Implementation;
 using ServiceControl.Persistence.EFCore.Implementation.BodyStorage;
@@ -64,6 +65,8 @@ static void RegisterBodyStorage(IServiceCollection services, EFPersisterSettings
             case FileSystemBodyStorageSettings fileSystem:
                 services.TryAddSingleton(fileSystem);
                 services.AddSingleton<IBodyStoragePersistence, FileSystemBodyStoragePersistence>();
+                services.AddSingleton<IDriveSpaceProvider, DriveInfoSpaceProvider>();
+                services.AddCustomCheck<FileSystemBodyStorageCustomCheck>();
                 break;
             case AzureBlobBodyStorageSettings azureBlob:
                 services.TryAddSingleton(azureBlob);

src/ServiceControl.Persistence.EFCore/Abstractions/EFPersistenceConfigurationBase.cs

@@ -9,7 +9,8 @@ public abstract class EFPersistenceConfigurationBase : IPersistenceConfiguration
     const string ConnectionStringKey = "Database/ConnectionString";
     const string CommandTimeoutKey = "Database/CommandTimeout";
     const string BodyStorageTypeKey = "MessageBody/StorageType";
-    const string MessageBodyStoragePathKey = "MessageBody/StoragePath";
+    const string FileSystemStoragePathKey = "MessageBody/FileSystem/StoragePath";
+    const string FileSystemDataSpaceRemainingThresholdKey = "MessageBody/FileSystem/DataSpaceRemainingThreshold";
     const string AzureConnectionStringKey = "MessageBody/Azure/ConnectionString";
     const string AzureServiceUriKey = "MessageBody/Azure/ServiceUri";
     const string AzureManagedIdentityClientIdKey = "MessageBody/Azure/ManagedIdentityClientId";
@@ -51,7 +52,8 @@ static BodyStorageSettings CreateBodyStorageSettings(SettingsRootNamespace setti
         {
             BodyStorageType.FileSystem => new FileSystemBodyStorageSettings
             {
-                StoragePath = GetRequiredSetting<string>(settingsRootNamespace, MessageBodyStoragePathKey)
+                StoragePath = GetRequiredSetting<string>(settingsRootNamespace, FileSystemStoragePathKey),
+                DataSpaceRemainingThreshold = ReadDataSpaceRemainingThreshold(settingsRootNamespace)
             },
             BodyStorageType.AzureBlob => CreateAzureBlobSettings(settingsRootNamespace),
             BodyStorageType.S3 => CreateS3Settings(settingsRootNamespace),
@@ -160,6 +162,18 @@ static BodyStorageType ReadBodyStorageType(SettingsRootNamespace settingsRootNam
         return value;
     }
 
+    static int ReadDataSpaceRemainingThreshold(SettingsRootNamespace settingsRootNamespace)
+    {
+        var threshold = SettingsReader.Read(settingsRootNamespace, FileSystemDataSpaceRemainingThresholdKey, FileSystemBodyStorageSettings.DefaultDataSpaceRemainingThreshold);
+
+        if (threshold is < 0 or > 100)
+        {
+            throw new Exception($"Setting {FileSystemDataSpaceRemainingThresholdKey} value '{threshold}' is not valid. The value is a percentage between 0 and 100.");
+        }
+
+        return threshold;
+    }
+
     static int ReadMaxBodySizeToStore(SettingsRootNamespace settingsRootNamespace)
     {
         var maxBodySizeToStore = SettingsReader.Read(settingsRootNamespace, MaxBodySizeToStoreKey, BodyStorageSettings.DefaultMaxBodySizeToStore);

src/ServiceControl.Persistence.EFCore/Abstractions/FileSystemBodyStorageSettings.cs

@@ -2,5 +2,9 @@ namespace ServiceControl.Persistence.EFCore.Abstractions;
 
 public sealed class FileSystemBodyStorageSettings : BodyStorageSettings
 {
+    public const int DefaultDataSpaceRemainingThreshold = 15;
+
     public required string StoragePath { get; set; }
+
+    public int DataSpaceRemainingThreshold { get; set; } = DefaultDataSpaceRemainingThreshold;
 }

src/ServiceControl.Persistence.EFCore/Implementation/BodyStorage/DriveInfoSpaceProvider.cs

@@ -0,0 +1,11 @@
+namespace ServiceControl.Persistence.EFCore.Implementation.BodyStorage;
+
+public class DriveInfoSpaceProvider : IDriveSpaceProvider
+{
+    public DriveSpace GetDriveSpace(string pathRoot)
+    {
+        var driveInfo = new DriveInfo(pathRoot);
+
+        return new DriveSpace(driveInfo.Name, driveInfo.AvailableFreeSpace, driveInfo.TotalSize);
+    }
+}

src/ServiceControl.Persistence.EFCore/Implementation/BodyStorage/FileSystemBodyStorageCustomCheck.cs

@@ -0,0 +1,53 @@
+namespace ServiceControl.Persistence.EFCore.Implementation.BodyStorage;
+
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using NServiceBus.CustomChecks;
+using ServiceControl.Persistence.EFCore.Abstractions;
+
+public class FileSystemBodyStorageCustomCheck(FileSystemBodyStorageSettings settings, IDriveSpaceProvider driveSpaceProvider, ILogger<FileSystemBodyStorageCustomCheck> logger)
+    : CustomCheck("ServiceControl body storage", "Storage space", TimeSpan.FromMinutes(15))
+{
+    public override Task<CheckResult> PerformCheck(CancellationToken cancellationToken = default)
+    {
+        logger.LogDebug("Check ServiceControl body storage drive space remaining custom check starting. Threshold {PercentageThreshold:P0}", percentageThreshold);
+
+        if (string.IsNullOrEmpty(dataPathRoot))
+        {
+            return CheckResult.Failed($"Unable to find the root of the message body storage path '{settings.StoragePath}'. An absolute path is required.");
+        }
+
+        DriveSpace driveSpace;
+
+        try
+        {
+            driveSpace = driveSpaceProvider.GetDriveSpace(dataPathRoot);
+        }
+        catch (Exception ex)
+        {
+            // Custom checks report an exception as an opaque failure, so the reason is captured here instead.
+            logger.LogError(ex, "Unable to read the free space of drive '{DataPathRoot}' for the message body storage path '{StoragePath}'", dataPathRoot, settings.StoragePath);
+
+            return CheckResult.Failed($"Unable to read the free space of drive '{dataPathRoot}' for the message body storage path '{settings.StoragePath}' on '{Environment.MachineName}': {ex.Message}");
+        }
+
+        if (driveSpace.TotalSize <= 0)
+        {
+            return CheckResult.Failed($"Unable to determine the size of drive '{driveSpace.Name}' for the message body storage path '{settings.StoragePath}' on '{Environment.MachineName}'.");
+        }
+
+        var percentRemaining = (decimal)driveSpace.AvailableFreeSpace / driveSpace.TotalSize;
+
+        logger.LogDebug("Free space: {FreeSpaceTotalBytesFree:N0}B | Total: {FreeSpaceTotalBytesAvailable:N0}B | Remaining {PercentRemaining:P1}", driveSpace.AvailableFreeSpace, driveSpace.TotalSize, percentRemaining);
+
+        return percentRemaining > percentageThreshold
+            ? CheckResult.Pass
+            : CheckResult.Failed($"{percentRemaining:P0} disk space remaining on the message body storage drive '{driveSpace.Name}' ({settings.StoragePath}) on '{Environment.MachineName}'.");
+    }
+
+    readonly string? dataPathRoot = Path.GetPathRoot(settings.StoragePath);
+    readonly decimal percentageThreshold = settings.DataSpaceRemainingThreshold / 100m;
+}

src/ServiceControl.Persistence.EFCore/Implementation/BodyStorage/IDriveSpaceProvider.cs

@@ -0,0 +1,8 @@
+namespace ServiceControl.Persistence.EFCore.Implementation.BodyStorage;
+
+public interface IDriveSpaceProvider
+{
+    DriveSpace GetDriveSpace(string pathRoot);
+}
+
+public readonly record struct DriveSpace(string Name, long AvailableFreeSpace, long TotalSize);

src/ServiceControl.Persistence.Tests.PostgreSql/ServiceControl.Persistence.Tests.PostgreSql.csproj

@@ -40,7 +40,6 @@
     <Compile Remove="..\ServiceControl.Persistence.Tests\Throughput\EndpointsTests.cs" />
     <Compile Remove="..\ServiceControl.Persistence.Tests\Throughput\ReportMasksTests.cs" />
     <Compile Remove="..\ServiceControl.Persistence.Tests\CustomChecksDataStoreTests.cs" />
-    <Compile Remove="..\ServiceControl.Persistence.Tests\CustomCheckTests.cs" />
     <Compile Remove="..\ServiceControl.Persistence.Tests\RetryStateTests.cs" />
   </ItemGroup>
 

src/ServiceControl.Persistence.Tests/CustomCheckTests.cs → src/ServiceControl.Persistence.Tests.RavenDB/CustomCheckTests.cs

@@ -1,4 +1,4 @@
-namespace ServiceControl.Persistence.Tests
+namespace ServiceControl.Persistence.Tests.RavenDB
 {
     using System;
     using System.Linq;

src/ServiceControl.Persistence.Tests.SqlServer/ServiceControl.Persistence.Tests.SqlServer.csproj

@@ -40,7 +40,6 @@
     <Compile Remove="..\ServiceControl.Persistence.Tests\Throughput\EndpointsTests.cs" />
     <Compile Remove="..\ServiceControl.Persistence.Tests\Throughput\ReportMasksTests.cs" />
     <Compile Remove="..\ServiceControl.Persistence.Tests\CustomChecksDataStoreTests.cs" />
-    <Compile Remove="..\ServiceControl.Persistence.Tests\CustomCheckTests.cs" />
     <Compile Remove="..\ServiceControl.Persistence.Tests\RetryStateTests.cs" />
   </ItemGroup>
 

src/ServiceControl.Persistence.Tests/EFCore/BodyStorageConfigurationTests.cs

@@ -19,7 +19,8 @@ class BodyStorageConfigurationTests
         "SERVICECONTROL_DATABASE_CONNECTIONSTRING",
         "SERVICECONTROL_ERRORRETENTIONPERIOD",
         "SERVICECONTROL_MESSAGEBODY_STORAGETYPE",
-        "SERVICECONTROL_MESSAGEBODY_STORAGEPATH",
+        "SERVICECONTROL_MESSAGEBODY_FILESYSTEM_STORAGEPATH",
+        "SERVICECONTROL_MESSAGEBODY_FILESYSTEM_DATASPACEREMAININGTHRESHOLD",
         "SERVICECONTROL_MESSAGEBODY_MINCOMPRESSIONSIZE",
         "SERVICECONTROL_MESSAGEBODY_AZURE_CONNECTIONSTRING",
         "SERVICECONTROL_MESSAGEBODY_AZURE_SERVICEURI",
@@ -48,12 +49,39 @@ public void SetUp()
     public void File_system_storage_yields_a_populated_path()
     {
         Set("SERVICECONTROL_MESSAGEBODY_STORAGETYPE", "FileSystem");
-        Set("SERVICECONTROL_MESSAGEBODY_STORAGEPATH", "/var/bodies");
+        Set("SERVICECONTROL_MESSAGEBODY_FILESYSTEM_STORAGEPATH", "/var/bodies");
 
         var bodyStorage = CreateBodyStorageSettings();
 
         Assert.That(bodyStorage, Is.TypeOf<FileSystemBodyStorageSettings>());
-        Assert.That(((FileSystemBodyStorageSettings)bodyStorage).StoragePath, Is.EqualTo("/var/bodies"));
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(((FileSystemBodyStorageSettings)bodyStorage).StoragePath, Is.EqualTo("/var/bodies"));
+            Assert.That(((FileSystemBodyStorageSettings)bodyStorage).DataSpaceRemainingThreshold, Is.EqualTo(FileSystemBodyStorageSettings.DefaultDataSpaceRemainingThreshold));
+        }
+    }
+
+    [Test]
+    public void File_system_storage_reads_the_data_space_remaining_threshold()
+    {
+        Set("SERVICECONTROL_MESSAGEBODY_STORAGETYPE", "FileSystem");
+        Set("SERVICECONTROL_MESSAGEBODY_FILESYSTEM_STORAGEPATH", "/var/bodies");
+        Set("SERVICECONTROL_MESSAGEBODY_FILESYSTEM_DATASPACEREMAININGTHRESHOLD", "30");
+
+        var bodyStorage = (FileSystemBodyStorageSettings)CreateBodyStorageSettings();
+
+        Assert.That(bodyStorage.DataSpaceRemainingThreshold, Is.EqualTo(30));
+    }
+
+    [TestCase("-1")]
+    [TestCase("101")]
+    public void A_data_space_remaining_threshold_outside_the_percentage_range_is_rejected(string threshold)
+    {
+        Set("SERVICECONTROL_MESSAGEBODY_STORAGETYPE", "FileSystem");
+        Set("SERVICECONTROL_MESSAGEBODY_FILESYSTEM_STORAGEPATH", "/var/bodies");
+        Set("SERVICECONTROL_MESSAGEBODY_FILESYSTEM_DATASPACEREMAININGTHRESHOLD", threshold);
+
+        Assert.That(CreateBodyStorageSettings, Throws.Exception.With.Message.Contains("MessageBody/FileSystem/DataSpaceRemainingThreshold"));
     }
 
     [Test]
@@ -137,7 +165,7 @@ public void File_system_storage_without_a_path_is_rejected()
     {
         Set("SERVICECONTROL_MESSAGEBODY_STORAGETYPE", "FileSystem");
 
-        Assert.That(CreateBodyStorageSettings, Throws.Exception.With.Message.Contains("MessageBody/StoragePath"));
+        Assert.That(CreateBodyStorageSettings, Throws.Exception.With.Message.Contains("MessageBody/FileSystem/StoragePath"));
     }
 
     [Test]