Merge branch 'crash-on-oom' into 'master'

Crash app on OutOfMemoryException by default See merge request vostok-libraries/applications.scheduled!1
vostok · Oct 5, 2023 · 9669b07 · 9669b07
2 parents 75426c6 + f6e4049
commit 9669b07
Show file tree

Hide file tree

Showing 7 changed files with 244 additions and 8 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.2.15 (03-10-2023):
+
+Crash app on `OutOfMemoryException` by default.
+
 ## 0.2.13 (03-11-2022):
 
 Fixed `OperationCanceledException` handling.

diff --git a/Vostok.Applications.Scheduled.Tests/ScheduledActionRunner_Tests.cs b/Vostok.Applications.Scheduled.Tests/ScheduledActionRunner_Tests.cs
@@ -0,0 +1,206 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using FluentAssertions;
+using NUnit.Framework;
+using Vostok.Commons.Time;
+using Vostok.Logging.Abstractions;
+using Vostok.Tracing.Abstractions;
+
+namespace Vostok.Applications.Scheduled.Tests;
+
+[TestFixture]
+[Parallelizable(ParallelScope.All)]
+internal class ScheduledActionRunner_Tests
+{
+    private const string ActionName = "TestAction";
+
+    [Test]
+    public void RunAsync_should_throw_payload_exception_when_CrashOnPayloadException_true()
+    {
+        var options = new ScheduledActionOptions
+        {
+            CrashOnPayloadException = true
+        };
+
+        var action = new ScheduledAction(
+            ActionName,
+            Scheduler.Periodical(50.Milliseconds()),
+            options,
+            _ => throw new InvalidOperationException("Error"));
+
+        var runner = new ScheduledActionRunner(action, new SilentLog(), new DevNullTracer(), diagnostics: null);
+
+        var run = async () => await runner.RunAsync(GetCancellationToken());
+
+        run.Should().Throw<InvalidOperationException>();
+    }
+
+    [Test]
+    public void RunAsync_should_not_throw_payload_exception_when_CrashOnPayloadException_false()
+    {
+        var options = new ScheduledActionOptions
+        {
+            CrashOnPayloadException = false
+        };
+
+        var action = new ScheduledAction(
+            ActionName,
+            Scheduler.Periodical(50.Milliseconds()),
+            options,
+            _ => throw new InvalidOperationException("Error"));
+
+        var runner = new ScheduledActionRunner(action, new SilentLog(), new DevNullTracer(), diagnostics: null);
+
+        var run = async () => await runner.RunAsync(GetCancellationToken());
+
+        run.Should().NotThrow();
+    }
+
+    [Test]
+    public void RunAsync_should_throw_payload_OutOfMemoryException_even_when_CrashOnPayloadException_false()
+    {
+        var options = new ScheduledActionOptions
+        {
+            CrashOnPayloadException = false
+        };
+
+        var action = new ScheduledAction(
+            ActionName,
+            Scheduler.Periodical(50.Milliseconds()),
+            options,
+            _ => throw new OutOfMemoryException());
+
+        var runner = new ScheduledActionRunner(action, new SilentLog(), new DevNullTracer(), diagnostics: null);
+
+        var run = async () => await runner.RunAsync(GetCancellationToken());
+
+        run.Should().Throw<OutOfMemoryException>();
+    }
+
+    [Test]
+    public void RunAsync_should_not_throw_payload_OutOfMemoryException_when_CrashOnPayloadOutOfMemoryException_false()
+    {
+        var options = new ScheduledActionOptions
+        {
+            CrashOnPayloadException = false,
+            CrashOnPayloadOutOfMemoryException = false
+        };
+
+        var action = new ScheduledAction(
+            ActionName,
+            Scheduler.Periodical(50.Milliseconds()),
+            options,
+            _ => throw new OutOfMemoryException());
+
+        var runner = new ScheduledActionRunner(action, new SilentLog(), new DevNullTracer(), diagnostics: null);
+
+        var run = async () => await runner.RunAsync(GetCancellationToken());
+
+        run.Should().NotThrow();
+    }
+
+    [Test]
+    public void RunAsync_should_throw_scheduler_exception_when_CrashOnSchedulerException_true()
+    {
+        var options = new ScheduledActionOptions
+        {
+            CrashOnSchedulerException = true
+        };
+
+        var action = new ScheduledAction(
+            ActionName,
+            new ExceptionThrowingScheduler(new InvalidOperationException("Error")),
+            options,
+            _ => Task.CompletedTask);
+
+        var runner = new ScheduledActionRunner(action, new SilentLog(), new DevNullTracer(), diagnostics: null);
+
+        var run = async () => await runner.RunAsync(GetCancellationToken());
+
+        run.Should().Throw<InvalidOperationException>();
+    }
+
+    [Test]
+    public void RunAsync_should_not_throw_scheduler_exception_when_CrashOnSchedulerException_false()
+    {
+        var options = new ScheduledActionOptions
+        {
+            CrashOnSchedulerException = false
+        };
+
+        var action = new ScheduledAction(
+            ActionName,
+            new ExceptionThrowingScheduler(new InvalidOperationException("Error")),
+            options,
+            _ => Task.CompletedTask);
+
+        var runner = new ScheduledActionRunner(action, new SilentLog(), new DevNullTracer(), diagnostics: null);
+
+        var run = async () => await runner.RunAsync(GetCancellationToken());
+
+        run.Should().NotThrow();
+    }
+
+    [Test]
+    public void RunAsync_should_throw_scheduler_OutOfMemoryException_even_when_CrashOnSchedulerException_false()
+    {
+        var options = new ScheduledActionOptions
+        {
+            CrashOnSchedulerException = false
+        };
+
+        var action = new ScheduledAction(
+            ActionName,
+            new ExceptionThrowingScheduler(new OutOfMemoryException()),
+            options,
+            _ => Task.CompletedTask);
+
+        var runner = new ScheduledActionRunner(action, new SilentLog(), new DevNullTracer(), diagnostics: null);
+
+        var run = async () => await runner.RunAsync(GetCancellationToken());
+
+        run.Should().Throw<OutOfMemoryException>();
+    }
+
+    [Test]
+    public void RunAsync_should_not_throw_scheduler_OutOfMemoryException_when_CrashOnSchedulerOutOfMemoryException_false()
+    {
+        var options = new ScheduledActionOptions
+        {
+            CrashOnSchedulerException = false,
+            CrashOnSchedulerOutOfMemoryException = false
+        };
+
+        var action = new ScheduledAction(
+            ActionName,
+            new ExceptionThrowingScheduler(new OutOfMemoryException()),
+            options,
+            _ => Task.CompletedTask);
+
+        var runner = new ScheduledActionRunner(action, new SilentLog(), new DevNullTracer(), diagnostics: null);
+
+        var run = async () => await runner.RunAsync(GetCancellationToken());
+
+        run.Should().NotThrow();
+    }
+
+    private static CancellationToken GetCancellationToken()
+    {
+        var cts = new CancellationTokenSource();
+        cts.CancelAfter(1.Seconds());
+        return cts.Token;
+    }
+
+    private class ExceptionThrowingScheduler : IScheduler
+    {
+        private readonly Exception exception;
+
+        public ExceptionThrowingScheduler(Exception exception)
+        {
+            this.exception = exception;
+        }
+
+        public DateTimeOffset? ScheduleNext(DateTimeOffset from) => throw exception;
+    }
+}
diff --git a/Vostok.Applications.Scheduled.Tests/Vostok.Applications.Scheduled.Tests.csproj b/Vostok.Applications.Scheduled.Tests/Vostok.Applications.Scheduled.Tests.csproj
@@ -19,4 +19,16 @@
     <Compile Include="..\..\vostok.commons.testing\Vostok.Commons.Testing\AssertionAssertions.cs" Link="Commons\AssertionAssertions.cs" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Reference Include="Vostok.Logging.Abstractions">
+      <HintPath>..\..\vostok.logging.abstractions\Vostok.Logging.Abstractions\bin\Release\$(ReferencesFramework)\Vostok.Logging.Abstractions.dll</HintPath>
+    </Reference>
+    <Reference Include="Vostok.Tracing.Abstractions">
+      <HintPath>..\..\vostok.tracing.abstractions\Vostok.Tracing.Abstractions\bin\Release\$(ReferencesFramework)\Vostok.Tracing.Abstractions.dll</HintPath>
+    </Reference>
+    <Reference Include="Vostok.Hosting.Abstractions">
+      <HintPath>..\..\vostok.hosting.abstractions\Vostok.Hosting.Abstractions\bin\Release\$(ReferencesFramework)\Vostok.Hosting.Abstractions.dll</HintPath>
+    </Reference>
+  </ItemGroup>
+
 </Project>
diff --git a/Vostok.Applications.Scheduled/PublicAPI.Shipped.txt b/Vostok.Applications.Scheduled/PublicAPI.Shipped.txt
@@ -65,8 +65,12 @@ Vostok.Applications.Scheduled.ScheduledActionOptions.AllowOverlappingExecution.g
 Vostok.Applications.Scheduled.ScheduledActionOptions.AllowOverlappingExecution.set -> void
 Vostok.Applications.Scheduled.ScheduledActionOptions.CrashOnPayloadException.get -> bool
 Vostok.Applications.Scheduled.ScheduledActionOptions.CrashOnPayloadException.set -> void
+Vostok.Applications.Scheduled.ScheduledActionOptions.CrashOnPayloadOutOfMemoryException.get -> bool
+Vostok.Applications.Scheduled.ScheduledActionOptions.CrashOnPayloadOutOfMemoryException.set -> void
 Vostok.Applications.Scheduled.ScheduledActionOptions.CrashOnSchedulerException.get -> bool
 Vostok.Applications.Scheduled.ScheduledActionOptions.CrashOnSchedulerException.set -> void
+Vostok.Applications.Scheduled.ScheduledActionOptions.CrashOnSchedulerOutOfMemoryException.get -> bool
+Vostok.Applications.Scheduled.ScheduledActionOptions.CrashOnSchedulerOutOfMemoryException.set -> void
 Vostok.Applications.Scheduled.ScheduledActionOptions.PreferSeparateThread.get -> bool
 Vostok.Applications.Scheduled.ScheduledActionOptions.PreferSeparateThread.set -> void
 Vostok.Applications.Scheduled.ScheduledActionOptions.ScheduledActionOptions() -> void

diff --git a/Vostok.Applications.Scheduled/ScheduledActionOptions.cs b/Vostok.Applications.Scheduled/ScheduledActionOptions.cs
@@ -8,13 +8,17 @@ namespace Vostok.Applications.Scheduled
     public class ScheduledActionOptions
     {
         public bool CrashOnPayloadException { get; set; }
-
+
+        public bool CrashOnPayloadOutOfMemoryException { get; set; } = true;
+
         public bool CrashOnSchedulerException { get; set; }
-
+
+        public bool CrashOnSchedulerOutOfMemoryException { get; set; } = true;
+
         public bool PreferSeparateThread { get; set; }
 
         public bool AllowOverlappingExecution { get; set; }
 
         public TimeSpan ActualizationPeriod { get; set; } = 1.Seconds();
     }
-}
+}
diff --git a/Vostok.Applications.Scheduled/ScheduledActionRunner.cs b/Vostok.Applications.Scheduled/ScheduledActionRunner.cs
@@ -201,7 +201,8 @@ async Task ExecutePayload()
                     span.SetOperationStatus(null, WellKnownStatuses.Error);
                     monitor.OnIterationFailed(error);
 
-                    if (action.Options.CrashOnPayloadException)
+                    if (action.Options.CrashOnPayloadException
+                        || (action.Options.CrashOnPayloadOutOfMemoryException && error is OutOfMemoryException))
                         throw;
 
                     log.Error(error, "Scheduled action threw an exception.");
@@ -229,6 +230,10 @@ async Task ExecutePayload()
             {
                 return action.Scheduler.ScheduleNextWithSource(from);
             }
+            catch (OutOfMemoryException) when (action.Options.CrashOnSchedulerOutOfMemoryException)
+            {
+                throw;
+            }
             catch (Exception error)
             {
                 if (action.Options.CrashOnSchedulerException)
@@ -245,8 +250,9 @@ private void LogNextExecutionTime(DateTimeOffset? nextExecutionTime)
             if (nextExecutionTime == null)
                 log.Info("Next execution time: unknown.");
             else
-                log.Info("Next execution time = {NextExecutionTime:yyyy-MM-dd HH:mm:ss.fff} (~{TimeToNextExecution} from now).", 
-                    nextExecutionTime.Value.DateTime, TimeSpanArithmetics.Max(TimeSpan.Zero, nextExecutionTime.Value - PreciseDateTime.Now).ToPrettyString());
+                log.Info("Next execution time = {NextExecutionTime:yyyy-MM-dd HH:mm:ss.fff} (~{TimeToNextExecution} from now).",
+                    nextExecutionTime.Value.DateTime,
+                    TimeSpanArithmetics.Max(TimeSpan.Zero, nextExecutionTime.Value - PreciseDateTime.Now).ToPrettyString());
         }
 
         private void RegisterDiagnostics(IVostokApplicationDiagnostics diagnostics)
@@ -260,4 +266,4 @@ private void RegisterDiagnostics(IVostokApplicationDiagnostics diagnostics)
             healthCheckRegistration = diagnostics.HealthTracker.RegisterCheck($"scheduled ({info.Name})", healthCheck);
         }
     }
-}
+}
diff --git a/Vostok.Applications.Scheduled/Vostok.Applications.Scheduled.csproj b/Vostok.Applications.Scheduled/Vostok.Applications.Scheduled.csproj
@@ -7,7 +7,7 @@
     <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
   </PropertyGroup>
   <PropertyGroup>
-    <VersionPrefix>0.2.14</VersionPrefix>
+    <VersionPrefix>0.2.15</VersionPrefix>
   </PropertyGroup>
   <PropertyGroup>
     <Title>Vostok.Applications.Scheduled</Title>