From b01763167a9149d4703b4251fd71e2a7d2c4577d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Michael=20R=C3=A4tzel?= <viir@viir.de>
Date: Sun, 2 Jul 2023 06:57:24 +0000
Subject: [PATCH] Adapt explicit disposal of volatile processes to Azure
 Pipelines

With the previous version, Azure Pipeline sometimes failed the execution of the tests, not completing the test step.
Looking into these failures, I also found the following message in the logs from Azure Pipelines:

> The STDIO streams did not close within 10 seconds of the exit event from process 'C:\hostedtoolcache\windows\dotnet\dotnet.exe'. This may indicate a child process inherited the STDIO streams and has not yet exited.

For discussions of the problem, see also:

+ https://github.com/microsoft/azure-pipelines-tasks/issues/17548
+ https://github.com/microsoft/azure-pipelines-tasks/issues/13033
+ https://github.com/microsoft/azure-pipelines-tasks/issues/18476
---
 .../elm-time/Pine/VolatileProcessNative.cs    | 34 +++++++++++++++----
 .../Platform/WebService/PublicAppState.cs     | 12 +++++++
 implement/elm-time/Program.cs                 |  2 +-
 implement/elm-time/elm-time.csproj            |  4 +--
 4 files changed, 42 insertions(+), 10 deletions(-)

diff --git a/implement/elm-time/Pine/VolatileProcessNative.cs b/implement/elm-time/Pine/VolatileProcessNative.cs
index 35a2a3f7..a57cf052 100644
--- a/implement/elm-time/Pine/VolatileProcessNative.cs
+++ b/implement/elm-time/Pine/VolatileProcessNative.cs
@@ -26,6 +26,8 @@ public record ReadAllFromNativeProcess(
 
     private readonly ConcurrentQueue<ReadOnlyMemory<byte>> stdErrQueue = new();
 
+    private readonly CancellationTokenSource disposedCancellationTokenSource = new();
+
     public VolatileProcessNative(
         Func<byte[], byte[]?>? getFileFromHashSHA256,
         ElmTime.Platform.WebService.InterfaceToHost.CreateVolatileProcessNativeStruct createRequest)
@@ -70,8 +72,15 @@ public record ReadAllFromNativeProcess(
 
         process.Start();
 
-        Task.Run(() => ReadFromStreamToQueue(stdOutQueue, process.StandardOutput.BaseStream));
-        Task.Run(() => ReadFromStreamToQueue(stdErrQueue, process.StandardError.BaseStream));
+        Task.Run(() => ReadFromStreamToQueueLoopAsync(
+            stdOutQueue,
+            process.StandardOutput.BaseStream,
+            cancellationToken: disposedCancellationTokenSource.Token));
+
+        Task.Run(() => ReadFromStreamToQueueLoopAsync(
+            stdErrQueue,
+            process.StandardError.BaseStream,
+            cancellationToken: disposedCancellationTokenSource.Token));
     }
 
     public void WriteToStdIn(ReadOnlyMemory<byte> data)
@@ -111,17 +120,25 @@ public static ReadOnlyMemory<byte> ReadAllFromStream(Stream stream)
         return CommonConversion.Concat(buffers);
     }
 
-    public static long ReadFromStreamToQueue(ConcurrentQueue<ReadOnlyMemory<byte>> queue, Stream stream)
+    public static async Task<long> ReadFromStreamToQueueLoopAsync(
+        ConcurrentQueue<ReadOnlyMemory<byte>> queue,
+        Stream stream,
+        CancellationToken cancellationToken)
     {
         long aggregateBytesRead = 0;
 
         try
         {
-            while (true)
+            while (!cancellationToken.IsCancellationRequested)
             {
-                var buffer = new byte[0x10_000];
+                var buffer = new byte[0x100_000];
 
-                var bytesRead = stream.ReadAtLeast(buffer, minimumBytes: 1, throwOnEndOfStream: true);
+                var bytesRead =
+                    await stream.ReadAtLeastAsync(
+                        buffer,
+                        minimumBytes: 1,
+                        throwOnEndOfStream: true,
+                        cancellationToken: cancellationToken);
 
                 if (bytesRead == 0)
                     continue;
@@ -133,14 +150,17 @@ public static long ReadFromStreamToQueue(ConcurrentQueue<ReadOnlyMemory<byte>> q
         }
         catch (EndOfStreamException)
         {
-            return aggregateBytesRead;
         }
+
+        return aggregateBytesRead;
     }
 
     public void Dispose()
     {
         try
         {
+            disposedCancellationTokenSource.Cancel();
+
             if (process is not null && !process.HasExited)
                 process.Kill();
 
diff --git a/implement/elm-time/Platform/WebService/PublicAppState.cs b/implement/elm-time/Platform/WebService/PublicAppState.cs
index 4ddef32a..a90c642d 100644
--- a/implement/elm-time/Platform/WebService/PublicAppState.cs
+++ b/implement/elm-time/Platform/WebService/PublicAppState.cs
@@ -136,6 +136,7 @@ public class PublicAppState
         {
             applicationStoppingCancellationTokenSource.Cancel();
             app.Logger?.LogInformation("Public app noticed ApplicationStopping.");
+            DisposeAsync().Wait();
         });
 
         app.UseResponseCompression();
@@ -151,6 +152,17 @@ public class PublicAppState
         return app;
     }
 
+    async System.Threading.Tasks.Task DisposeAsync()
+    {
+        await
+            System.Threading.Tasks.Task.WhenAll(
+                volatileProcesses.Select(kvp =>
+                System.Threading.Tasks.Task.Run(() =>
+                {
+                    (kvp.Value as IDisposable)?.Dispose();
+                })));
+    }
+
     private void ConfigureServices(
         IServiceCollection services,
         ILogger logger)
diff --git a/implement/elm-time/Program.cs b/implement/elm-time/Program.cs
index ae8c523f..183ad9fc 100644
--- a/implement/elm-time/Program.cs
+++ b/implement/elm-time/Program.cs
@@ -17,7 +17,7 @@ namespace ElmTime;
 
 public class Program
 {
-    public static string AppVersionId => "2023-06-30";
+    public static string AppVersionId => "2023-07-01";
 
     private static int AdminInterfaceDefaultPort => 4000;
 
diff --git a/implement/elm-time/elm-time.csproj b/implement/elm-time/elm-time.csproj
index 63c68299..321cb7cd 100644
--- a/implement/elm-time/elm-time.csproj
+++ b/implement/elm-time/elm-time.csproj
@@ -5,8 +5,8 @@
     <TargetFramework>net7.0</TargetFramework>
     <RootNamespace>ElmTime</RootNamespace>
     <AssemblyName>elm-time</AssemblyName>
-    <AssemblyVersion>2023.0630.0.0</AssemblyVersion>
-    <FileVersion>2023.0630.0.0</FileVersion>
+    <AssemblyVersion>2023.0701.0.0</AssemblyVersion>
+    <FileVersion>2023.0701.0.0</FileVersion>
     <Nullable>enable</Nullable>
     <GenerateEmbeddedFilesManifest>true</GenerateEmbeddedFilesManifest>
   </PropertyGroup>