🐛 Connecting to RavenDB database can timeout with `DatabaseLoadTimeou…

…tException`. Will retry until the cancellationtoken is set
Particular · Jan 8, 2024 · ef3e553 · ef3e553
1 parent 9762cdd
commit ef3e553
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 4 deletions.
diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenEmbeddedPersistenceLifecycle.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenEmbeddedPersistenceLifecycle.cs
@@ -3,7 +3,9 @@
     using System;
     using System.Threading;
     using System.Threading.Tasks;
+    using NServiceBus.Logging;
     using Raven.Client.Documents;
+    using Raven.Client.Exceptions.Database;
 
     class RavenEmbeddedPersistenceLifecycle : IRavenPersistenceLifecycle
     {
@@ -26,7 +28,20 @@ public async Task Start(CancellationToken cancellationToken)
         {
             database = EmbeddedDatabase.Start(databaseConfiguration);
 
-            documentStore = await database.Connect(cancellationToken);
+            while (true)
+            {
+                cancellationToken.ThrowIfCancellationRequested();
+
+                try
+                {
+                    documentStore = await database.Connect(cancellationToken);
+                }
+                catch (DatabaseLoadTimeoutException e)
+                {
+                    Log.Warn("Could not connect to database. Retrying in 500ms...", e);
+                    await Task.Delay(500, cancellationToken);
+                }
+            }
         }
 
         public Task Stop(CancellationToken cancellationToken)
@@ -39,7 +54,7 @@ public Task Stop(CancellationToken cancellationToken)
 
         IDocumentStore documentStore;
         EmbeddedDatabase database;
-
+        static readonly ILog Log = LogManager.GetLogger(typeof(RavenEmbeddedPersistenceLifecycle));
         readonly DatabaseConfiguration databaseConfiguration;
     }
 }
diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs
@@ -4,6 +4,7 @@
     using System.IO;
     using System.Threading;
     using System.Threading.Tasks;
+    using NServiceBus.Logging;
     using NUnit.Framework;
     using ServiceControl.Audit.Persistence.RavenDB;
     using TestHelper;
@@ -40,8 +41,9 @@ public static async Task<EmbeddedDatabase> GetInstance(CancellationToken cancell
 
                         return embeddedDatabase;
                     }
-                    catch (Exception)
+                    catch (Exception e)
                     {
+                        Log.Warn("Could not connect to database. Retrying in 500ms...", e);
                         await Task.Delay(500, cancellationToken);
                     }
                 }
@@ -68,5 +70,6 @@ public static async Task Stop(CancellationToken cancellationToken = default)
 
         static EmbeddedDatabase embeddedDatabase;
         static SemaphoreSlim semaphoreSlim = new SemaphoreSlim(1, 1);
+        static readonly ILog Log = LogManager.GetLogger(typeof(SharedEmbeddedServer));
     }
 }
diff --git a/src/ServiceControl.Persistence.RavenDB/RavenEmbeddedPersistenceLifecycle.cs b/src/ServiceControl.Persistence.RavenDB/RavenEmbeddedPersistenceLifecycle.cs
@@ -4,7 +4,9 @@
     using System.Diagnostics;
     using System.Threading;
     using System.Threading.Tasks;
+    using NServiceBus.Logging;
     using Raven.Client.Documents;
+    using Raven.Client.Exceptions.Database;
     using ServiceControl.Persistence;
 
     class RavenEmbeddedPersistenceLifecycle : IPersistenceLifecycle, IDisposable
@@ -27,7 +29,21 @@ public IDocumentStore GetDocumentStore()
         public async Task Initialize(CancellationToken cancellationToken)
         {
             database = EmbeddedDatabase.Start(databaseConfiguration);
-            documentStore = await database.Connect(cancellationToken);
+
+            while (true)
+            {
+                cancellationToken.ThrowIfCancellationRequested();
+
+                try
+                {
+                    documentStore = await database.Connect(cancellationToken);
+                }
+                catch (DatabaseLoadTimeoutException e)
+                {
+                    Log.Warn("Could not connect to database. Retrying in 500ms...", e);
+                    await Task.Delay(500, cancellationToken);
+                }
+            }
         }
 
         public void Dispose()
@@ -41,6 +57,7 @@ public void Dispose()
         EmbeddedDatabase database;
 
         readonly RavenPersisterSettings databaseConfiguration;
+        static readonly ILog Log = LogManager.GetLogger(typeof(RavenEmbeddedPersistenceLifecycle));
 
         ~RavenEmbeddedPersistenceLifecycle()
         {