Check for RavenDB index errors and delay start until no stale indexes

src/ServiceControl.Audit/Infrastructure/RavenDB/RavenBootstrapper.cs

@@ -9,6 +9,7 @@
     using NServiceBus.Logging;
     using Raven.Client.Embedded;
     using Raven.Client.Indexes;
+    using ServiceControl.Infrastructure.RavenDB;
     using ServiceControl.SagaAudit;
     using Settings;
 
@@ -90,6 +91,12 @@ public void StartRaven(EmbeddableDocumentStore documentStore, Settings settings,
 
             documentStore.Initialize();
 
+            if (!maintenanceMode)
+            {
+                documentStore.ThrowWhenIndexErrors();
+                documentStore.WaitUntilNoStaleIndexes();
+            }
+
             Logger.Info("Index creation started");
 
             IndexCreation.CreateIndexes(typeof(RavenBootstrapper).Assembly, documentStore);

src/ServiceControl.Infrastructure.RavenDB/Extensions.cs

@@ -1,8 +1,11 @@
 namespace ServiceControl.Infrastructure.RavenDB
 {
     using System;
+    using System.Text;
     using System.Threading;
+    using NServiceBus.Logging;
     using Raven.Abstractions.Data;
+    using Raven.Client.Embedded;
     using Raven.Database;
     using Raven.Json.Linq;
 
@@ -16,5 +19,48 @@ public static void Query<TState>(this DocumentDatabase db, string index, IndexQu
                 onItem(doc, state);
             }
         }
+
+        public static void ThrowWhenIndexErrors(this EmbeddableDocumentStore documentStore)
+        {
+            var statistics = documentStore.DatabaseCommands.GetStatistics();
+
+            if (statistics.Errors.Length > 0)
+            {
+                var text = new StringBuilder();
+                text.AppendLine("Detected RavenDB index errors, please start maintenance mode and resolve the following issues:");
+                foreach (var indexError in statistics.Errors)
+                {
+                    text.AppendLine($"- Index [{indexError.IndexName}] error: {indexError.Error} (Action: {indexError.Action},  Doc: {indexError.Document}, At: {indexError.Timestamp})");
+                }
+                throw new Exception(text.ToString());
+            }
+        }
+
+        public static void WaitUntilNoStaleIndexes(this EmbeddableDocumentStore documentStore)
+        {
+            var interval = TimeSpan.FromMinutes(1);
+            var next = DateTime.MinValue;
+            string[] staleIndexes;
+
+            // Check for the number of stale indexes every second, but report only an update only every 1 minutes
+            while ((staleIndexes = documentStore.DatabaseCommands.GetStatistics().StaleIndexes).Length > 0)
+            {
+                var now = DateTime.UtcNow;
+                if (next < now)
+                {
+                    var text = new StringBuilder();
+                    text.AppendLine("Stale indexes detected, delaying start until all indexes are non-stale. DO NOT KILL THIS PROCESS! Operation can run for a very long time!");
+                    foreach (var staleIndex in staleIndexes)
+                    {
+                        text.AppendLine($"- {staleIndex}");
+                    }
+                    Log.Warn(text.ToString());
+                    next = now + interval;
+                }
+                Thread.Sleep(1000);
+            }
+        }
+
+        static ILog Log = LogManager.GetLogger(typeof(Extensions).Namespace);
     }
 }

src/ServiceControl/Infrastructure/RavenDB/RavenBootstrapper.cs

@@ -100,6 +100,14 @@ public void StartRaven(EmbeddableDocumentStore documentStore, Settings settings,
 
             documentStore.Initialize();
 
+            if (!maintenanceMode)
+            {
+                documentStore.ThrowWhenIndexErrors();
+                // Only purge endpoints when not in maintenance mode
+                PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicate(documentStore);
+                documentStore.WaitUntilNoStaleIndexes();
+            }
+
             Logger.Info("Index creation started");
 
             IndexCreation.CreateIndexes(typeof(RavenBootstrapper).Assembly, documentStore);