diff --git a/src/ServiceControl.Audit/Infrastructure/RavenDB/RavenBootstrapper.cs b/src/ServiceControl.Audit/Infrastructure/RavenDB/RavenBootstrapper.cs index aeec8f6f90..a8e788eb9e 100644 --- a/src/ServiceControl.Audit/Infrastructure/RavenDB/RavenBootstrapper.cs +++ b/src/ServiceControl.Audit/Infrastructure/RavenDB/RavenBootstrapper.cs @@ -9,6 +9,7 @@ using NServiceBus.Logging; using Raven.Client.Embedded; using Raven.Client.Indexes; + using ServiceControl.Infrastructure.RavenDB; using ServiceControl.SagaAudit; using Settings; @@ -90,6 +91,12 @@ public void StartRaven(EmbeddableDocumentStore documentStore, Settings settings, documentStore.Initialize(); + if (!maintenanceMode) + { + documentStore.ThrowWhenIndexErrors(); + documentStore.WaitUntilNoStaleIndexes(); + } + Logger.Info("Index creation started"); IndexCreation.CreateIndexes(typeof(RavenBootstrapper).Assembly, documentStore); diff --git a/src/ServiceControl.Infrastructure.RavenDB/Extensions.cs b/src/ServiceControl.Infrastructure.RavenDB/Extensions.cs index c2dc61f6f4..d87e3a2bbb 100644 --- a/src/ServiceControl.Infrastructure.RavenDB/Extensions.cs +++ b/src/ServiceControl.Infrastructure.RavenDB/Extensions.cs @@ -1,8 +1,11 @@ namespace ServiceControl.Infrastructure.RavenDB { using System; + using System.Text; using System.Threading; + using NServiceBus.Logging; using Raven.Abstractions.Data; + using Raven.Client.Embedded; using Raven.Database; using Raven.Json.Linq; @@ -16,5 +19,48 @@ public static void Query(this DocumentDatabase db, string index, IndexQu onItem(doc, state); } } + + public static void ThrowWhenIndexErrors(this EmbeddableDocumentStore documentStore) + { + var statistics = documentStore.DatabaseCommands.GetStatistics(); + + if (statistics.Errors.Length > 0) + { + var text = new StringBuilder(); + text.AppendLine("Detected RavenDB index errors, please start maintenance mode and resolve the following issues:"); + foreach (var indexError in statistics.Errors) + { + text.AppendLine($"- Index [{indexError.IndexName}] error: {indexError.Error} (Action: {indexError.Action}, Doc: {indexError.Document}, At: {indexError.Timestamp})"); + } + throw new Exception(text.ToString()); + } + } + + public static void WaitUntilNoStaleIndexes(this EmbeddableDocumentStore documentStore) + { + var interval = TimeSpan.FromMinutes(1); + var next = DateTime.MinValue; + string[] staleIndexes; + + // Check for the number of stale indexes every second, but report only an update only every 1 minutes + while ((staleIndexes = documentStore.DatabaseCommands.GetStatistics().StaleIndexes).Length > 0) + { + var now = DateTime.UtcNow; + if (next < now) + { + var text = new StringBuilder(); + text.AppendLine("Stale indexes detected, delaying start until all indexes are non-stale. DO NOT KILL THIS PROCESS! Operation can run for a very long time!"); + foreach (var staleIndex in staleIndexes) + { + text.AppendLine($"- {staleIndex}"); + } + Log.Warn(text.ToString()); + next = now + interval; + } + Thread.Sleep(1000); + } + } + + static ILog Log = LogManager.GetLogger(typeof(Extensions).Namespace); } } \ No newline at end of file diff --git a/src/ServiceControl/Infrastructure/RavenDB/RavenBootstrapper.cs b/src/ServiceControl/Infrastructure/RavenDB/RavenBootstrapper.cs index afc340aa2c..c813400f4f 100644 --- a/src/ServiceControl/Infrastructure/RavenDB/RavenBootstrapper.cs +++ b/src/ServiceControl/Infrastructure/RavenDB/RavenBootstrapper.cs @@ -100,6 +100,14 @@ public void StartRaven(EmbeddableDocumentStore documentStore, Settings settings, documentStore.Initialize(); + if (!maintenanceMode) + { + documentStore.ThrowWhenIndexErrors(); + // Only purge endpoints when not in maintenance mode + PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicate(documentStore); + documentStore.WaitUntilNoStaleIndexes(); + } + Logger.Info("Index creation started"); IndexCreation.CreateIndexes(typeof(RavenBootstrapper).Assembly, documentStore);