From c5155bafc4d7452e90e253807d6acb2712217240 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 7 Apr 2021 12:54:21 +0200 Subject: [PATCH 1/3] Check for RavenDB index errors and delay start until no stale indexes. --- .../RavenDB/RavenBootstrapper.cs | 7 ++++ .../Extensions.cs | 37 +++++++++++++++++++ .../RavenDB/RavenBootstrapper.cs | 18 ++++++++- 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/src/ServiceControl.Audit/Infrastructure/RavenDB/RavenBootstrapper.cs b/src/ServiceControl.Audit/Infrastructure/RavenDB/RavenBootstrapper.cs index aeec8f6f90..a8e788eb9e 100644 --- a/src/ServiceControl.Audit/Infrastructure/RavenDB/RavenBootstrapper.cs +++ b/src/ServiceControl.Audit/Infrastructure/RavenDB/RavenBootstrapper.cs @@ -9,6 +9,7 @@ using NServiceBus.Logging; using Raven.Client.Embedded; using Raven.Client.Indexes; + using ServiceControl.Infrastructure.RavenDB; using ServiceControl.SagaAudit; using Settings; @@ -90,6 +91,12 @@ public void StartRaven(EmbeddableDocumentStore documentStore, Settings settings, documentStore.Initialize(); + if (!maintenanceMode) + { + documentStore.ThrowWhenIndexErrors(); + documentStore.WaitUntilNoStaleIndexes(); + } + Logger.Info("Index creation started"); IndexCreation.CreateIndexes(typeof(RavenBootstrapper).Assembly, documentStore); diff --git a/src/ServiceControl.Infrastructure.RavenDB/Extensions.cs b/src/ServiceControl.Infrastructure.RavenDB/Extensions.cs index c2dc61f6f4..fe759980e0 100644 --- a/src/ServiceControl.Infrastructure.RavenDB/Extensions.cs +++ b/src/ServiceControl.Infrastructure.RavenDB/Extensions.cs @@ -1,8 +1,10 @@ namespace ServiceControl.Infrastructure.RavenDB { using System; + using System.Text; using System.Threading; using Raven.Abstractions.Data; + using Raven.Client.Embedded; using Raven.Database; using Raven.Json.Linq; @@ -16,5 +18,40 @@ public static void Query(this DocumentDatabase db, string index, IndexQu onItem(doc, state); } } + + public static void ThrowWhenIndexErrors(this EmbeddableDocumentStore documentStore) + { + var statistics = documentStore.DatabaseCommands.GetStatistics(); + + if (statistics.Errors.Length > 0) + { + var text = new StringBuilder(); + text.AppendLine("Detected RavenDB index errors, please start maintenance mode and resolve the following issues:"); + foreach (var indexError in statistics.Errors) + { + text.AppendLine($"Index [{indexError.IndexName}] error: {indexError.Error} (Action: {indexError.Action}, Doc: {indexError.Document}, At: {indexError.Timestamp})"); + } + throw new Exception(text.ToString()); + } + } + + public static void WaitUntilNoStaleIndexes(this EmbeddableDocumentStore documentStore) + { + var interval = TimeSpan.FromMinutes(1); + var next = DateTime.MinValue; + int staleIndexes; + + while ((staleIndexes = documentStore.DatabaseCommands.GetStatistics().StaleIndexes.Length) > 0) + { + var now = DateTime.UtcNow; + if (next < now) + { + Console.WriteLine($"Stale indexes detected ({staleIndexes}), will delay starting until all indexes are non-stale. DO NOT KILL THIS PROCESS! Operation can run for a very long time!"); + next = now + interval; + } + + Thread.Sleep(1000); + } + } } } \ No newline at end of file diff --git a/src/ServiceControl/Infrastructure/RavenDB/RavenBootstrapper.cs b/src/ServiceControl/Infrastructure/RavenDB/RavenBootstrapper.cs index afc340aa2c..a37faa8f1d 100644 --- a/src/ServiceControl/Infrastructure/RavenDB/RavenBootstrapper.cs +++ b/src/ServiceControl/Infrastructure/RavenDB/RavenBootstrapper.cs @@ -1,10 +1,18 @@ -namespace ServiceControl.Infrastructure.RavenDB +using System; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Raven.Client.Embedded; + +namespace ServiceControl.Infrastructure.RavenDB { using System; using System.ComponentModel.Composition.Hosting; using System.IO; using System.Linq; using System.Runtime.Serialization; + using System.Threading; + using System.Threading.Tasks; using Audit.Monitoring; using Monitoring; using NServiceBus; @@ -100,6 +108,14 @@ public void StartRaven(EmbeddableDocumentStore documentStore, Settings settings, documentStore.Initialize(); + if (!maintenanceMode) + { + documentStore.ThrowWhenIndexErrors(); + // Only purge endpoints when not in maintenance mode + PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicate(documentStore); + documentStore.WaitUntilNoStaleIndexes(); + } + Logger.Info("Index creation started"); IndexCreation.CreateIndexes(typeof(RavenBootstrapper).Assembly, documentStore); From dbf7f99047c952f890d97b5ba688b21b6ce95859 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 7 Apr 2021 18:46:34 +0200 Subject: [PATCH 2/3] Unused namespace cleanup --- .../Infrastructure/RavenDB/RavenBootstrapper.cs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/ServiceControl/Infrastructure/RavenDB/RavenBootstrapper.cs b/src/ServiceControl/Infrastructure/RavenDB/RavenBootstrapper.cs index a37faa8f1d..c813400f4f 100644 --- a/src/ServiceControl/Infrastructure/RavenDB/RavenBootstrapper.cs +++ b/src/ServiceControl/Infrastructure/RavenDB/RavenBootstrapper.cs @@ -1,18 +1,10 @@ -using System; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Raven.Client.Embedded; - -namespace ServiceControl.Infrastructure.RavenDB +namespace ServiceControl.Infrastructure.RavenDB { using System; using System.ComponentModel.Composition.Hosting; using System.IO; using System.Linq; using System.Runtime.Serialization; - using System.Threading; - using System.Threading.Tasks; using Audit.Monitoring; using Monitoring; using NServiceBus; From 8f6e75099f5fd2473816cf65da4410943da70d13 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 8 Apr 2021 09:13:42 +0200 Subject: [PATCH 3/3] Now writing to log instead of console with improved message. --- .../Extensions.cs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/ServiceControl.Infrastructure.RavenDB/Extensions.cs b/src/ServiceControl.Infrastructure.RavenDB/Extensions.cs index fe759980e0..d87e3a2bbb 100644 --- a/src/ServiceControl.Infrastructure.RavenDB/Extensions.cs +++ b/src/ServiceControl.Infrastructure.RavenDB/Extensions.cs @@ -3,6 +3,7 @@ using System; using System.Text; using System.Threading; + using NServiceBus.Logging; using Raven.Abstractions.Data; using Raven.Client.Embedded; using Raven.Database; @@ -29,7 +30,7 @@ public static void ThrowWhenIndexErrors(this EmbeddableDocumentStore documentSto text.AppendLine("Detected RavenDB index errors, please start maintenance mode and resolve the following issues:"); foreach (var indexError in statistics.Errors) { - text.AppendLine($"Index [{indexError.IndexName}] error: {indexError.Error} (Action: {indexError.Action}, Doc: {indexError.Document}, At: {indexError.Timestamp})"); + text.AppendLine($"- Index [{indexError.IndexName}] error: {indexError.Error} (Action: {indexError.Action}, Doc: {indexError.Document}, At: {indexError.Timestamp})"); } throw new Exception(text.ToString()); } @@ -39,19 +40,27 @@ public static void WaitUntilNoStaleIndexes(this EmbeddableDocumentStore document { var interval = TimeSpan.FromMinutes(1); var next = DateTime.MinValue; - int staleIndexes; + string[] staleIndexes; - while ((staleIndexes = documentStore.DatabaseCommands.GetStatistics().StaleIndexes.Length) > 0) + // Check for the number of stale indexes every second, but report only an update only every 1 minutes + while ((staleIndexes = documentStore.DatabaseCommands.GetStatistics().StaleIndexes).Length > 0) { var now = DateTime.UtcNow; if (next < now) { - Console.WriteLine($"Stale indexes detected ({staleIndexes}), will delay starting until all indexes are non-stale. DO NOT KILL THIS PROCESS! Operation can run for a very long time!"); + var text = new StringBuilder(); + text.AppendLine("Stale indexes detected, delaying start until all indexes are non-stale. DO NOT KILL THIS PROCESS! Operation can run for a very long time!"); + foreach (var staleIndex in staleIndexes) + { + text.AppendLine($"- {staleIndex}"); + } + Log.Warn(text.ToString()); next = now + interval; } - Thread.Sleep(1000); } } + + static ILog Log = LogManager.GetLogger(typeof(Extensions).Namespace); } } \ No newline at end of file