Better handling of OOME during indexing

Raven.Database/Indexing/AbstractIndexingExecuter.cs

@@ -40,16 +40,30 @@ public void Execute()
 				try
 				{
 					foundWork = ExecuteIndexing();
-					while(context.DoWork) // we want to drain all of the pending tasks before the next run
+					while (context.DoWork) // we want to drain all of the pending tasks before the next run
 					{
 						if (ExecuteTasks() == false)
 							break;
 						foundWork = true;
 					}
-
+
+				}
+				catch (OutOfMemoryException oome)
+				{
+					foundWork = true; // otherwise, we wouldn't be getting OOME
+					log.WarnException(
+						@"Failed to execute indexing because of an out of memory exception. Will force a full GC cycle and then become more conservative with regards to memory",
+						oome);
+
+					// On the face of it, this is stupid, because OOME will not be thrown if the GC could release
+					// memory, but we are actually aware that during indexing, the GC couldn't find garbage to clean,
+					// but in here, we are AFTER the index was done, so there is likely to be a lot of garbage.
+					GC.Collect(GC.MaxGeneration);
+					autoTuner.OutOfMemoryExceptionHappened();
 				}
 				catch (Exception e)
 				{
+					foundWork = true; // we want to keep on trying, anyway, not wait for the timeout or more work
 					log.ErrorException("Failed to execute indexing", e);
 				}
 				if (foundWork == false)
@@ -127,7 +141,7 @@ protected bool ExecuteIndexing()
 				return false;
 
 			ExecuteIndxingWork(indexesToWorkOn);
-			
+
 			return true;
 		}
 

Raven.Database/Indexing/FairIndexingSchedulerWithNewIndexesBias.cs

@@ -14,7 +14,14 @@ public class FairIndexingSchedulerWithNewIndexesBias : IIndexingScheduler
 		private int current;
 		private int currentRepeated;
 		private bool activeFiltering;
-		private int lastAmountOfItemsToIndex;
+		private List<int> lastAmountOfItemsToIndex = new List<int>();
+
+		public FairIndexingSchedulerWithNewIndexesBias()
+		{
+			LastAmountOfItemsToIndexToRemember = 1;
+		}
+
+		public int LastAmountOfItemsToIndexToRemember { get; set; }
 
 		public IList<IndexToWorkOn> FilterMapIndexes(IList<IndexToWorkOn> indexes)
 		{
@@ -55,18 +62,31 @@ public IList<IndexToWorkOn> FilterMapIndexes(IList<IndexToWorkOn> indexes)
 			return indexToWorkOns.ToList();
 		}
 
-		public int LastAmountOfItemsToIndex
+		public void RecordAmountOfItemsToIndex(int value)
 		{
-			get { return lastAmountOfItemsToIndex; }
-			set
+			var currentLastAmountOfItemsToIndex = lastAmountOfItemsToIndex;
+			var amountOfItemsToIndex = activeFiltering
+			                           	? // if we are actively filtering, we have multiple levels, so we have to assume 
+			                           // that the max amount is still the current one, this prevent the different levels of indexing batch
+			                           // size from "fighting" over the batch size.
+			                           Math.Max(currentLastAmountOfItemsToIndex.Max(), value)
+			                           	: value;
+
+			var amountToTake = currentLastAmountOfItemsToIndex.Count;
+			if (amountToTake + 1 >= LastAmountOfItemsToIndexToRemember)
 			{
-				lastAmountOfItemsToIndex = activeFiltering
-				                           	? // if we are actively filtering, we have multiple levels, so we have to assume 
-				                           // that the max amount is still the current one, this prevent the different levels of indexing batch
-				                           // size from "fighting" over the batch size.
-				                           Math.Max(lastAmountOfItemsToIndex, value)
-				                           	: value;
+				amountToTake = currentLastAmountOfItemsToIndex.Count-1;
 			}
+			lastAmountOfItemsToIndex = new List<int>(currentLastAmountOfItemsToIndex.Take(amountToTake))
+				{
+					amountOfItemsToIndex
+				};
+
+		}
+
+		public IEnumerable<int> GetLastAmountOfItemsToIndex()
+		{
+			return lastAmountOfItemsToIndex;
 		}
 
 		// here we compare, but only up to the last 116 bits, not the full 128 bits

Raven.Database/Indexing/IIndexingScheduler.cs

@@ -5,6 +5,8 @@ namespace Raven.Database.Indexing
 	public interface IIndexingScheduler
 	{
 		IList<IndexToWorkOn> FilterMapIndexes(IList<IndexToWorkOn> indexes);
-		int LastAmountOfItemsToIndex { get; set; }
+		void RecordAmountOfItemsToIndex(int value);
+		IEnumerable<int> GetLastAmountOfItemsToIndex();
+		int LastAmountOfItemsToIndexToRemember { get; set; }
 	}
 }