From 9c38decabb5dbf4180045b4e4d1477ad5f801341 Mon Sep 17 00:00:00 2001 From: Guido Tagliavini Ponce Date: Wed, 11 Feb 2015 14:37:41 -0800 Subject: [PATCH 1/7] Added maxSize >= 0 check. --- src/Lucene.Net.Core/Util/PriorityQueue.cs | 42 ++++++++++++++--------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/src/Lucene.Net.Core/Util/PriorityQueue.cs b/src/Lucene.Net.Core/Util/PriorityQueue.cs index bd06be0ab8..f601de8d6f 100644 --- a/src/Lucene.Net.Core/Util/PriorityQueue.cs +++ b/src/Lucene.Net.Core/Util/PriorityQueue.cs @@ -45,33 +45,41 @@ public PriorityQueue(int maxSize) public PriorityQueue(int maxSize, bool prepopulate) { int heapSize; - if (0 == maxSize) + if (maxSize < 0) { - // We allocate 1 extra to avoid if statement in top() - heapSize = 2; + throw new System.ArgumentException("maxSize must be >= 0; got: " + maxSize); } else { - if (maxSize > ArrayUtil.MAX_ARRAY_LENGTH) + if (0 == maxSize) { - // Don't wrap heapSize to -1, in this case, which - // causes a confusing NegativeArraySizeException. - // Note that very likely this will simply then hit - // an OOME, but at least that's more indicative to - // caller that this values is too big. We don't +1 - // in this case, but it's very unlikely in practice - // one will actually insert this many objects into - // the PQ: - // Throw exception to prevent confusing OOME: - throw new System.ArgumentException("maxSize must be <= " + ArrayUtil.MAX_ARRAY_LENGTH + "; got: " + maxSize); + // We allocate 1 extra to avoid if statement in top() + heapSize = 2; } else { - // NOTE: we add +1 because all access to heap is - // 1-based not 0-based. heap[0] is unused. - heapSize = maxSize + 1; + if (maxSize >= ArrayUtil.MAX_ARRAY_LENGTH) + { + // Don't wrap heapSize to -1, in this case, which + // causes a confusing NegativeArraySizeException. + // Note that very likely this will simply then hit + // an OOME, but at least that's more indicative to + // caller that this values is too big. We don't +1 + // in this case, but it's very unlikely in practice + // one will actually insert this many objects into + // the PQ: + // Throw exception to prevent confusing OOME: + throw new System.ArgumentException("maxSize must be < " + ArrayUtil.MAX_ARRAY_LENGTH + "; got: " + maxSize); + } + else + { + // NOTE: we add +1 because all access to heap is + // 1-based not 0-based. heap[0] is unused. + heapSize = maxSize + 1; + } } } + // T is unbounded type, so this unchecked cast works always: T[] h = new T[heapSize]; this.Heap = h; From 5361c00d9d1a74d0b9c07e0b17a8469a6c966591 Mon Sep 17 00:00:00 2001 From: Guido Tagliavini Ponce Date: Thu, 12 Feb 2015 14:19:21 -0800 Subject: [PATCH 2/7] Changed a variable name. Changed and added comments. --- src/Lucene.Net.Core/Util/PriorityQueue.cs | 49 ++++++++++++----------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/src/Lucene.Net.Core/Util/PriorityQueue.cs b/src/Lucene.Net.Core/Util/PriorityQueue.cs index f601de8d6f..3938745284 100644 --- a/src/Lucene.Net.Core/Util/PriorityQueue.cs +++ b/src/Lucene.Net.Core/Util/PriorityQueue.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; namespace Lucene.Net.Util { @@ -21,8 +22,8 @@ namespace Lucene.Net.Util /// /// A PriorityQueue maintains a partial ordering of its elements such that the - /// least element can always be found in constant time. Put()'s and pop()'s - /// require log(size) time. + /// element with least priority can always be found in constant time. It is represented as a + /// Min-Heap so that Add()'s and Pop()'s require log(size) time. /// ///

NOTE: this class will pre-allocate a full array of /// length maxSize+1 if instantiated via the @@ -31,9 +32,10 @@ namespace Lucene.Net.Util /// /// @lucene.internal ///

+ public abstract class PriorityQueue { - private int Size_Renamed = 0; + private int QueueSize = 0; private readonly int MaxSize; private readonly T[] Heap; @@ -41,7 +43,7 @@ public PriorityQueue(int maxSize) : this(maxSize, true) { } - + public PriorityQueue(int maxSize, bool prepopulate) { int heapSize; @@ -96,7 +98,7 @@ public PriorityQueue(int maxSize, bool prepopulate) { Heap[i] = SentinelObject; } - Size_Renamed = maxSize; + QueueSize = maxSize; } } } @@ -157,13 +159,13 @@ protected internal virtual T SentinelObject /// /// Adds an Object to a PriorityQueue in log(size) time. If one tries to add /// more objects than maxSize from initialize an - /// is thrown. + /// is thrown. /// /// the new 'top' element in the queue. public T Add(T element) { - Size_Renamed++; - Heap[Size_Renamed] = element; + QueueSize++; + Heap[QueueSize] = element; UpHeap(); return Heap[1]; } @@ -180,16 +182,16 @@ public T Add(T element) /// public virtual T InsertWithOverflow(T element) { - if (Size_Renamed < MaxSize) + if (QueueSize < MaxSize) { Add(element); return default(T); } - else if (Size_Renamed > 0 && !LessThan(element, Heap[1])) + else if (QueueSize > 0 && !LessThan(element, Heap[1])) { T ret = Heap[1]; Heap[1] = element; - UpdateTop(); + DownHeap(); return ret; } else @@ -199,7 +201,8 @@ public virtual T InsertWithOverflow(T element) } /// - /// Returns the least element of the PriorityQueue in constant time. + /// Returns the least element of the PriorityQueue in constant time. + /// Returns null if the queue is empty. public T Top() { // We don't need to check size here: if maxSize is 0, @@ -214,12 +217,12 @@ public T Top() /// public T Pop() { - if (Size_Renamed > 0) + if (QueueSize > 0) { T result = Heap[1]; // save first value - Heap[1] = Heap[Size_Renamed]; // move last to first - Heap[Size_Renamed] = default(T); // permit GC of objects - Size_Renamed--; + Heap[1] = Heap[QueueSize]; // move last to first + Heap[QueueSize] = default(T); // permit GC of objects + QueueSize--; DownHeap(); // adjust heap return result; } @@ -257,23 +260,23 @@ public T UpdateTop() /// Returns the number of elements currently stored in the PriorityQueue. public int Size() { - return Size_Renamed; + return QueueSize; } /// /// Removes all entries from the PriorityQueue. public void Clear() { - for (int i = 0; i <= Size_Renamed; i++) + for (int i = 0; i <= QueueSize; i++) { Heap[i] = default(T); } - Size_Renamed = 0; + QueueSize = 0; } private void UpHeap() { - int i = Size_Renamed; + int i = QueueSize; T node = Heap[i]; // save bottom node int j = (int)((uint)i >> 1); while (j > 0 && LessThan(node, Heap[j])) @@ -291,17 +294,17 @@ private void DownHeap() T node = Heap[i]; // save top node int j = i << 1; // find smaller child int k = j + 1; - if (k <= Size_Renamed && LessThan(Heap[k], Heap[j])) + if (k <= QueueSize && LessThan(Heap[k], Heap[j])) { j = k; } - while (j <= Size_Renamed && LessThan(Heap[j], node)) + while (j <= QueueSize && LessThan(Heap[j], node)) { Heap[i] = Heap[j]; // shift up child i = j; j = i << 1; k = j + 1; - if (k <= Size_Renamed && LessThan(Heap[k], Heap[j])) + if (k <= QueueSize && LessThan(Heap[k], Heap[j])) { j = k; } From aeb5e0384cf154e97ef16fad522276e501e0818a Mon Sep 17 00:00:00 2001 From: Guido Tagliavini Ponce Date: Thu, 12 Feb 2015 14:20:01 -0800 Subject: [PATCH 3/7] Added PQ tests. --- .../core/Util/TestPriorityQueue.cs | 315 +++++++++++++++++- 1 file changed, 313 insertions(+), 2 deletions(-) diff --git a/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs b/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs index 9237bd5db1..d7e18a485f 100644 --- a/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs +++ b/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs @@ -23,6 +23,8 @@ namespace Lucene.Net.Util [TestFixture] public class TestPriorityQueue : LuceneTestCase { + private static readonly int MAX_PQ_SIZE = ArrayUtil.MAX_ARRAY_LENGTH - 1; + private class IntegerQueue : PriorityQueue { public IntegerQueue(int count) @@ -30,12 +32,309 @@ public IntegerQueue(int count) { } + public IntegerQueue(int count, bool prepopulate) + : base(count, prepopulate) + { + } + public override bool LessThan(int? a, int? b) { return (a < b); } } + private class IntegerQueueWithSentinel : IntegerQueue + { + public IntegerQueueWithSentinel(int count, bool prepopulate) + : base(count, prepopulate) + { + } + + protected override int? SentinelObject + { + get + { + return int.MaxValue; + } + } + } + + [Ignore] // Increase heap size to run this test + [Test] + public static void TestMaxSizeBounds() + { + // Minimum size is 0 + int maxSize = 0; + PriorityQueue pq = new IntegerQueue(maxSize); + + // Maximum size is ArrayUtil.MAX_ARRAY_LENGTH - 1 + maxSize = MAX_PQ_SIZE; + pq = new IntegerQueue(maxSize); + + // A valid maximum size + maxSize = 12345; + pq = new IntegerQueue(maxSize); + + // Cannot construct a negative size heap + maxSize = -3; + try + { + pq = new IntegerQueue(maxSize); + // Should had thrown an exception + Assert.Fail(); + } + catch (ArgumentException) + { + } + + maxSize = MAX_PQ_SIZE; + try + { + pq = new IntegerQueue(maxSize); + } + catch (ArgumentException) + { + } + } + + [Test] + public static void TestPrepopulation() + { + int maxSize = 10; + // Populates the internal array + PriorityQueue pq = new IntegerQueueWithSentinel(maxSize, true); + Assert.AreEqual(pq.Top(), int.MaxValue); + Assert.AreEqual(pq.Size(), 10); + + // Does not populate it + pq = new IntegerQueue(maxSize, false); + Assert.AreEqual(pq.Top(), default(int?)); + Assert.AreEqual(pq.Size(), 0); + } + + [Test] + public static void TestAdd() + { + int maxSize = 10; + PriorityQueue pq = new IntegerQueue(maxSize); + + // Add mixed elements + pq.Add(5); + Assert.AreEqual(pq.Top(), 5); + Assert.AreEqual(pq.Size(), 1); + pq.Add(1); + Assert.AreEqual(pq.Top(), 1); + Assert.AreEqual(pq.Size(), 2); + pq.Add(3); + Assert.AreEqual(pq.Top(), 1); + Assert.AreEqual(pq.Size(), 3); + pq.Add(-1); + Assert.AreEqual(pq.Top(), -1); + Assert.AreEqual(pq.Size(), 4); + pq.Add(-111111); + Assert.AreEqual(pq.Top(), -111111); + Assert.AreEqual(pq.Size(), 5); + + // Add a sorted list of elements + pq = new IntegerQueue(maxSize); + pq.Add(-111111); + Assert.AreEqual(pq.Top(), -111111); + Assert.AreEqual(pq.Size(), 1); + pq.Add(-1); + Assert.AreEqual(pq.Top(), -111111); + Assert.AreEqual(pq.Size(), 2); + pq.Add(1); + Assert.AreEqual(pq.Top(), -111111); + Assert.AreEqual(pq.Size(), 3); + pq.Add(3); + Assert.AreEqual(pq.Top(), -111111); + Assert.AreEqual(pq.Size(), 4); + pq.Add(5); + Assert.AreEqual(pq.Top(), -111111); + Assert.AreEqual(pq.Size(), 5); + + // Add a reversed sorted list of elements + pq = new IntegerQueue(maxSize); + pq.Add(5); + Assert.AreEqual(pq.Top(), 5); + Assert.AreEqual(pq.Size(), 1); + pq.Add(3); + Assert.AreEqual(pq.Top(), 3); + Assert.AreEqual(pq.Size(), 2); + pq.Add(1); + Assert.AreEqual(pq.Top(), 1); + Assert.AreEqual(pq.Size(), 3); + pq.Add(-1); + Assert.AreEqual(pq.Top(), -1); + Assert.AreEqual(pq.Size(), 4); + pq.Add(-111111); + Assert.AreEqual(pq.Top(), -111111); + Assert.AreEqual(pq.Size(), 5); + } + + [Test] + public static void TestDuplicates() + { + // Tests that the queue doesn't absorb elements with duplicate keys + int maxSize = 10; + PriorityQueue pq = new IntegerQueue(maxSize); + + pq.Add(3); + pq.Add(3); + Assert.AreEqual(pq.Size(), 2); + + pq.Add(3); + Assert.AreEqual(pq.Size(), 3); + + pq.Add(17); + pq.Add(17); + pq.Add(17); + pq.Add(17); + Assert.AreEqual(pq.Size(), 7); + } + + [Test] + public static void TestPop() + { + int maxSize = 10; + PriorityQueue pq = new IntegerQueue(maxSize); + + // Add one element and pop it + pq.Add(7); + pq.Pop(); + Assert.AreEqual(pq.Size(), 0); + + // Add a bunch of elements, pop them all + pq.Add(1); + pq.Add(20); + pq.Add(1); + pq.Add(15); + pq.Add(4); + pq.Add(12); + pq.Add(1000); + pq.Add(-3); + pq.Pop(); + Assert.AreEqual(pq.Size(), 7); + pq.Pop(); + pq.Pop(); + pq.Pop(); + pq.Pop(); + pq.Pop(); + pq.Pop(); + pq.Pop(); + Assert.AreEqual(pq.Size(), 0); + + // Interleaved adds and pops + pq.Add(1); + pq.Add(20); + pq.Pop(); + Assert.AreEqual(pq.Size(), 1); + pq.Add(1); + pq.Add(15); + pq.Add(4); + pq.Pop(); + pq.Pop(); + Assert.AreEqual(pq.Size(), 2); + pq.Add(12); + pq.Add(1000); + pq.Add(-3); + pq.Pop(); + pq.Pop(); + Assert.AreEqual(pq.Size(), 3); + + // Pop an empty PQ + pq.Pop(); + pq.Pop(); + pq.Pop(); + pq.Pop(); + pq.Pop(); + pq.Pop(); + } + + [Test] + public static void TestOverflow() + { + // Tests adding elements to full queues + // Add's documentation claims throwing an IndexOutOfRangeException in this situation + + // Add an element to a prepopulated queue + int maxSize = 10; + PriorityQueue pq = new IntegerQueueWithSentinel(maxSize, true); + + try + { + pq.Add(3); + Assert.Fail(); + } + catch (IndexOutOfRangeException) + { + } + + // Populate manually + maxSize = 5; + pq = new IntegerQueue(maxSize); + pq.Add(1); + pq.Add(4); + pq.Add(-1); + pq.Add(0); + pq.Add(10); + + try + { + pq.Add(666); + } + catch (IndexOutOfRangeException) + { + } + } + + [Test] + public static void TestStress() + { + int maxSize = AtLeast(100000); + + PriorityQueue pq = new IntegerQueue(maxSize); + int sum = 0, sum2 = 0; + + DateTime start, end; + TimeSpan total; + start = DateTime.Now; + + // Add a lot of elements + for (int i = 0; i < maxSize; i++) + { + int next = Random().Next(); + sum += next; + pq.Add(next); + } + + end = DateTime.Now; + total = end - start; + // Note that this measurement considers the random number generation + System.Console.WriteLine("Total adding time: {0} ticks or {1}ms", total.Ticks, total.Milliseconds); + System.Console.WriteLine("Time per add: {0} ticks", total.Ticks / maxSize); + + // Pop them and check that the elements are taken in sorted order + start = DateTime.Now; + int last = int.MinValue; + for (int i = 0; i < maxSize; i++) + { + int? next = pq.Pop(); + Assert.IsTrue((int)next >= last); + last = (int)next; + sum2 += last; + } + + end = DateTime.Now; + total = end - start; + // Note that this measurement considers the random number generation + System.Console.WriteLine("Total poping time: {0} ticks or {1}ms", total.Ticks, total.Milliseconds); + System.Console.WriteLine("Time per pop: {0} ticks", total.Ticks / maxSize); + + // Loose checking that we didn't lose data in the process + Assert.AreEqual(sum, sum2); + } + [Test] public virtual void TestPQ() { @@ -90,8 +389,10 @@ public virtual void TestClear() } [Test] - public virtual void TestFixedSize() + public virtual void TestInsertWithOverflowDoesNotOverflow() { + // Tests that InsertWithOverflow does not cause overflow + PriorityQueue pq = new IntegerQueue(3); pq.InsertWithOverflow(2); pq.InsertWithOverflow(3); @@ -104,8 +405,11 @@ public virtual void TestFixedSize() } [Test] - public virtual void TestInsertWithOverflow() + public virtual void TestInsertWithOverflowDiscardsRight() { + // Tests that InsertWithOverflow discards the correct value, + // and the resulting PQ preserves its structure + int size = 4; PriorityQueue pq = new IntegerQueue(size); int? i1 = 2; @@ -123,6 +427,13 @@ public virtual void TestInsertWithOverflow() Assert.IsTrue(pq.InsertWithOverflow(i6) == i6); // i6 should not have been inserted Assert.AreEqual(size, pq.Size()); Assert.AreEqual((int?)2, pq.Top()); + + pq.Pop(); + Assert.AreEqual((int?)3, pq.Top()); + pq.Pop(); + Assert.AreEqual((int?)5, pq.Top()); + pq.Pop(); + Assert.AreEqual((int?)7, pq.Top()); } } } \ No newline at end of file From 84e29b595b638bd5f5ee1022a6207be6676b847b Mon Sep 17 00:00:00 2001 From: Guido Tagliavini Ponce Date: Thu, 12 Feb 2015 14:26:31 -0800 Subject: [PATCH 4/7] Removed old tests. --- .../core/Util/TestPriorityQueue.cs | 41 ------------------- 1 file changed, 41 deletions(-) diff --git a/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs b/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs index d7e18a485f..1209854227 100644 --- a/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs +++ b/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs @@ -335,47 +335,6 @@ public static void TestStress() Assert.AreEqual(sum, sum2); } - [Test] - public virtual void TestPQ() - { - TestPQ(AtLeast(10000), Random()); - } - - public static void TestPQ(int count, Random gen) - { - PriorityQueue pq = new IntegerQueue(count); - int sum = 0, sum2 = 0; - - for (int i = 0; i < count; i++) - { - int next = gen.Next(); - sum += next; - pq.Add(next); - } - - // Date end = new Date(); - - // System.out.print(((float)(end.getTime()-start.getTime()) / count) * 1000); - // System.out.println(" microseconds/put"); - - // start = new Date(); - - int last = int.MinValue; - for (int i = 0; i < count; i++) - { - int? next = pq.Pop(); - Assert.IsTrue((int)next >= last); - last = (int)next; - sum2 += last; - } - - Assert.AreEqual(sum, sum2); - // end = new Date(); - - // System.out.print(((float)(end.getTime()-start.getTime()) / count) * 1000); - // System.out.println(" microseconds/pop"); - } - [Test] public virtual void TestClear() { From 707647d20fc482abcec0713fd47e8f6da39d2c3f Mon Sep 17 00:00:00 2001 From: Guido Tagliavini Ponce Date: Thu, 12 Feb 2015 15:01:58 -0800 Subject: [PATCH 5/7] Added one more test. --- .../core/Util/TestPriorityQueue.cs | 145 ++++++++++++------ 1 file changed, 98 insertions(+), 47 deletions(-) diff --git a/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs b/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs index 1209854227..21d2d5b7a3 100644 --- a/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs +++ b/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs @@ -59,6 +59,28 @@ protected override int? SentinelObject } } + private class MyType + { + public MyType(int field) + { + Field = field; + } + public int Field { get; set; } + } + + private class MyQueue : PriorityQueue + { + public MyQueue(int count) + : base(count) + { + } + + public override bool LessThan(MyType a, MyType b) + { + return (a.Field < b.Field); + } + } + [Ignore] // Increase heap size to run this test [Test] public static void TestMaxSizeBounds() @@ -251,6 +273,35 @@ public static void TestPop() pq.Pop(); } + [Test] + public static void TestUpdateTop() + { + // Mostly to reflect the usage of UpdateTop + int maxSize = 10; + PriorityQueue pq = new MyQueue(maxSize); + + pq.Add(new MyType(1)); + pq.Add(new MyType(20)); + pq.Add(new MyType(1)); + pq.Add(new MyType(15)); + pq.Add(new MyType(4)); + pq.Add(new MyType(12)); + pq.Add(new MyType(1000)); + pq.Add(new MyType(-300)); + + Assert.AreEqual(pq.Top().Field, -300); + MyType topElement = pq.Top(); + topElement.Field = 25; // Now this should no longer be at the top of the queue + pq.UpdateTop(); // Hence we need to update the top queue + Assert.AreEqual(pq.Top().Field, 1); + + // The less eficient way to do this is the following + topElement = pq.Pop(); + topElement.Field = 678; + pq.Add(topElement); + Assert.AreEqual(pq.Top().Field, 1); + } + [Test] public static void TestOverflow() { @@ -288,53 +339,6 @@ public static void TestOverflow() } } - [Test] - public static void TestStress() - { - int maxSize = AtLeast(100000); - - PriorityQueue pq = new IntegerQueue(maxSize); - int sum = 0, sum2 = 0; - - DateTime start, end; - TimeSpan total; - start = DateTime.Now; - - // Add a lot of elements - for (int i = 0; i < maxSize; i++) - { - int next = Random().Next(); - sum += next; - pq.Add(next); - } - - end = DateTime.Now; - total = end - start; - // Note that this measurement considers the random number generation - System.Console.WriteLine("Total adding time: {0} ticks or {1}ms", total.Ticks, total.Milliseconds); - System.Console.WriteLine("Time per add: {0} ticks", total.Ticks / maxSize); - - // Pop them and check that the elements are taken in sorted order - start = DateTime.Now; - int last = int.MinValue; - for (int i = 0; i < maxSize; i++) - { - int? next = pq.Pop(); - Assert.IsTrue((int)next >= last); - last = (int)next; - sum2 += last; - } - - end = DateTime.Now; - total = end - start; - // Note that this measurement considers the random number generation - System.Console.WriteLine("Total poping time: {0} ticks or {1}ms", total.Ticks, total.Milliseconds); - System.Console.WriteLine("Time per pop: {0} ticks", total.Ticks / maxSize); - - // Loose checking that we didn't lose data in the process - Assert.AreEqual(sum, sum2); - } - [Test] public virtual void TestClear() { @@ -394,5 +398,52 @@ public virtual void TestInsertWithOverflowDiscardsRight() pq.Pop(); Assert.AreEqual((int?)7, pq.Top()); } + + [Test] + public static void TestStress() + { + int maxSize = AtLeast(100000); + + PriorityQueue pq = new IntegerQueue(maxSize); + int sum = 0, sum2 = 0; + + DateTime start, end; + TimeSpan total; + start = DateTime.Now; + + // Add a lot of elements + for (int i = 0; i < maxSize; i++) + { + int next = Random().Next(); + sum += next; + pq.Add(next); + } + + end = DateTime.Now; + total = end - start; + // Note that this measurement considers the random number generation + System.Console.WriteLine("Total adding time: {0} ticks or {1}ms", total.Ticks, total.Milliseconds); + System.Console.WriteLine("Time per add: {0} ticks", total.Ticks / maxSize); + + // Pop them and check that the elements are taken in sorted order + start = DateTime.Now; + int last = int.MinValue; + for (int i = 0; i < maxSize; i++) + { + int? next = pq.Pop(); + Assert.IsTrue((int)next >= last); + last = (int)next; + sum2 += last; + } + + end = DateTime.Now; + total = end - start; + // Note that this measurement considers the random number generation + System.Console.WriteLine("Total poping time: {0} ticks or {1}ms", total.Ticks, total.Milliseconds); + System.Console.WriteLine("Time per pop: {0} ticks", total.Ticks / maxSize); + + // Loose checking that we didn't lose data in the process + Assert.AreEqual(sum, sum2); + } } } \ No newline at end of file From 3c85fc8103b7f53052311a7d49cece32da769a1a Mon Sep 17 00:00:00 2001 From: Guido Tagliavini Ponce Date: Fri, 13 Feb 2015 13:45:02 -0800 Subject: [PATCH 6/7] TestStress is now a real stress test. The benchmarks were moved to a special function. Added persistance test. --- .../core/Util/TestPriorityQueue.cs | 281 +++++++++++++----- 1 file changed, 207 insertions(+), 74 deletions(-) diff --git a/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs b/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs index 21d2d5b7a3..845701d54f 100644 --- a/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs +++ b/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs @@ -1,5 +1,8 @@ +using System.Collections.Generic; +using Lucene.Net.JavaCompatibility; using NUnit.Framework; using System; +using NUnit.Framework.Constraints; namespace Lucene.Net.Util { @@ -39,7 +42,7 @@ public IntegerQueue(int count, bool prepopulate) public override bool LessThan(int? a, int? b) { - return (a < b); + return (a <= b); } } @@ -81,6 +84,26 @@ public override bool LessThan(MyType a, MyType b) } } + private class Less : IComparer + { + public int Compare(int? a, int? b) + { + Assert.IsNotNull(a); + Assert.IsNotNull(b); + return (int) (a - b); + } + } + + private class Greater : IComparer + { + public int Compare(int? a, int? b) + { + Assert.IsNotNull(a); + Assert.IsNotNull(b); + return (int) (a - b); + } + } + [Ignore] // Increase heap size to run this test [Test] public static void TestMaxSizeBounds() @@ -134,6 +157,13 @@ public static void TestPrepopulation() Assert.AreEqual(pq.Size(), 0); } + private static void AddAndTest(PriorityQueue pq, T element, T expectedTop, int expectedSize) + { + pq.Add(element); + Assert.AreEqual(pq.Top(), expectedTop); + Assert.AreEqual(pq.Size(), expectedSize); + } + [Test] public static void TestAdd() { @@ -141,57 +171,27 @@ public static void TestAdd() PriorityQueue pq = new IntegerQueue(maxSize); // Add mixed elements - pq.Add(5); - Assert.AreEqual(pq.Top(), 5); - Assert.AreEqual(pq.Size(), 1); - pq.Add(1); - Assert.AreEqual(pq.Top(), 1); - Assert.AreEqual(pq.Size(), 2); - pq.Add(3); - Assert.AreEqual(pq.Top(), 1); - Assert.AreEqual(pq.Size(), 3); - pq.Add(-1); - Assert.AreEqual(pq.Top(), -1); - Assert.AreEqual(pq.Size(), 4); - pq.Add(-111111); - Assert.AreEqual(pq.Top(), -111111); - Assert.AreEqual(pq.Size(), 5); - + AddAndTest(pq, 5, 5, 1); + AddAndTest(pq, 1, 1, 2); + AddAndTest(pq, 3, 1, 3); + AddAndTest(pq, -1, -1, 4); + AddAndTest(pq, -111111, -111111, 5); + // Add a sorted list of elements pq = new IntegerQueue(maxSize); - pq.Add(-111111); - Assert.AreEqual(pq.Top(), -111111); - Assert.AreEqual(pq.Size(), 1); - pq.Add(-1); - Assert.AreEqual(pq.Top(), -111111); - Assert.AreEqual(pq.Size(), 2); - pq.Add(1); - Assert.AreEqual(pq.Top(), -111111); - Assert.AreEqual(pq.Size(), 3); - pq.Add(3); - Assert.AreEqual(pq.Top(), -111111); - Assert.AreEqual(pq.Size(), 4); - pq.Add(5); - Assert.AreEqual(pq.Top(), -111111); - Assert.AreEqual(pq.Size(), 5); + AddAndTest(pq, -111111, -111111, 1); + AddAndTest(pq, -1, -111111, 2); + AddAndTest(pq, 1, -111111, 3); + AddAndTest(pq, 3, -111111, 4); + AddAndTest(pq, 5, -111111, 5); // Add a reversed sorted list of elements pq = new IntegerQueue(maxSize); - pq.Add(5); - Assert.AreEqual(pq.Top(), 5); - Assert.AreEqual(pq.Size(), 1); - pq.Add(3); - Assert.AreEqual(pq.Top(), 3); - Assert.AreEqual(pq.Size(), 2); - pq.Add(1); - Assert.AreEqual(pq.Top(), 1); - Assert.AreEqual(pq.Size(), 3); - pq.Add(-1); - Assert.AreEqual(pq.Top(), -1); - Assert.AreEqual(pq.Size(), 4); - pq.Add(-111111); - Assert.AreEqual(pq.Top(), -111111); - Assert.AreEqual(pq.Size(), 5); + AddAndTest(pq, 5, 5, 1); + AddAndTest(pq, 3, 3, 2); + AddAndTest(pq, 1, 1, 3); + AddAndTest(pq, -1, -1, 4); + AddAndTest(pq, -111111, -111111, 5); } [Test] @@ -399,51 +399,184 @@ public virtual void TestInsertWithOverflowDiscardsRight() Assert.AreEqual((int?)7, pq.Top()); } - [Test] - public static void TestStress() + private static void AddElements(PriorityQueue pq, T[] elements) { - int maxSize = AtLeast(100000); + int size = (int)elements.size(); - PriorityQueue pq = new IntegerQueue(maxSize); - int sum = 0, sum2 = 0; + for (int i = 0; i < size; i++) + { + pq.Add(elements[i]); + } + } + + private static void PopElements(PriorityQueue pq) + { + int size = pq.Size(); + + for (int i = 0; i < size; i++) + { + pq.Pop(); + } + } + + private static void PopAndTestElements(PriorityQueue pq, T[] elements) + { + int size = pq.Size(); + + for (int i = 0; i < size; i++) + { + Assert.AreEqual(pq.Pop(), elements[i]); + } + } + private static void PopAndTestElements(PriorityQueue pq) + { + int size = pq.Size(); + T last = pq.Pop(); + + for (int i = 1; i < size; i++) + { + T next = pq.Pop(); + Assert.IsTrue(pq.LessThan(last, next)); + last = next; + } + } + + private static void TimedAddAndPop(PriorityQueue pq, T[] elements) + { + int size = (int)elements.size(); DateTime start, end; TimeSpan total; + start = DateTime.Now; - // Add a lot of elements - for (int i = 0; i < maxSize; i++) - { - int next = Random().Next(); - sum += next; - pq.Add(next); - } + AddElements(pq, elements); end = DateTime.Now; total = end - start; - // Note that this measurement considers the random number generation + System.Console.WriteLine("Total adding time: {0} ticks or {1}ms", total.Ticks, total.Milliseconds); - System.Console.WriteLine("Time per add: {0} ticks", total.Ticks / maxSize); + System.Console.WriteLine("Average time per add: {0} ticks", total.Ticks / size); - // Pop them and check that the elements are taken in sorted order start = DateTime.Now; - int last = int.MinValue; - for (int i = 0; i < maxSize; i++) - { - int? next = pq.Pop(); - Assert.IsTrue((int)next >= last); - last = (int)next; - sum2 += last; - } + + PopElements(pq); end = DateTime.Now; total = end - start; - // Note that this measurement considers the random number generation + System.Console.WriteLine("Total poping time: {0} ticks or {1}ms", total.Ticks, total.Milliseconds); - System.Console.WriteLine("Time per pop: {0} ticks", total.Ticks / maxSize); + System.Console.WriteLine("Average time per pop: {0} ticks", total.Ticks / size); + } + + [Test] + public static void TestPersistance() + { + // Tests that a big number of elements are added and popped (in the correct order) + // without losing any information + + int maxSize = AtLeast(100000); + PriorityQueue pq = new IntegerQueue(maxSize); + + int?[] elements = new int?[maxSize]; + for (int i = 0; i < maxSize; i++) + { + elements[i] = Random().Next(); + } + + AddElements(pq, elements); + + ArrayUtil.IntroSort(elements, new Less()); + + PopAndTestElements(pq, elements); + } + + [Test, Timeout(0)] + public static void TestStress() + { + int atLeast = 10000000; + int maxSize = AtLeast(atLeast); + int size; + PriorityQueue pq = new IntegerQueue(maxSize); + + // Add a lot of elements + for (int i = 0; i < maxSize; i++) + { + pq.Add(Random().Next()); + } + + // Pop some of them + while (pq.Size() > atLeast/2) + { + pq.Pop(); + } + + // Add some more + while (pq.Size() < (atLeast*3)/4) + { + pq.Add(Random().Next()); + } + + PopAndTestElements(pq); + + Assert.AreEqual(pq.Size(), 0); - // Loose checking that we didn't lose data in the process - Assert.AreEqual(sum, sum2); + // We fill it again + for (int i = 0; 2 * i < maxSize; i++) + { + pq.Add(Random().Next()); + } + + Assert.AreEqual(pq.Size(), (maxSize + 1) / 2); + pq.Clear(); + Assert.AreEqual(pq.Size(), 0); + + // One last time + for (int i = 0; i < 2 * maxSize; i++) + { + pq.Add(Random().Next()); + } + + PopAndTestElements(pq); + Assert.AreEqual(pq.Size(), 0); + } + + [Test] + public static void Benchmarks() + { + if (!VERBOSE) + { + // You won't see the results + return; + } + + int maxSize = AtLeast(100000); + PriorityQueue pq = new IntegerQueue(maxSize); + int?[] elements = new int?[maxSize]; + + for (int i = 0; i < maxSize; i++) + { + elements[i] = Random().Next(); + } + + System.Console.WriteLine("Random list of elements..."); + + TimedAddAndPop(pq, elements); + pq.Clear(); + + System.Console.WriteLine("\nSorted list of elements..."); + + pq = new IntegerQueue(maxSize); + ArrayUtil.IntroSort(elements, new Less()); + TimedAddAndPop(pq, elements); + pq.Clear(); + + System.Console.WriteLine("\nReverse sorted list of elements..."); + + pq = new IntegerQueue(maxSize); + ArrayUtil.IntroSort(elements, new Greater()); + TimedAddAndPop(pq, elements); + pq.Clear(); } } } \ No newline at end of file From 04945497e04e239ee54b30c4988baf1026c880bd Mon Sep 17 00:00:00 2001 From: Guido Tagliavini Ponce Date: Fri, 13 Feb 2015 13:52:16 -0800 Subject: [PATCH 7/7] Typo. --- src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs b/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs index 845701d54f..0c94de5f96 100644 --- a/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs +++ b/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs @@ -465,12 +465,12 @@ private static void TimedAddAndPop(PriorityQueue pq, T[] elements) end = DateTime.Now; total = end - start; - System.Console.WriteLine("Total poping time: {0} ticks or {1}ms", total.Ticks, total.Milliseconds); + System.Console.WriteLine("Total popping time: {0} ticks or {1}ms", total.Ticks, total.Milliseconds); System.Console.WriteLine("Average time per pop: {0} ticks", total.Ticks / size); } [Test] - public static void TestPersistance() + public static void TestPersistence() { // Tests that a big number of elements are added and popped (in the correct order) // without losing any information