From c67366fd4b383c31971a8ae68b458dec7aee2d7f Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Mon, 5 Sep 2016 21:25:35 +0700 Subject: [PATCH 01/47] Setup InternalsVisibleTo Lucene.Net.Core <- Lucene.Net.Tests for testing. --- src/Lucene.Net.Core/AssemblyInfo.cs | 22 ++++++++++++------- .../Index/TestConcurrentMergeScheduler.cs | 4 ++-- .../core/Index/TestFieldsReader.cs | 2 +- .../core/Index/TestIndexReaderClose.cs | 2 +- .../core/Index/TestIndexWriter.cs | 4 ++-- .../core/Index/TestIndexWriterDelete.cs | 2 +- .../core/Index/TestIndexWriterExceptions.cs | 2 +- .../core/Search/JustCompileSearch.cs | 4 ++-- .../Search/Spans/JustCompileSearchSpans.cs | 2 +- .../core/Search/TestBooleanQuery.cs | 2 +- .../core/Search/TestDocIdSet.cs | 4 ++-- .../core/Search/TestFilteredQuery.cs | 2 +- .../core/Search/TestLiveFieldValues.cs | 2 +- .../core/Search/TestMultiTermQueryRewrites.cs | 2 +- .../core/Search/TestPrefixRandom.cs | 2 +- .../core/Search/TestQueryRescorer.cs | 6 ++--- .../core/Search/TestRegexpRandom2.cs | 2 +- .../core/Search/TestTopDocsCollector.cs | 2 +- .../core/Store/TestBufferedIndexInput.cs | 2 +- .../core/Store/TestHugeRamFile.cs | 2 +- .../core/TestMergeSchedulerExternal.cs | 6 ++--- .../core/Util/TestCloseableThreadLocal.cs | 2 +- .../core/Util/TestFilterIterator.cs | 16 +++++++------- .../core/Util/TestPriorityQueue.cs | 2 +- .../core/Util/TestRollingBuffer.cs | 2 +- 25 files changed, 53 insertions(+), 47 deletions(-) diff --git a/src/Lucene.Net.Core/AssemblyInfo.cs b/src/Lucene.Net.Core/AssemblyInfo.cs index dc4e47dc14..7eb9f7ea6b 100644 --- a/src/Lucene.Net.Core/AssemblyInfo.cs +++ b/src/Lucene.Net.Core/AssemblyInfo.cs @@ -39,11 +39,17 @@ [assembly: AssemblyInformationalVersionAttribute("4.8.0")] // for testing -[assembly: InternalsVisibleTo("Lucene.Net.Test, PublicKey=002400000480000094000000060200000024000052534131000400000100010075a07ce602f88e" + - "f263c7db8cb342c58ebd49ecdcc210fac874260b0213fb929ac3dcaf4f5b39744b800f99073eca" + - "72aebfac5f7284e1d5f2c82012a804a140f06d7d043d83e830cdb606a04da2ad5374cc92c0a495" + - "08437802fb4f8fb80a05e59f80afb99f4ccd0dfe44065743543c4b053b669509d29d332cd32a0c" + - "b1e97e84")] +//[assembly: InternalsVisibleTo("Lucene.Net.Test, PublicKey=002400000480000094000000060200000024000052534131000400000100010075a07ce602f88e" + +// "f263c7db8cb342c58ebd49ecdcc210fac874260b0213fb929ac3dcaf4f5b39744b800f99073eca" + +// "72aebfac5f7284e1d5f2c82012a804a140f06d7d043d83e830cdb606a04da2ad5374cc92c0a495" + +// "08437802fb4f8fb80a05e59f80afb99f4ccd0dfe44065743543c4b053b669509d29d332cd32a0c" + +// "b1e97e84")] + +// LUCENENET NOTE: For now it is not possible to use a SNK because we have unmanaged references in Analysis.Common. +// However, we still need InternalsVisibleTo in order to prevent making everything public just for the sake of testing. +// This has broad implications, though because many methods are marked "protected internal", which means other assemblies +// must update overridden methods to match. +[assembly: InternalsVisibleTo("Lucene.Net.Tests")] // // Version information for an assembly consists of the following four values: @@ -84,6 +90,6 @@ // (*) Delay Signing is an advanced option - see the Microsoft .NET Framework // documentation for more information on this. // -[assembly: AssemblyDelaySign(false)] -[assembly: AssemblyKeyFile("")] -[assembly: AssemblyKeyName("")] +//[assembly: AssemblyDelaySign(false)] +//[assembly: AssemblyKeyFile("")] +//[assembly: AssemblyKeyName("")] diff --git a/src/Lucene.Net.Tests/core/Index/TestConcurrentMergeScheduler.cs b/src/Lucene.Net.Tests/core/Index/TestConcurrentMergeScheduler.cs index 14397b9154..888a0e4d87 100644 --- a/src/Lucene.Net.Tests/core/Index/TestConcurrentMergeScheduler.cs +++ b/src/Lucene.Net.Tests/core/Index/TestConcurrentMergeScheduler.cs @@ -351,7 +351,7 @@ public ConcurrentMergeSchedulerAnonymousInnerClassHelper(TestConcurrentMergeSche this.Failed = failed; } - protected override void DoMerge(MergePolicy.OneMerge merge) + protected internal override void DoMerge(MergePolicy.OneMerge merge) { try { @@ -401,7 +401,7 @@ public TrackingCMS() SetMaxMergesAndThreads(5, 5); } - protected override void DoMerge(MergePolicy.OneMerge merge) + protected internal override void DoMerge(MergePolicy.OneMerge merge) { TotMergedBytes += merge.TotalBytesSize(); base.DoMerge(merge); diff --git a/src/Lucene.Net.Tests/core/Index/TestFieldsReader.cs b/src/Lucene.Net.Tests/core/Index/TestFieldsReader.cs index 3aa88421bc..fa54837f6c 100644 --- a/src/Lucene.Net.Tests/core/Index/TestFieldsReader.cs +++ b/src/Lucene.Net.Tests/core/Index/TestFieldsReader.cs @@ -185,7 +185,7 @@ internal virtual void SimOutage() } } - protected override void ReadInternal(byte[] b, int offset, int length) + protected internal override void ReadInternal(byte[] b, int offset, int length) { SimOutage(); @delegate.Seek(FilePointer); diff --git a/src/Lucene.Net.Tests/core/Index/TestIndexReaderClose.cs b/src/Lucene.Net.Tests/core/Index/TestIndexReaderClose.cs index 1d4b7fc195..038dfab72d 100644 --- a/src/Lucene.Net.Tests/core/Index/TestIndexReaderClose.cs +++ b/src/Lucene.Net.Tests/core/Index/TestIndexReaderClose.cs @@ -113,7 +113,7 @@ public FilterAtomicReaderAnonymousInnerClassHelper(TestIndexReaderClose outerIns this.ThrowOnClose = throwOnClose; } - protected override void DoClose() + protected internal override void DoClose() { base.DoClose(); if (ThrowOnClose) diff --git a/src/Lucene.Net.Tests/core/Index/TestIndexWriter.cs b/src/Lucene.Net.Tests/core/Index/TestIndexWriter.cs index 3627ed9e22..64b23483ad 100644 --- a/src/Lucene.Net.Tests/core/Index/TestIndexWriter.cs +++ b/src/Lucene.Net.Tests/core/Index/TestIndexWriter.cs @@ -957,12 +957,12 @@ public MockIndexWriter(Directory dir, IndexWriterConfig conf) internal bool AfterWasCalled; internal bool BeforeWasCalled; - protected override void DoAfterFlush() + protected internal override void DoAfterFlush() { AfterWasCalled = true; } - protected override void DoBeforeFlush() + protected internal override void DoBeforeFlush() { BeforeWasCalled = true; } diff --git a/src/Lucene.Net.Tests/core/Index/TestIndexWriterDelete.cs b/src/Lucene.Net.Tests/core/Index/TestIndexWriterDelete.cs index 4d640f3cde..272c331530 100644 --- a/src/Lucene.Net.Tests/core/Index/TestIndexWriterDelete.cs +++ b/src/Lucene.Net.Tests/core/Index/TestIndexWriterDelete.cs @@ -1368,7 +1368,7 @@ public IndexWriterAnonymousInnerClassHelper(TestIndexWriterDelete outerInstance, this.SawAfterFlush = sawAfterFlush; } - protected override void DoAfterFlush() + protected internal override void DoAfterFlush() { Assert.IsTrue(Closing.Get() || DocsInSegment.Get() >= 7, "only " + DocsInSegment.Get() + " in segment"); DocsInSegment.Set(0); diff --git a/src/Lucene.Net.Tests/core/Index/TestIndexWriterExceptions.cs b/src/Lucene.Net.Tests/core/Index/TestIndexWriterExceptions.cs index 56371db8f1..fdc54896bb 100644 --- a/src/Lucene.Net.Tests/core/Index/TestIndexWriterExceptions.cs +++ b/src/Lucene.Net.Tests/core/Index/TestIndexWriterExceptions.cs @@ -2440,7 +2440,7 @@ public ConcurrentMergeSchedulerAnonymousInnerClassHelper(TestIndexWriterExceptio this.OuterInstance = outerInstance; } - protected override void HandleMergeException(Exception exc) + protected internal override void HandleMergeException(Exception exc) { // suppress only FakeIOException: if (!(exc is MockDirectoryWrapper.FakeIOException)) diff --git a/src/Lucene.Net.Tests/core/Search/JustCompileSearch.cs b/src/Lucene.Net.Tests/core/Search/JustCompileSearch.cs index 8859494293..91604a4184 100644 --- a/src/Lucene.Net.Tests/core/Search/JustCompileSearch.cs +++ b/src/Lucene.Net.Tests/core/Search/JustCompileSearch.cs @@ -201,7 +201,7 @@ public JustCompileFilteredDocIdSet(DocIdSet innerSet) { } - protected override bool Match(int docid) + protected internal override bool Match(int docid) { throw new System.NotSupportedException(UNSUPPORTED_MSG); } @@ -214,7 +214,7 @@ public JustCompileFilteredDocIdSetIterator(DocIdSetIterator innerIter) { } - protected override bool Match(int doc) + protected internal override bool Match(int doc) { throw new System.NotSupportedException(UNSUPPORTED_MSG); } diff --git a/src/Lucene.Net.Tests/core/Search/Spans/JustCompileSearchSpans.cs b/src/Lucene.Net.Tests/core/Search/Spans/JustCompileSearchSpans.cs index 9cc35e7084..9e28b683ac 100644 --- a/src/Lucene.Net.Tests/core/Search/Spans/JustCompileSearchSpans.cs +++ b/src/Lucene.Net.Tests/core/Search/Spans/JustCompileSearchSpans.cs @@ -162,7 +162,7 @@ protected internal JustCompileSpanScorer(Spans spans, Weight weight, Similarity. { } - protected override bool SetFreqCurrentDoc() + protected internal override bool SetFreqCurrentDoc() { throw new System.NotSupportedException(UNSUPPORTED_MSG); } diff --git a/src/Lucene.Net.Tests/core/Search/TestBooleanQuery.cs b/src/Lucene.Net.Tests/core/Search/TestBooleanQuery.cs index 2e3272f6cf..b7e25821ac 100644 --- a/src/Lucene.Net.Tests/core/Search/TestBooleanQuery.cs +++ b/src/Lucene.Net.Tests/core/Search/TestBooleanQuery.cs @@ -396,7 +396,7 @@ public IndexSearcherAnonymousInnerClassHelper(TestBooleanQuery outerInstance, In this.OuterInstance = outerInstance; } - protected override void Search(IList leaves, Weight weight, Collector collector) + protected internal override void Search(IList leaves, Weight weight, Collector collector) { Assert.AreEqual(-1, collector.GetType().Name.IndexOf("OutOfOrder")); base.Search(leaves, weight, collector); diff --git a/src/Lucene.Net.Tests/core/Search/TestDocIdSet.cs b/src/Lucene.Net.Tests/core/Search/TestDocIdSet.cs index e322c180b6..3cf73b98fa 100644 --- a/src/Lucene.Net.Tests/core/Search/TestDocIdSet.cs +++ b/src/Lucene.Net.Tests/core/Search/TestDocIdSet.cs @@ -135,7 +135,7 @@ public FilteredDocIdSetAnonymousInnerClassHelper(TestDocIdSet outerInstance, Doc this.OuterInstance = outerInstance; } - protected override bool Match(int docid) + protected internal override bool Match(int docid) { return docid % 2 == 0; //validate only even docids } @@ -244,7 +244,7 @@ public FilteredDocIdSetAnonymousInnerClassHelper2(FilterAnonymousInnerClassHelpe this.OuterInstance = outerInstance; } - protected override bool Match(int docid) + protected internal override bool Match(int docid) { return true; } diff --git a/src/Lucene.Net.Tests/core/Search/TestFilteredQuery.cs b/src/Lucene.Net.Tests/core/Search/TestFilteredQuery.cs index 29eeac3632..adae39e271 100644 --- a/src/Lucene.Net.Tests/core/Search/TestFilteredQuery.cs +++ b/src/Lucene.Net.Tests/core/Search/TestFilteredQuery.cs @@ -462,7 +462,7 @@ public RandomAccessFilterStrategyAnonymousInnerClassHelper() { } - protected override bool UseRandomAccess(Bits bits, int firstFilterDoc) + protected internal override bool UseRandomAccess(Bits bits, int firstFilterDoc) { return true; } diff --git a/src/Lucene.Net.Tests/core/Search/TestLiveFieldValues.cs b/src/Lucene.Net.Tests/core/Search/TestLiveFieldValues.cs index 4857fe1acd..c19b7cae6d 100644 --- a/src/Lucene.Net.Tests/core/Search/TestLiveFieldValues.cs +++ b/src/Lucene.Net.Tests/core/Search/TestLiveFieldValues.cs @@ -111,7 +111,7 @@ public LiveFieldValuesAnonymousInnerClassHelper(SearcherManager mgr, int missing { } - protected override int? LookupFromSearcher(IndexSearcher s, string id) + protected internal override int? LookupFromSearcher(IndexSearcher s, string id) { TermQuery tq = new TermQuery(new Term("id", id)); TopDocs hits = s.Search(tq, 1); diff --git a/src/Lucene.Net.Tests/core/Search/TestMultiTermQueryRewrites.cs b/src/Lucene.Net.Tests/core/Search/TestMultiTermQueryRewrites.cs index 2449b89858..61ee69bb0c 100644 --- a/src/Lucene.Net.Tests/core/Search/TestMultiTermQueryRewrites.cs +++ b/src/Lucene.Net.Tests/core/Search/TestMultiTermQueryRewrites.cs @@ -226,7 +226,7 @@ public TermRangeTermsEnumAnonymousInnerClassHelper(MultiTermQueryAnonymousInnerC internal readonly IBoostAttribute boostAtt; - protected override AcceptStatus Accept(BytesRef term) + protected internal override AcceptStatus Accept(BytesRef term) { boostAtt.Boost = Convert.ToSingle(term.Utf8ToString()); return base.Accept(term); diff --git a/src/Lucene.Net.Tests/core/Search/TestPrefixRandom.cs b/src/Lucene.Net.Tests/core/Search/TestPrefixRandom.cs index 06bdf3af31..3384aa81ac 100644 --- a/src/Lucene.Net.Tests/core/Search/TestPrefixRandom.cs +++ b/src/Lucene.Net.Tests/core/Search/TestPrefixRandom.cs @@ -117,7 +117,7 @@ internal SimplePrefixTermsEnum(TestPrefixRandom.DumbPrefixQuery outerInstance, T InitialSeekTerm = new BytesRef(""); } - protected override AcceptStatus Accept(BytesRef term) + protected internal override AcceptStatus Accept(BytesRef term) { return StringHelper.StartsWith(term, Prefix) ? AcceptStatus.YES : AcceptStatus.NO; } diff --git a/src/Lucene.Net.Tests/core/Search/TestQueryRescorer.cs b/src/Lucene.Net.Tests/core/Search/TestQueryRescorer.cs index f7dfd3e8a5..ac811cab3e 100644 --- a/src/Lucene.Net.Tests/core/Search/TestQueryRescorer.cs +++ b/src/Lucene.Net.Tests/core/Search/TestQueryRescorer.cs @@ -174,7 +174,7 @@ public QueryRescorerAnonymousInnerClassHelper(TestQueryRescorer outerInstance, P this.OuterInstance = outerInstance; } - protected override float Combine(float firstPassScore, bool secondPassMatches, float secondPassScore) + protected internal override float Combine(float firstPassScore, bool secondPassMatches, float secondPassScore) { float score = firstPassScore; if (secondPassMatches) @@ -262,7 +262,7 @@ public QueryRescorerAnonymousInnerClassHelper2(TestQueryRescorer outerInstance, this.OuterInstance = outerInstance; } - protected override float Combine(float firstPassScore, bool secondPassMatches, float secondPassScore) + protected internal override float Combine(float firstPassScore, bool secondPassMatches, float secondPassScore) { float score = firstPassScore; if (secondPassMatches) @@ -403,7 +403,7 @@ public QueryRescorerAnonymousInnerClassHelper3(TestQueryRescorer outerInstance, this.OuterInstance = outerInstance; } - protected override float Combine(float firstPassScore, bool secondPassMatches, float secondPassScore) + protected internal override float Combine(float firstPassScore, bool secondPassMatches, float secondPassScore) { return secondPassScore; } diff --git a/src/Lucene.Net.Tests/core/Search/TestRegexpRandom2.cs b/src/Lucene.Net.Tests/core/Search/TestRegexpRandom2.cs index 6171c873e6..e09d7878fe 100644 --- a/src/Lucene.Net.Tests/core/Search/TestRegexpRandom2.cs +++ b/src/Lucene.Net.Tests/core/Search/TestRegexpRandom2.cs @@ -150,7 +150,7 @@ internal SimpleAutomatonTermsEnum(TestRegexpRandom2.DumbRegexpQuery outerInstanc InitialSeekTerm = new BytesRef(""); } - protected override AcceptStatus Accept(BytesRef term) + protected internal override AcceptStatus Accept(BytesRef term) { UnicodeUtil.UTF8toUTF16(term.Bytes, term.Offset, term.Length, Utf16); return RunAutomaton.Run(Utf16.Chars, 0, Utf16.Length) ? AcceptStatus.YES : AcceptStatus.NO; diff --git a/src/Lucene.Net.Tests/core/Search/TestTopDocsCollector.cs b/src/Lucene.Net.Tests/core/Search/TestTopDocsCollector.cs index 6addfca38f..555b18a38f 100644 --- a/src/Lucene.Net.Tests/core/Search/TestTopDocsCollector.cs +++ b/src/Lucene.Net.Tests/core/Search/TestTopDocsCollector.cs @@ -39,7 +39,7 @@ public MyTopsDocCollector(int size) { } - protected override TopDocs NewTopDocs(ScoreDoc[] results, int start) + protected internal override TopDocs NewTopDocs(ScoreDoc[] results, int start) { if (results == null) { diff --git a/src/Lucene.Net.Tests/core/Store/TestBufferedIndexInput.cs b/src/Lucene.Net.Tests/core/Store/TestBufferedIndexInput.cs index 86335f09eb..aa0f80a80c 100644 --- a/src/Lucene.Net.Tests/core/Store/TestBufferedIndexInput.cs +++ b/src/Lucene.Net.Tests/core/Store/TestBufferedIndexInput.cs @@ -242,7 +242,7 @@ public MyBufferedIndexInput() // an infinite file } - protected override void ReadInternal(byte[] b, int offset, int length) + protected internal override void ReadInternal(byte[] b, int offset, int length) { for (int i = offset; i < offset + length; i++) { diff --git a/src/Lucene.Net.Tests/core/Store/TestHugeRamFile.cs b/src/Lucene.Net.Tests/core/Store/TestHugeRamFile.cs index 1ee2c48378..c3714fc9af 100644 --- a/src/Lucene.Net.Tests/core/Store/TestHugeRamFile.cs +++ b/src/Lucene.Net.Tests/core/Store/TestHugeRamFile.cs @@ -40,7 +40,7 @@ private class DenseRAMFile : RAMFile private long Capacity = 0; private readonly Dictionary SingleBuffers = new Dictionary(); - protected override byte[] NewBuffer(int size) + protected internal override byte[] NewBuffer(int size) { Capacity += size; if (Capacity <= MAX_VALUE) diff --git a/src/Lucene.Net.Tests/core/TestMergeSchedulerExternal.cs b/src/Lucene.Net.Tests/core/TestMergeSchedulerExternal.cs index 59a7ffd330..7512d5d74b 100644 --- a/src/Lucene.Net.Tests/core/TestMergeSchedulerExternal.cs +++ b/src/Lucene.Net.Tests/core/TestMergeSchedulerExternal.cs @@ -72,7 +72,7 @@ public MyMergeThread(TestMergeSchedulerExternal.MyMergeScheduler outerInstance, } } - protected override MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) + protected internal override MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) { MergeThread thread = new MyMergeThread(this, writer, merge); thread.ThreadPriority = MergeThreadPriority; @@ -81,12 +81,12 @@ protected override MergeThread GetMergeThread(IndexWriter writer, MergePolicy.On return thread; } - protected override void HandleMergeException(Exception t) + protected internal override void HandleMergeException(Exception t) { OuterInstance.ExcCalled = true; } - protected override void DoMerge(MergePolicy.OneMerge merge) + protected internal override void DoMerge(MergePolicy.OneMerge merge) { OuterInstance.MergeCalled = true; base.DoMerge(merge); diff --git a/src/Lucene.Net.Tests/core/Util/TestCloseableThreadLocal.cs b/src/Lucene.Net.Tests/core/Util/TestCloseableThreadLocal.cs index f1091985d1..ca202f92da 100644 --- a/src/Lucene.Net.Tests/core/Util/TestCloseableThreadLocal.cs +++ b/src/Lucene.Net.Tests/core/Util/TestCloseableThreadLocal.cs @@ -60,7 +60,7 @@ public InitValueThreadLocal(TestIDisposableThreadLocal outerInstance) this.OuterInstance = outerInstance; } - protected override object InitialValue() + protected internal override object InitialValue() { return TEST_VALUE; } diff --git a/src/Lucene.Net.Tests/core/Util/TestFilterIterator.cs b/src/Lucene.Net.Tests/core/Util/TestFilterIterator.cs index 5d1616c007..dbc4f9bdc7 100644 --- a/src/Lucene.Net.Tests/core/Util/TestFilterIterator.cs +++ b/src/Lucene.Net.Tests/core/Util/TestFilterIterator.cs @@ -50,7 +50,7 @@ public FilterIteratorAnonymousInnerClassHelper(TestFilterIterator outerInstance, this.OuterInstance = outerInstance; } - protected override bool PredicateFunction(string s) + protected internal override bool PredicateFunction(string s) { return false; } @@ -75,7 +75,7 @@ public FilterIteratorAnonymousInnerClassHelper2(TestFilterIterator outerInstance this.OuterInstance = outerInstance; } - protected override bool PredicateFunction(string s) + protected internal override bool PredicateFunction(string s) { return "a".Equals(s); } @@ -101,7 +101,7 @@ public FilterIteratorAnonymousInnerClassHelper3(TestFilterIterator outerInstance this.OuterInstance = outerInstance; } - protected override bool PredicateFunction(string s) + protected internal override bool PredicateFunction(string s) { return "a".Equals(s); } @@ -126,7 +126,7 @@ public FilterIteratorAnonymousInnerClassHelper4(TestFilterIterator outerInstance this.OuterInstance = outerInstance; } - protected override bool PredicateFunction(string s) + protected internal override bool PredicateFunction(string s) { return "b".Equals(s); } @@ -152,7 +152,7 @@ public FilterIteratorAnonymousInnerClassHelper5(TestFilterIterator outerInstance this.OuterInstance = outerInstance; } - protected override bool PredicateFunction(string s) + protected internal override bool PredicateFunction(string s) { return "b".Equals(s); } @@ -181,7 +181,7 @@ public FilterIteratorAnonymousInnerClassHelper6(TestFilterIterator outerInstance this.OuterInstance = outerInstance; } - protected override bool PredicateFunction(string s) + protected internal override bool PredicateFunction(string s) { return true; } @@ -210,7 +210,7 @@ public FilterIteratorAnonymousInnerClassHelper7(TestFilterIterator outerInstance this.OuterInstance = outerInstance; } - protected override bool PredicateFunction(string s) + protected internal override bool PredicateFunction(string s) { return true; } @@ -243,7 +243,7 @@ public FilterIteratorAnonymousInnerClassHelper8(TestFilterIterator outerInstance this.OuterInstance = outerInstance; } - protected override bool PredicateFunction(string s) + protected internal override bool PredicateFunction(string s) { return true; } diff --git a/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs b/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs index 3868ee3783..59bc78db73 100644 --- a/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs +++ b/src/Lucene.Net.Tests/core/Util/TestPriorityQueue.cs @@ -57,7 +57,7 @@ public IntegerQueueWithSentinel(int count, bool prepopulate) { } - protected override int? SentinelObject + protected internal override int? SentinelObject { get { diff --git a/src/Lucene.Net.Tests/core/Util/TestRollingBuffer.cs b/src/Lucene.Net.Tests/core/Util/TestRollingBuffer.cs index 336b1edfea..e0fac3e5e4 100644 --- a/src/Lucene.Net.Tests/core/Util/TestRollingBuffer.cs +++ b/src/Lucene.Net.Tests/core/Util/TestRollingBuffer.cs @@ -103,7 +103,7 @@ public static Position NewInstanceFunc() return pos; } - protected override Position NewInstance() + protected internal override Position NewInstance() { Position pos = new Position(); pos.Pos = -1; From b56ebc22bb06fddc941b467d8310cfd735e5f9e9 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Tue, 6 Sep 2016 13:17:13 +0700 Subject: [PATCH 02/47] Setup InternalsVisibleTo Lucene.Net.TestFramework <- Lucene.Net.Tests for testing. --- .../Lucene.Net.TestFramework.csproj | 2 +- .../Properties/AssemblyInfo.cs | 13 +++++++++++++ .../core/Analysis/TestLookaheadTokenFilter.cs | 2 +- .../core/Analysis/TrivialLookaheadFilter.cs | 4 ++-- .../Codecs/Lucene40/TestLucene40DocValuesFormat.cs | 2 +- .../Codecs/Lucene42/TestLucene42DocValuesFormat.cs | 2 +- .../Codecs/Perfield/TestPerFieldDocValuesFormat.cs | 2 +- .../core/Index/TestDocValuesFormat.cs | 2 +- .../core/Index/TestLogMergePolicy.cs | 2 +- src/Lucene.Net.Tests/core/Index/TestNRTThreads.cs | 12 ++++++------ .../core/Index/TestTieredMergePolicy.cs | 2 +- .../core/Search/TestSearcherManager.cs | 12 ++++++------ 12 files changed, 35 insertions(+), 22 deletions(-) diff --git a/src/Lucene.Net.TestFramework/Lucene.Net.TestFramework.csproj b/src/Lucene.Net.TestFramework/Lucene.Net.TestFramework.csproj index 8d3d4e5e69..cf0ba78604 100644 --- a/src/Lucene.Net.TestFramework/Lucene.Net.TestFramework.csproj +++ b/src/Lucene.Net.TestFramework/Lucene.Net.TestFramework.csproj @@ -36,7 +36,7 @@ false - true + false diff --git a/src/Lucene.Net.TestFramework/Properties/AssemblyInfo.cs b/src/Lucene.Net.TestFramework/Properties/AssemblyInfo.cs index 467e175a8f..a6ade6d3f5 100644 --- a/src/Lucene.Net.TestFramework/Properties/AssemblyInfo.cs +++ b/src/Lucene.Net.TestFramework/Properties/AssemblyInfo.cs @@ -22,6 +22,19 @@ // The following GUID is for the ID of the typelib if this project is exposed to COM [assembly: Guid("5f36e3cf-82ac-4d97-af1a-6dabe60e9180")] +// for testing +//[assembly: InternalsVisibleTo("Lucene.Net.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010075a07ce602f88e" + +// "f263c7db8cb342c58ebd49ecdcc210fac874260b0213fb929ac3dcaf4f5b39744b800f99073eca" + +// "72aebfac5f7284e1d5f2c82012a804a140f06d7d043d83e830cdb606a04da2ad5374cc92c0a495" + +// "08437802fb4f8fb80a05e59f80afb99f4ccd0dfe44065743543c4b053b669509d29d332cd32a0c" + +// "b1e97e84")] + +// LUCENENET NOTE: For now it is not possible to use a SNK because we have unmanaged references in Analysis.Common. +// However, we still need InternalsVisibleTo in order to prevent making everything public just for the sake of testing. +// This has broad implications, though because many methods are marked "protected internal", which means other assemblies +// must update overridden methods to match. +[assembly: InternalsVisibleTo("Lucene.Net.Tests")] + // Version information for an assembly consists of the following four values: // // Major Version diff --git a/src/Lucene.Net.Tests/core/Analysis/TestLookaheadTokenFilter.cs b/src/Lucene.Net.Tests/core/Analysis/TestLookaheadTokenFilter.cs index f8b9b78c51..44933dc037 100644 --- a/src/Lucene.Net.Tests/core/Analysis/TestLookaheadTokenFilter.cs +++ b/src/Lucene.Net.Tests/core/Analysis/TestLookaheadTokenFilter.cs @@ -58,7 +58,7 @@ public NeverPeeksLookaheadTokenFilter(TokenStream input) { } - protected override LookaheadTokenFilter.Position NewPosition() + protected internal override LookaheadTokenFilter.Position NewPosition() { return new LookaheadTokenFilter.Position(); } diff --git a/src/Lucene.Net.Tests/core/Analysis/TrivialLookaheadFilter.cs b/src/Lucene.Net.Tests/core/Analysis/TrivialLookaheadFilter.cs index e513e570be..a79b17c868 100644 --- a/src/Lucene.Net.Tests/core/Analysis/TrivialLookaheadFilter.cs +++ b/src/Lucene.Net.Tests/core/Analysis/TrivialLookaheadFilter.cs @@ -39,7 +39,7 @@ internal TrivialLookaheadFilter(TokenStream input) OffsetAtt = AddAttribute(); } - protected override TestPosition NewPosition() + protected internal override TestPosition NewPosition() { return new TestPosition(); } @@ -62,7 +62,7 @@ public override void Reset() InsertUpto = -1; } - protected override void AfterPosition() + protected internal override void AfterPosition() { if (InsertUpto < OutputPos) { diff --git a/src/Lucene.Net.Tests/core/Codecs/Lucene40/TestLucene40DocValuesFormat.cs b/src/Lucene.Net.Tests/core/Codecs/Lucene40/TestLucene40DocValuesFormat.cs index 813c638574..9ee39e7cfc 100644 --- a/src/Lucene.Net.Tests/core/Codecs/Lucene40/TestLucene40DocValuesFormat.cs +++ b/src/Lucene.Net.Tests/core/Codecs/Lucene40/TestLucene40DocValuesFormat.cs @@ -46,7 +46,7 @@ protected override Codec Codec } // LUCENE-4583: this codec should throw IAE on huge binary values: - protected override bool CodecAcceptsHugeBinaryValues(string field) + protected internal override bool CodecAcceptsHugeBinaryValues(string field) { return false; } diff --git a/src/Lucene.Net.Tests/core/Codecs/Lucene42/TestLucene42DocValuesFormat.cs b/src/Lucene.Net.Tests/core/Codecs/Lucene42/TestLucene42DocValuesFormat.cs index 4cf9c7356b..7f4438bee8 100644 --- a/src/Lucene.Net.Tests/core/Codecs/Lucene42/TestLucene42DocValuesFormat.cs +++ b/src/Lucene.Net.Tests/core/Codecs/Lucene42/TestLucene42DocValuesFormat.cs @@ -47,7 +47,7 @@ protected override Codec Codec } } - protected override bool CodecAcceptsHugeBinaryValues(string field) + protected internal override bool CodecAcceptsHugeBinaryValues(string field) { return false; } diff --git a/src/Lucene.Net.Tests/core/Codecs/Perfield/TestPerFieldDocValuesFormat.cs b/src/Lucene.Net.Tests/core/Codecs/Perfield/TestPerFieldDocValuesFormat.cs index f2898ab7fd..bf0a5fe57a 100644 --- a/src/Lucene.Net.Tests/core/Codecs/Perfield/TestPerFieldDocValuesFormat.cs +++ b/src/Lucene.Net.Tests/core/Codecs/Perfield/TestPerFieldDocValuesFormat.cs @@ -72,7 +72,7 @@ protected override Codec Codec } } - protected override bool CodecAcceptsHugeBinaryValues(string field) + protected internal override bool CodecAcceptsHugeBinaryValues(string field) { return TestUtil.FieldSupportsHugeBinaryDocValues(field); } diff --git a/src/Lucene.Net.Tests/core/Index/TestDocValuesFormat.cs b/src/Lucene.Net.Tests/core/Index/TestDocValuesFormat.cs index 5db90c9b2a..1efe78149c 100644 --- a/src/Lucene.Net.Tests/core/Index/TestDocValuesFormat.cs +++ b/src/Lucene.Net.Tests/core/Index/TestDocValuesFormat.cs @@ -34,7 +34,7 @@ protected override Codec Codec } } - protected override bool CodecAcceptsHugeBinaryValues(string field) + protected internal override bool CodecAcceptsHugeBinaryValues(string field) { return TestUtil.FieldSupportsHugeBinaryDocValues(field); } diff --git a/src/Lucene.Net.Tests/core/Index/TestLogMergePolicy.cs b/src/Lucene.Net.Tests/core/Index/TestLogMergePolicy.cs index 35e360342c..e99678b6a0 100644 --- a/src/Lucene.Net.Tests/core/Index/TestLogMergePolicy.cs +++ b/src/Lucene.Net.Tests/core/Index/TestLogMergePolicy.cs @@ -19,7 +19,7 @@ namespace Lucene.Net.Index public class TestLogMergePolicy : BaseMergePolicyTestCase { - protected override MergePolicy MergePolicy() + protected internal override MergePolicy MergePolicy() { return NewLogMergePolicy(Random()); } diff --git a/src/Lucene.Net.Tests/core/Index/TestNRTThreads.cs b/src/Lucene.Net.Tests/core/Index/TestNRTThreads.cs index 658d9f37dc..e248d2e7e1 100644 --- a/src/Lucene.Net.Tests/core/Index/TestNRTThreads.cs +++ b/src/Lucene.Net.Tests/core/Index/TestNRTThreads.cs @@ -45,7 +45,7 @@ public virtual void SetUp() UseNonNrtReaders = Random().NextBoolean(); } - protected override void DoSearching(TaskScheduler es, DateTime stopTime) + protected internal override void DoSearching(TaskScheduler es, DateTime stopTime) { bool anyOpenDelFiles = false; @@ -114,7 +114,7 @@ protected override void DoSearching(TaskScheduler es, DateTime stopTime) Assert.IsFalse(anyOpenDelFiles, "saw non-zero open-but-deleted count"); } - protected override Directory GetDirectory(Directory @in) + protected internal override Directory GetDirectory(Directory @in) { Debug.Assert(@in is MockDirectoryWrapper); if (!UseNonNrtReaders) @@ -124,7 +124,7 @@ protected override Directory GetDirectory(Directory @in) return @in; } - protected override void DoAfterWriter(TaskScheduler es) + protected internal override void DoAfterWriter(TaskScheduler es) { // Force writer to do reader pooling, always, so that // all merged segments, even for merges before @@ -134,7 +134,7 @@ protected override void DoAfterWriter(TaskScheduler es) private IndexSearcher FixedSearcher; - protected override IndexSearcher CurrentSearcher + protected internal override IndexSearcher CurrentSearcher { get { @@ -142,7 +142,7 @@ protected override IndexSearcher CurrentSearcher } } - protected override void ReleaseSearcher(IndexSearcher s) + protected internal override void ReleaseSearcher(IndexSearcher s) { if (s != FixedSearcher) { @@ -151,7 +151,7 @@ protected override void ReleaseSearcher(IndexSearcher s) } } - protected override IndexSearcher FinalSearcher + protected internal override IndexSearcher FinalSearcher { get { diff --git a/src/Lucene.Net.Tests/core/Index/TestTieredMergePolicy.cs b/src/Lucene.Net.Tests/core/Index/TestTieredMergePolicy.cs index efeb08b353..5dc9b6c979 100644 --- a/src/Lucene.Net.Tests/core/Index/TestTieredMergePolicy.cs +++ b/src/Lucene.Net.Tests/core/Index/TestTieredMergePolicy.cs @@ -32,7 +32,7 @@ namespace Lucene.Net.Index [TestFixture] public class TestTieredMergePolicy : BaseMergePolicyTestCase { - protected override MergePolicy MergePolicy() + protected internal override MergePolicy MergePolicy() { return NewTieredMergePolicy(); } diff --git a/src/Lucene.Net.Tests/core/Search/TestSearcherManager.cs b/src/Lucene.Net.Tests/core/Search/TestSearcherManager.cs index 240f310f9c..234f09c973 100644 --- a/src/Lucene.Net.Tests/core/Search/TestSearcherManager.cs +++ b/src/Lucene.Net.Tests/core/Search/TestSearcherManager.cs @@ -56,7 +56,7 @@ public virtual void TestSearcherManager_Mem() RunTest("TestSearcherManager"); } - protected override IndexSearcher FinalSearcher + protected internal override IndexSearcher FinalSearcher { get { @@ -74,7 +74,7 @@ protected override IndexSearcher FinalSearcher private readonly IList PastSearchers = new List(); private bool IsNRT; - protected override void DoAfterWriter(TaskScheduler es) + protected internal override void DoAfterWriter(TaskScheduler es) { SearcherFactory factory = new SearcherFactoryAnonymousInnerClassHelper(this, es); if (Random().NextBoolean()) @@ -118,7 +118,7 @@ public override IndexSearcher NewSearcher(IndexReader r) } } - protected override void DoSearching(TaskScheduler es, DateTime stopTime) + protected internal override void DoSearching(TaskScheduler es, DateTime stopTime) { ThreadClass reopenThread = new ThreadAnonymousInnerClassHelper(this, stopTime); reopenThread.SetDaemon(true); @@ -180,7 +180,7 @@ public override void Run() } } - protected override IndexSearcher CurrentSearcher + protected internal override IndexSearcher CurrentSearcher { get { @@ -240,12 +240,12 @@ protected override IndexSearcher CurrentSearcher } } - protected override void ReleaseSearcher(IndexSearcher s) + protected internal override void ReleaseSearcher(IndexSearcher s) { s.IndexReader.DecRef(); } - protected override void DoClose() + protected internal override void DoClose() { Assert.IsTrue(WarmCalled); if (VERBOSE) From 243ada716c47dafeaf8d2aa39c9ebd8f4c0e8bd7 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Mon, 5 Sep 2016 18:21:23 +0700 Subject: [PATCH 03/47] Fixed bugs in FST that were causing test failures and Debug.Assert failures in Misc.Util.Fst. --- src/Lucene.Net.Core/Util/Fst/BytesStore.cs | 18 ++--- src/Lucene.Net.Core/Util/Fst/FST.cs | 18 ++++- src/Lucene.Net.Core/Util/Fst/NodeHash.cs | 80 +++++++++++++++---- .../Util/fst/FSTTester.cs | 24 +++++- 4 files changed, 110 insertions(+), 30 deletions(-) diff --git a/src/Lucene.Net.Core/Util/Fst/BytesStore.cs b/src/Lucene.Net.Core/Util/Fst/BytesStore.cs index de949b8f6c..471cd68276 100644 --- a/src/Lucene.Net.Core/Util/Fst/BytesStore.cs +++ b/src/Lucene.Net.Core/Util/Fst/BytesStore.cs @@ -375,7 +375,7 @@ public virtual void Truncate(long newLen) blockIndex--; NextWrite = BlockSize; } - Blocks.GetRange(blockIndex + 1, Blocks.Count).Clear(); + Blocks.RemoveRange(blockIndex + 1, Blocks.Count - (blockIndex + 1)); if (newLen == 0) { Current = null; @@ -430,7 +430,7 @@ public ForwardBytesReaderAnonymousInner(BytesStore outerInstance) nextRead = outerInstance.BlockSize; } - private sbyte[] Current; + private byte[] Current; private int nextBuffer; private int nextRead; @@ -438,15 +438,15 @@ public override byte ReadByte() { if (nextRead == OuterInstance.BlockSize) { - OuterInstance.Current = OuterInstance.Blocks[nextBuffer++]; + Current = OuterInstance.Blocks[nextBuffer++]; nextRead = 0; } - return OuterInstance.Current[nextRead++]; + return Current[nextRead++]; } public override void SkipBytes(int count) { - Position = OuterInstance.Position + count; + Position = Position + count; } public override void ReadBytes(byte[] b, int offset, int len) @@ -456,7 +456,7 @@ public override void ReadBytes(byte[] b, int offset, int len) int chunkLeft = OuterInstance.BlockSize - nextRead; if (len <= chunkLeft) { - Array.Copy(OuterInstance.Current, nextRead, b, offset, len); + Array.Copy(Current, nextRead, b, offset, len); nextRead += len; break; } @@ -464,11 +464,11 @@ public override void ReadBytes(byte[] b, int offset, int len) { if (chunkLeft > 0) { - Array.Copy(OuterInstance.Current, nextRead, b, offset, chunkLeft); + Array.Copy(Current, nextRead, b, offset, chunkLeft); offset += chunkLeft; len -= chunkLeft; } - OuterInstance.Current = OuterInstance.Blocks[nextBuffer++]; + Current = OuterInstance.Blocks[nextBuffer++]; nextRead = 0; } } @@ -484,7 +484,7 @@ public override long Position { int bufferIndex = (int)(value >> OuterInstance.blockBits); nextBuffer = bufferIndex + 1; - OuterInstance.Current = OuterInstance.Blocks[bufferIndex]; + Current = OuterInstance.Blocks[bufferIndex]; nextRead = (int)(value & OuterInstance.BlockMask); Debug.Assert(this.Position == value, "pos=" + value + " getPos()=" + this.Position); } diff --git a/src/Lucene.Net.Core/Util/Fst/FST.cs b/src/Lucene.Net.Core/Util/Fst/FST.cs index d9dd78c8f2..74beb96c58 100644 --- a/src/Lucene.Net.Core/Util/Fst/FST.cs +++ b/src/Lucene.Net.Core/Util/Fst/FST.cs @@ -6,6 +6,7 @@ namespace Lucene.Net.Util.Fst { using Lucene.Net.Util; + using System.Collections; using System.IO; using ByteArrayDataOutput = Lucene.Net.Store.ByteArrayDataOutput; @@ -439,7 +440,22 @@ private bool AssertRootArcs() Debug.Assert(root.Flags == asserting.Flags); Debug.Assert(root.Label == asserting.Label); Debug.Assert(root.NextArc == asserting.NextArc); - Debug.Assert(root.NextFinalOutput.Equals(asserting.NextFinalOutput)); + // LUCENENET NOTE: In .NET, IEnumerable will not equal another identical IEnumerable + // because it checks for reference equality, not that the list contents + // are the same. + if (root.NextFinalOutput is IEnumerable && asserting.NextFinalOutput is IEnumerable) + { + var iter = (asserting.NextFinalOutput as IEnumerable).GetEnumerator(); + foreach (object value in root.NextFinalOutput as IEnumerable) + { + iter.MoveNext(); + Debug.Assert(object.Equals(value, iter.Current)); + } + } + else + { + Debug.Assert(root.NextFinalOutput.Equals(asserting.NextFinalOutput)); + } Debug.Assert(root.Node == asserting.Node); Debug.Assert(root.NumArcs == asserting.NumArcs); Debug.Assert(root.Output.Equals(asserting.Output)); diff --git a/src/Lucene.Net.Core/Util/Fst/NodeHash.cs b/src/Lucene.Net.Core/Util/Fst/NodeHash.cs index 4e8bc7320f..ad8f2d3e91 100644 --- a/src/Lucene.Net.Core/Util/Fst/NodeHash.cs +++ b/src/Lucene.Net.Core/Util/Fst/NodeHash.cs @@ -2,22 +2,23 @@ namespace Lucene.Net.Util.Fst { + using System.Collections; /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ using PackedInts = Lucene.Net.Util.Packed.PackedInts; using PagedGrowableWriter = Lucene.Net.Util.Packed.PagedGrowableWriter; @@ -87,7 +88,29 @@ private long Hash(Builder.UnCompiledNode node) long n = ((Builder.CompiledNode)arc.Target).Node; h = PRIME * h + (int)(n ^ (n >> 32)); h = PRIME * h + arc.Output.GetHashCode(); - h = PRIME * h + arc.NextFinalOutput.GetHashCode(); + + // LUCENENET: Since lists do not compare values by default in .NET, + // we need this workaround to get the hashcode of the type + all of the + // values. + if (arc.NextFinalOutput is IEnumerable) + { + h = PRIME * h + arc.NextFinalOutput.GetType().GetHashCode(); + foreach (object value in arc.NextFinalOutput as IEnumerable) + { + if (value != null) + { + h = PRIME * h + value.GetHashCode(); + } + else + { + h = PRIME * h + 0; // 0 for null + } + } + } + else + { + h = PRIME * h + arc.NextFinalOutput.GetHashCode(); + } if (arc.IsFinal) { h += 17; @@ -110,7 +133,30 @@ private long Hash(long node) h = PRIME * h + ScratchArc.Label; h = PRIME * h + (int)(ScratchArc.Target ^ (ScratchArc.Target >> 32)); h = PRIME * h + ScratchArc.Output.GetHashCode(); - h = PRIME * h + ScratchArc.NextFinalOutput.GetHashCode(); + + // LUCENENET: Since lists do not compare values by default in .NET, + // we need this workaround to get the hashcode of the type + all of the + // values. + if (ScratchArc.NextFinalOutput is IEnumerable) + { + h = PRIME * h + ScratchArc.NextFinalOutput.GetType().GetHashCode(); + foreach (object value in ScratchArc.NextFinalOutput as IEnumerable) + { + if (value != null) + { + h = PRIME * h + value.GetHashCode(); + } + else + { + h = PRIME * h + 0; // 0 for null + } + } + } + else + { + h = PRIME * h + ScratchArc.NextFinalOutput.GetHashCode(); + } + if (ScratchArc.Final) { h += 17; diff --git a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs index 9a72d60027..d86b695e32 100644 --- a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs +++ b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs @@ -327,11 +327,11 @@ internal virtual FST DoTest(int prune1, int prune2, bool allowRandomSuffixSha foreach (InputOutput pair in Pairs) { - if (pair.Output is IList) + if (pair.Output is IEnumerable) { - IList longValues = (IList)pair.Output; Builder builderObject = builder as Builder; - foreach (long value in longValues) + var values = pair.Output as IEnumerable; + foreach (object value in values) { builderObject.Add(pair.Input, value); } @@ -395,6 +395,24 @@ internal virtual FST DoTest(int prune1, int prune2, bool allowRandomSuffixSha protected internal virtual bool OutputsEqual(T a, T b) { + // LUCENENET: In .NET, lists do not automatically test to ensure + // their values are equal, so we need to do that manually. + // Note that we are testing the values without regard to whether + // the enumerable type is nullable. + if (a is IEnumerable && b is IEnumerable) + { + var iter = (b as IEnumerable).GetEnumerator(); + foreach (object value in a as IEnumerable) + { + iter.MoveNext(); + if (!object.Equals(value, iter.Current)) + { + return false; + } + } + return true; + } + return a.Equals(b); } From 9a562204f973b8380ead3ed89fc3ee527fa904a9 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Mon, 5 Sep 2016 19:21:45 +0700 Subject: [PATCH 04/47] Ported Core.Util.Fst.TestBytesStore --- src/Lucene.Net.Tests/Lucene.Net.Tests.csproj | 1 + .../core/Util/Fst/TestBytesStore.cs | 74 +++++++++---------- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj index 5c148a2629..582a278871 100644 --- a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj +++ b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj @@ -461,6 +461,7 @@ Code + Code diff --git a/src/Lucene.Net.Tests/core/Util/Fst/TestBytesStore.cs b/src/Lucene.Net.Tests/core/Util/Fst/TestBytesStore.cs index 52757e3e6d..ec5b8e6355 100644 --- a/src/Lucene.Net.Tests/core/Util/Fst/TestBytesStore.cs +++ b/src/Lucene.Net.Tests/core/Util/Fst/TestBytesStore.cs @@ -4,7 +4,6 @@ namespace Lucene.Net.Util.Fst { - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -26,12 +25,14 @@ namespace Lucene.Net.Util.Fst using IOContext = Lucene.Net.Store.IOContext; using IndexInput = Lucene.Net.Store.IndexInput; using IndexOutput = Lucene.Net.Store.IndexOutput; + using Support; + using Attributes; [TestFixture] public class TestBytesStore : LuceneTestCase { - [Test] + [Test, LongRunningTest, Timeout(720000)] public virtual void TestRandom() { @@ -39,7 +40,7 @@ public virtual void TestRandom() for (int iter = 0; iter < iters; iter++) { int numBytes = TestUtil.NextInt(Random(), 1, 200000); - sbyte[] expected = new sbyte[numBytes]; + byte[] expected = new byte[numBytes]; int blockBits = TestUtil.NextInt(Random(), 8, 15); BytesStore bytes = new BytesStore(blockBits); if (VERBOSE) @@ -61,7 +62,7 @@ public virtual void TestRandom() case 0: { // write random byte - sbyte b = (sbyte)Random().Next(256); + byte b = (byte)Random().Next(256); if (VERBOSE) { Console.WriteLine(" writeByte b=" + b); @@ -76,7 +77,7 @@ public virtual void TestRandom() { // write random byte[] int len = Random().Next(Math.Min(numBytes - pos, 100)); - sbyte[] temp = new sbyte[len]; + byte[] temp = new byte[len]; Random().NextBytes(temp); if (VERBOSE) { @@ -99,11 +100,11 @@ public virtual void TestRandom() { Console.WriteLine(" abs writeInt pos=" + randomPos + " x=" + x); } - bytes.writeInt(randomPos, x); - expected[randomPos++] = (sbyte)(x >> 24); - expected[randomPos++] = (sbyte)(x >> 16); - expected[randomPos++] = (sbyte)(x >> 8); - expected[randomPos++] = (sbyte)x; + bytes.WriteInt(randomPos, x); + expected[randomPos++] = (byte)(x >> 24); + expected[randomPos++] = (byte)(x >> 16); + expected[randomPos++] = (byte)(x >> 8); + expected[randomPos++] = (byte)x; } } break; @@ -128,11 +129,11 @@ public virtual void TestRandom() { Console.WriteLine(" reverse start=" + start + " end=" + end + " len=" + len + " pos=" + pos); } - bytes.reverse(start, end); + bytes.Reverse(start, end); while (start <= end) { - sbyte b = expected[end]; + byte b = expected[end]; expected[end] = expected[start]; expected[start] = b; start++; @@ -149,7 +150,7 @@ public virtual void TestRandom() { int randomPos = Random().Next(pos - 1); int len = TestUtil.NextInt(Random(), 1, Math.Min(pos - randomPos - 1, 100)); - sbyte[] temp = new sbyte[len]; + byte[] temp = new byte[len]; Random().NextBytes(temp); if (VERBOSE) { @@ -174,7 +175,7 @@ public virtual void TestRandom() Console.WriteLine(" copyBytes src=" + src + " dest=" + dest + " len=" + len); } Array.Copy(expected, src, expected, dest, len); - bytes.copyBytes(src, dest, len); + bytes.CopyBytes(src, dest, len); } } break; @@ -190,13 +191,13 @@ public virtual void TestRandom() } pos += len; - bytes.skipBytes(len); + bytes.SkipBytes(len); // NOTE: must fill in zeros in case truncate was // used, else we get false fails: if (len > 0) { - sbyte[] zeros = new sbyte[len]; + byte[] zeros = new byte[len]; bytes.WriteBytes(pos - len, zeros, 0, len); } } @@ -208,7 +209,7 @@ public virtual void TestRandom() if (pos > 0) { int dest = Random().Next(pos); - sbyte b = (sbyte)Random().Next(256); + byte b = (byte)Random().Next(256); expected[dest] = b; bytes.WriteByte(dest, b); } @@ -222,9 +223,9 @@ public virtual void TestRandom() { // truncate int len = TestUtil.NextInt(Random(), 1, Math.Min(pos, 100)); - bytes.truncate(pos - len); + bytes.Truncate(pos - len); pos -= len; - Arrays.fill(expected, pos, pos + len, (sbyte)0); + Arrays.Fill(expected, pos, pos + len, (byte)0); if (VERBOSE) { Console.WriteLine(" truncate len=" + len + " newPos=" + pos); @@ -247,7 +248,7 @@ public virtual void TestRandom() } Directory dir = NewDirectory(); IndexOutput @out = dir.CreateOutput("bytes", IOContext.DEFAULT); - bytes.writeTo(@out); + bytes.WriteTo(@out); @out.Dispose(); IndexInput @in = dir.OpenInput("bytes", IOContext.DEFAULT); bytesToVerify = new BytesStore(@in, numBytes, TestUtil.NextInt(Random(), 256, int.MaxValue)); @@ -263,7 +264,7 @@ public virtual void TestRandom() } } - private void Verify(BytesStore bytes, sbyte[] expected, int totalLength) + private void Verify(BytesStore bytes, byte[] expected, int totalLength) { Assert.AreEqual(totalLength, bytes.Position); if (totalLength == 0) @@ -276,7 +277,7 @@ private void Verify(BytesStore bytes, sbyte[] expected, int totalLength) } // First verify whole thing in one blast: - sbyte[] actual = new sbyte[totalLength]; + byte[] actual = new byte[totalLength]; if (Random().NextBoolean()) { if (VERBOSE) @@ -284,15 +285,15 @@ private void Verify(BytesStore bytes, sbyte[] expected, int totalLength) Console.WriteLine(" bulk: reversed"); } // reversed - FST.BytesReader r = bytes.ReverseReader; - Assert.IsTrue(r.reversed()); - r.Position = totalLength - 1; - r.ReadBytes(actual, 0, actual.Length); + FST.BytesReader r2 = bytes.ReverseReader; + Assert.IsTrue(r2.Reversed()); + r2.Position = totalLength - 1; + r2.ReadBytes(actual, 0, actual.Length); int start = 0; int end = totalLength - 1; while (start < end) { - sbyte b = actual[start]; + byte b = actual[start]; actual[start] = actual[end]; actual[end] = b; start++; @@ -306,14 +307,14 @@ private void Verify(BytesStore bytes, sbyte[] expected, int totalLength) { Console.WriteLine(" bulk: forward"); } - FST.BytesReader r = bytes.ForwardReader; - Assert.IsFalse(r.reversed()); - r.ReadBytes(actual, 0, actual.Length); + FST.BytesReader r3 = bytes.ForwardReader; + Assert.IsFalse(r3.Reversed()); + r3.ReadBytes(actual, 0, actual.Length); } for (int i = 0; i < totalLength; i++) { - Assert.AreEqual("byte @ index=" + i, expected[i], actual[i]); + assertEquals("byte @ index=" + i, expected[i], actual[i]); } FST.BytesReader r; @@ -357,13 +358,13 @@ private void Verify(BytesStore bytes, sbyte[] expected, int totalLength) { Console.WriteLine(" op iter=" + op + " reversed=" + reversed + " numBytes=" + numBytes + " pos=" + pos); } - sbyte[] temp = new sbyte[numBytes]; + byte[] temp = new byte[numBytes]; r.Position = pos; Assert.AreEqual(pos, r.Position); r.ReadBytes(temp, 0, temp.Length); for (int i = 0; i < numBytes; i++) { - sbyte expectedByte; + byte expectedByte; if (reversed) { expectedByte = expected[pos - i]; @@ -372,7 +373,7 @@ private void Verify(BytesStore bytes, sbyte[] expected, int totalLength) { expectedByte = expected[pos + i]; } - Assert.AreEqual("byte @ index=" + i, expectedByte, temp[i]); + assertEquals("byte @ index=" + i, expectedByte, temp[i]); } int left; @@ -418,12 +419,11 @@ private void Verify(BytesStore bytes, sbyte[] expected, int totalLength) Console.WriteLine(" readInt"); } - r.skipBytes(skipBytes); - Assert.AreEqual(expectedInt, r.readInt()); + r.SkipBytes(skipBytes); + Assert.AreEqual(expectedInt, r.ReadInt()); } } } } } - } \ No newline at end of file From 297f92653a0e8e64c25b171444ca18f0387f190a Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Tue, 6 Sep 2016 14:18:19 +0700 Subject: [PATCH 05/47] .NETify FST: Public member names should be PascalCase. --- src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs | 2 +- src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs | 2 +- src/Lucene.Net.Core/Util/Fst/Util.cs | 6 +++--- src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs b/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs index d2942e2a73..5e082dc5c9 100644 --- a/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs +++ b/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs @@ -848,7 +848,7 @@ private Frame LoadNextFrame(Frame top, Frame frame) private Frame LoadCeilFrame(int label, Frame top, Frame frame) { var arc = frame.arc; - arc = Util.readCeilArc(label, fst, top.arc, arc, fstReader); + arc = Util.ReadCeilArc(label, fst, top.arc, arc, fstReader); if (arc == null) { return null; diff --git a/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs b/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs index 2d17c5ab77..011eed6849 100644 --- a/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs +++ b/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs @@ -762,7 +762,7 @@ internal Frame LoadNextFrame(Frame top, Frame frame) internal Frame LoadCeilFrame(int label, Frame top, Frame frame) { FST.Arc arc = frame.fstArc; - arc = Util.readCeilArc(label, fst, top.fstArc, arc, fstReader); + arc = Util.ReadCeilArc(label, fst, top.fstArc, arc, fstReader); if (arc == null) { return null; diff --git a/src/Lucene.Net.Core/Util/Fst/Util.cs b/src/Lucene.Net.Core/Util/Fst/Util.cs index 87514d9232..00d2507f94 100644 --- a/src/Lucene.Net.Core/Util/Fst/Util.cs +++ b/src/Lucene.Net.Core/Util/Fst/Util.cs @@ -678,7 +678,7 @@ System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() /// Starting from node, find the top N min cost /// completions to a final node. /// - public static TopResults shortestPaths(FST fst, FST.Arc fromNode, T startOutput, IComparer comparator, int topN, bool allowEmptyString) + public static TopResults ShortestPaths(FST fst, FST.Arc fromNode, T startOutput, IComparer comparator, int topN, bool allowEmptyString) { // All paths are kept, so we can pass topN for // maxQueueDepth and the pruning is admissible: @@ -721,7 +721,7 @@ public static TopResults shortestPaths(FST fst, FST.Arc fromNode, /// binary format. Expands the graph considerably. /// /// - public static void toDot(FST fst, TextWriter @out, bool sameRank, bool labelStates) + public static void ToDot(FST fst, TextWriter @out, bool sameRank, bool labelStates) { const string expandedNodeColor = "blue"; @@ -1066,7 +1066,7 @@ public static BytesRef ToBytesRef(IntsRef input, BytesRef scratch) /// the arc to follow reading the label from /// the arc to read into in place /// the fst's - public static FST.Arc readCeilArc(int label, FST fst, FST.Arc follow, FST.Arc arc, FST.BytesReader @in) + public static FST.Arc ReadCeilArc(int label, FST fst, FST.Arc follow, FST.Arc arc, FST.BytesReader @in) { // TODO maybe this is a useful in the FST class - we could simplify some other code like FSTEnum? if (label == FST.END_LABEL) diff --git a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs index d86b695e32..3e05e6de9d 100644 --- a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs +++ b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs @@ -364,7 +364,7 @@ internal virtual FST DoTest(int prune1, int prune2, bool allowRandomSuffixSha if (LuceneTestCase.VERBOSE && Pairs.Count <= 20 && fst != null) { TextWriter w = new StreamWriter(new FileStream("out.dot", FileMode.Open), IOUtils.CHARSET_UTF_8); - Util.toDot(fst, w, false, false); + Util.ToDot(fst, w, false, false); w.Close(); Console.WriteLine("SAVED out.dot"); } From 3da18fb7d9101d42fedc5932fb22afc7254c52a4 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Tue, 6 Sep 2016 18:55:43 +0700 Subject: [PATCH 06/47] Ported Core.Util.Fst.TestFSTs --- src/Lucene.Net.Core/Util/Fst/FST.cs | 6 +- .../Util/fst/FSTTester.cs | 16 +- src/Lucene.Net.Tests/Lucene.Net.Tests.csproj | 1 + .../core/Util/Fst/TestFSTs.cs | 311 +++++++++--------- 4 files changed, 169 insertions(+), 165 deletions(-) diff --git a/src/Lucene.Net.Core/Util/Fst/FST.cs b/src/Lucene.Net.Core/Util/Fst/FST.cs index 74beb96c58..0a21a54c30 100644 --- a/src/Lucene.Net.Core/Util/Fst/FST.cs +++ b/src/Lucene.Net.Core/Util/Fst/FST.cs @@ -902,16 +902,16 @@ public Arc GetFirstArc(Arc arc) { if (emptyOutput != null) { - arc.Flags = (sbyte)(BIT_FINAL_ARC | BIT_LAST_ARC); + arc.Flags = BIT_FINAL_ARC | BIT_LAST_ARC; arc.NextFinalOutput = emptyOutput; if (!emptyOutput.Equals(NO_OUTPUT)) { - arc.Flags |= (sbyte)BIT_ARC_HAS_FINAL_OUTPUT; + arc.Flags |= BIT_ARC_HAS_FINAL_OUTPUT; } } else { - arc.Flags = (sbyte)BIT_LAST_ARC; + arc.Flags = BIT_LAST_ARC; arc.NextFinalOutput = NO_OUTPUT; } arc.Output = NO_OUTPUT; diff --git a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs index 3e05e6de9d..4a38575bea 100644 --- a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs +++ b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs @@ -346,9 +346,10 @@ internal virtual FST DoTest(int prune1, int prune2, bool allowRandomSuffixSha if (Random.NextBoolean() && fst != null && !willRewrite) { IOContext context = LuceneTestCase.NewIOContext(Random); - IndexOutput @out = Dir.CreateOutput("fst.bin", context); - fst.Save(@out); - @out.Dispose(); + using (IndexOutput @out = Dir.CreateOutput("fst.bin", context)) + { + fst.Save(@out); + } IndexInput @in = Dir.OpenInput("fst.bin", context); try { @@ -363,9 +364,10 @@ internal virtual FST DoTest(int prune1, int prune2, bool allowRandomSuffixSha if (LuceneTestCase.VERBOSE && Pairs.Count <= 20 && fst != null) { - TextWriter w = new StreamWriter(new FileStream("out.dot", FileMode.Open), IOUtils.CHARSET_UTF_8); - Util.ToDot(fst, w, false, false); - w.Close(); + using (TextWriter w = new StreamWriter(new FileStream("out.dot", FileMode.OpenOrCreate), IOUtils.CHARSET_UTF_8)) + { + Util.ToDot(fst, w, false, false); + } Console.WriteLine("SAVED out.dot"); } @@ -829,7 +831,7 @@ private void VerifyPruned(int inputMode, FST fst, int prune1, int prune2) for (int idx = 0; idx <= pair.Input.Length; idx++) { scratch.Length = idx; - CountMinOutput cmo = prefixes[scratch]; + CountMinOutput cmo = prefixes.ContainsKey(scratch) ? prefixes[scratch] : null; if (cmo == null) { cmo = new CountMinOutput(); diff --git a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj index 582a278871..f030b0d39d 100644 --- a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj +++ b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj @@ -462,6 +462,7 @@ Code + Code diff --git a/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs b/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs index 553e0715d5..2cabfd2748 100644 --- a/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs +++ b/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs @@ -8,7 +8,6 @@ namespace Lucene.Net.Util.Fst { - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -27,8 +26,8 @@ namespace Lucene.Net.Util.Fst */ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; - using Document = Lucene.Net.Document.Document; - using Field = Lucene.Net.Document.Field; + using Document = Lucene.Net.Documents.Document; + using Field = Lucene.Net.Documents.Field; using DirectoryReader = Lucene.Net.Index.DirectoryReader; using IndexReader = Lucene.Net.Index.IndexReader; using IndexWriter = Lucene.Net.Index.IndexWriter; @@ -54,12 +53,13 @@ namespace Lucene.Net.Util.Fst //using InputOutput = Lucene.Net.Util.Fst.BytesRefFSTEnum.InputOutput; //using Arc = Lucene.Net.Util.Fst.FST.Arc; using BytesReader = Lucene.Net.Util.Fst.FST.BytesReader; - using Pair = Lucene.Net.Util.Fst.PairOutputs.Pair; - using ResultLong = Lucene.Net.Util.Fst.Util.Result; - using ResultPair = Lucene.Net.Util.Fst.Util.Result; + using Pair = Lucene.Net.Util.Fst.PairOutputs.Pair; + //using ResultLong = Lucene.Net.Util.Fst.Util.Result; + //using ResultPair = Lucene.Net.Util.Fst.Util.Result; using PackedInts = Lucene.Net.Util.Packed.PackedInts; using Lucene.Net.Support; using NUnit.Framework; + using System.Globalization; /* import static Lucene.Net.Util.Fst.FSTTester.GetRandomString; @@ -107,11 +107,11 @@ public virtual void TestBasicFSA() for (int idx = 0; idx < strings.Length; idx++) { - terms[idx] = ToIntsRef(strings[idx], inputMode); + terms[idx] = FSTTester.ToIntsRef(strings[idx], inputMode); } for (int idx = 0; idx < strings2.Length; idx++) { - terms2[idx] = ToIntsRef(strings2[idx], inputMode); + terms2[idx] = FSTTester.ToIntsRef(strings2[idx], inputMode); } Array.Sort(terms2); @@ -123,7 +123,7 @@ public virtual void TestBasicFSA() { Outputs outputs = NoOutputs.Singleton; object NO_OUTPUT = outputs.NoOutput; - IList.InputOutput> pairs = new List.InputOutput>(terms2.Length); + List.InputOutput> pairs = new List.InputOutput>(terms2.Length); foreach (IntsRef term in terms2) { pairs.Add(new FSTTester.InputOutput(term, NO_OUTPUT)); @@ -137,12 +137,12 @@ public virtual void TestBasicFSA() // FST ord pos int { PositiveIntOutputs outputs = PositiveIntOutputs.Singleton; - IList.InputOutput> pairs = new List.InputOutput>(terms2.Length); + List.InputOutput> pairs = new List.InputOutput>(terms2.Length); for (int idx = 0; idx < terms2.Length; idx++) { - pairs.Add(new FSTTester.InputOutput(terms2[idx], (long)idx)); + pairs.Add(new FSTTester.InputOutput(terms2[idx], (long?)idx)); } - FST fst = (new FSTTester(Random(), Dir, inputMode, pairs, outputs, true)).DoTest(0, 0, false); + FST fst = (new FSTTester(Random(), Dir, inputMode, pairs, outputs, true)).DoTest(0, 0, false); Assert.IsNotNull(fst); Assert.AreEqual(22, fst.NodeCount); Assert.AreEqual(27, fst.ArcCount); @@ -152,7 +152,7 @@ public virtual void TestBasicFSA() { ByteSequenceOutputs outputs = ByteSequenceOutputs.Singleton; BytesRef NO_OUTPUT = outputs.NoOutput; - IList.InputOutput> pairs = new List.InputOutput>(terms2.Length); + List.InputOutput> pairs = new List.InputOutput>(terms2.Length); for (int idx = 0; idx < terms2.Length; idx++) { BytesRef output = Random().Next(30) == 17 ? NO_OUTPUT : new BytesRef(Convert.ToString(idx)); @@ -175,7 +175,7 @@ private void DoTest(int inputMode, IntsRef[] terms) { Outputs outputs = NoOutputs.Singleton; object NO_OUTPUT = outputs.NoOutput; - IList.InputOutput> pairs = new List.InputOutput>(terms.Length); + List.InputOutput> pairs = new List.InputOutput>(terms.Length); foreach (IntsRef term in terms) { pairs.Add(new FSTTester.InputOutput(term, NO_OUTPUT)); @@ -186,51 +186,51 @@ private void DoTest(int inputMode, IntsRef[] terms) // PositiveIntOutput (ord) { PositiveIntOutputs outputs = PositiveIntOutputs.Singleton; - IList.InputOutput> pairs = new List.InputOutput>(terms.Length); + List.InputOutput> pairs = new List.InputOutput>(terms.Length); for (int idx = 0; idx < terms.Length; idx++) { - pairs.Add(new FSTTester.InputOutput(terms[idx], (long)idx)); + pairs.Add(new FSTTester.InputOutput(terms[idx], (long?)idx)); } - (new FSTTester(Random(), Dir, inputMode, pairs, outputs, true)).DoTest(true); + (new FSTTester(Random(), Dir, inputMode, pairs, outputs, true)).DoTest(true); } // PositiveIntOutput (random monotonically increasing positive number) { PositiveIntOutputs outputs = PositiveIntOutputs.Singleton; - IList.InputOutput> pairs = new List.InputOutput>(terms.Length); + List.InputOutput> pairs = new List.InputOutput>(terms.Length); long lastOutput = 0; for (int idx = 0; idx < terms.Length; idx++) { long value = lastOutput + TestUtil.NextInt(Random(), 1, 1000); lastOutput = value; - pairs.Add(new FSTTester.InputOutput(terms[idx], value)); + pairs.Add(new FSTTester.InputOutput(terms[idx], value)); } - (new FSTTester(Random(), Dir, inputMode, pairs, outputs, true)).DoTest(true); + (new FSTTester(Random(), Dir, inputMode, pairs, outputs, true)).DoTest(true); } // PositiveIntOutput (random positive number) { PositiveIntOutputs outputs = PositiveIntOutputs.Singleton; - IList.InputOutput> pairs = new List.InputOutput>(terms.Length); + List.InputOutput> pairs = new List.InputOutput>(terms.Length); for (int idx = 0; idx < terms.Length; idx++) { - pairs.Add(new FSTTester.InputOutput(terms[idx], TestUtil.NextLong(Random(), 0, long.MaxValue))); + pairs.Add(new FSTTester.InputOutput(terms[idx], TestUtil.NextLong(Random(), 0, long.MaxValue))); } - (new FSTTester(Random(), Dir, inputMode, pairs, outputs, false)).DoTest(true); + (new FSTTester(Random(), Dir, inputMode, pairs, outputs, false)).DoTest(true); } // Pair { PositiveIntOutputs o1 = PositiveIntOutputs.Singleton; PositiveIntOutputs o2 = PositiveIntOutputs.Singleton; - PairOutputs outputs = new PairOutputs(o1, o2); - IList.InputOutput> pairs = new List.InputOutput>(terms.Length); + PairOutputs outputs = new PairOutputs(o1, o2); + List.InputOutput> pairs = new List.InputOutput>(terms.Length); long lastOutput = 0; for (int idx = 0; idx < terms.Length; idx++) { long value = lastOutput + TestUtil.NextInt(Random(), 1, 1000); lastOutput = value; - pairs.Add(new FSTTester.InputOutput(terms[idx], outputs.NewPair((long)idx, value))); + pairs.Add(new FSTTester.InputOutput(terms[idx], outputs.NewPair((long?)idx, value))); } (new FSTTester(Random(), Dir, inputMode, pairs, outputs, false)).DoTest(true); } @@ -239,7 +239,7 @@ private void DoTest(int inputMode, IntsRef[] terms) { ByteSequenceOutputs outputs = ByteSequenceOutputs.Singleton; BytesRef NO_OUTPUT = outputs.NoOutput; - IList.InputOutput> pairs = new List.InputOutput>(terms.Length); + List.InputOutput> pairs = new List.InputOutput>(terms.Length); for (int idx = 0; idx < terms.Length; idx++) { BytesRef output = Random().Next(30) == 17 ? NO_OUTPUT : new BytesRef(Convert.ToString(idx)); @@ -251,7 +251,7 @@ private void DoTest(int inputMode, IntsRef[] terms) // Sequence-of-ints { IntSequenceOutputs outputs = IntSequenceOutputs.Singleton; - IList.InputOutput> pairs = new List.InputOutput>(terms.Length); + List.InputOutput> pairs = new List.InputOutput>(terms.Length); for (int idx = 0; idx < terms.Length; idx++) { string s = Convert.ToString(idx); @@ -304,8 +304,8 @@ private void TestRandomWords(int maxNumWords, int numIter) IntsRef[] terms = new IntsRef[numWords]; while (termsSet.Count < numWords) { - string term = GetRandomString(random); - termsSet.Add(ToIntsRef(term, inputMode)); + string term = FSTTester.GetRandomString(random); + termsSet.Add(FSTTester.ToIntsRef(term, inputMode)); } DoTest(inputMode, termsSet.ToArray(/*new IntsRef[termsSet.Count]*/)); } @@ -347,7 +347,7 @@ public virtual void TestRealTerms() bool doRewrite = Random().NextBoolean(); - Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, int.MaxValue, outputs, null, doRewrite, PackedInts.DEFAULT, true, 15); + Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, int.MaxValue, outputs, null, doRewrite, PackedInts.DEFAULT, true, 15); bool storeOrd = Random().NextBoolean(); if (VERBOSE) @@ -415,7 +415,7 @@ public virtual void TestRealTerms() Console.WriteLine(ord + " terms..."); } } - FST fst = builder.Finish(); + FST fst = builder.Finish(); if (VERBOSE) { Console.WriteLine("FST: " + docCount + " docs; " + ord + " terms; " + fst.NodeCount + " nodes; " + fst.ArcCount + " arcs;" + " " + fst.SizeInBytes() + " bytes"); @@ -426,11 +426,11 @@ public virtual void TestRealTerms() Random random = new Random(Random().Next()); // Now confirm BytesRefFSTEnum and TermsEnum act the // same: - BytesRefFSTEnum fstEnum = new BytesRefFSTEnum(fst); + BytesRefFSTEnum fstEnum = new BytesRefFSTEnum(fst); int num = AtLeast(1000); for (int iter = 0; iter < num; iter++) { - BytesRef randomTerm = new BytesRef(GetRandomString(random)); + BytesRef randomTerm = new BytesRef(FSTTester.GetRandomString(random)); if (VERBOSE) { @@ -438,11 +438,11 @@ public virtual void TestRealTerms() } TermsEnum.SeekStatus seekResult = termsEnum.SeekCeil(randomTerm); - BytesRefFSTEnum.InputOutput fstSeekResult = fstEnum.SeekCeil(randomTerm); + BytesRefFSTEnum.InputOutput fstSeekResult = fstEnum.SeekCeil(randomTerm); if (seekResult == TermsEnum.SeekStatus.END) { - Assert.IsNull(fstSeekResult.Equals(), "got " + (fstSeekResult == null ? "null" : fstSeekResult.Input.Utf8ToString()) + " but expected null"); + Assert.IsNull(fstSeekResult, "got " + (fstSeekResult == null ? "null" : fstSeekResult.Input.Utf8ToString()) + " but expected null"); } else { @@ -472,7 +472,7 @@ public virtual void TestRealTerms() { Console.WriteLine(" end!"); } - BytesRefFSTEnum.InputOutput nextResult = fstEnum.Next(); + BytesRefFSTEnum.InputOutput nextResult = fstEnum.Next(); if (nextResult != null) { Console.WriteLine("expected null but got: input=" + nextResult.Input.Utf8ToString() + " output=" + outputs.OutputToString(nextResult.Output)); @@ -538,7 +538,7 @@ public VisitTerms(string dirOut, string wordsFileIn, int inputMode, int prune, O public virtual void Run(int limit, bool verify, bool verifyByOutput) { - BufferedReader @is = new BufferedReader(new InputStreamReader(new FileInputStream(WordsFileIn), StandardCharsets.UTF_8), 65536); + TextReader @is = new StreamReader(new FileStream(WordsFileIn, FileMode.Open), Encoding.UTF8); try { IntsRef intsRef = new IntsRef(10); @@ -546,18 +546,18 @@ public virtual void Run(int limit, bool verify, bool verifyByOutput) int ord = 0; while (true) { - string w = @is.readLine(); + string w = @is.ReadLine(); if (w == null) { break; } - ToIntsRef(w, InputMode, intsRef); + FSTTester.ToIntsRef(w, InputMode, intsRef); Builder.Add(intsRef, GetOutput(intsRef, ord)); ord++; if (ord % 500000 == 0) { - Console.WriteLine(string.format(Locale.ROOT, "%6.2fs: %9d...", ((DateTime.Now.Millisecond - tStart) / 1000.0), ord)); + Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "{0:000000.000}s: {1:000000000}...", ((DateTime.Now.Millisecond - tStart) / 1000.0), ord)); } if (ord >= limit) { @@ -586,13 +586,13 @@ public virtual void Run(int limit, bool verify, bool verifyByOutput) Console.WriteLine(ord + " terms; " + fst.NodeCount + " nodes; " + fst.ArcCount + " arcs; " + fst.ArcWithOutputCount + " arcs w/ output; tot size " + fst.SizeInBytes()); if (fst.NodeCount < 100) { - Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"), StandardCharsets.UTF_8); - Util.toDot(fst, w, false, false); + TextWriter w = new StreamWriter(new FileStream("out.dot", FileMode.Create), Encoding.UTF8); + Util.ToDot(fst, w, false, false); w.Dispose(); Console.WriteLine("Wrote FST to out.dot"); } - Directory dir = FSDirectory.Open(new File(DirOut)); + Directory dir = FSDirectory.Open(new DirectoryInfo(DirOut)); IndexOutput @out = dir.CreateOutput("fst.bin", IOContext.DEFAULT); fst.Save(@out); @out.Dispose(); @@ -616,18 +616,18 @@ public virtual void Run(int limit, bool verify, bool verifyByOutput) for (int iter = 0; iter < 2; iter++) { @is.Close(); - @is = new BufferedReader(new InputStreamReader(new FileInputStream(WordsFileIn), StandardCharsets.UTF_8), 65536); + @is = new StreamReader(new FileStream(WordsFileIn, FileMode.Open), Encoding.UTF8); ord = 0; tStart = DateTime.Now.Millisecond; while (true) { - string w = @is.readLine(); + string w = @is.ReadLine(); if (w == null) { break; } - ToIntsRef(w, InputMode, intsRef); + FSTTester.ToIntsRef(w, InputMode, intsRef); if (iter == 0) { T expected = GetOutput(intsRef, ord); @@ -644,8 +644,8 @@ public virtual void Run(int limit, bool verify, bool verifyByOutput) else { // Get by output - long output = GetOutput(intsRef, ord); - IntsRef actual = Util.GetByOutput((FST)fst, (long)output); + long? output = GetOutput(intsRef, ord) as long?; + IntsRef actual = Util.GetByOutput(fst as FST, output.GetValueOrDefault()); if (actual == null) { throw new Exception("unexpected null input from output=" + output); @@ -812,12 +812,12 @@ public virtual void Run(int limit, bool verify, bool verifyByOutput) private class VisitTermsAnonymousInnerClassHelper : VisitTerms { - private PairOutputs Outputs; + private PairOutputs outputs; - public VisitTermsAnonymousInnerClassHelper(string dirOut, string wordsFileIn, int inputMode, int prune, PairOutputs outputs, bool doPack, bool noArcArrays) + public VisitTermsAnonymousInnerClassHelper(string dirOut, string wordsFileIn, int inputMode, int prune, PairOutputs outputs, bool doPack, bool noArcArrays) : base(dirOut, wordsFileIn, inputMode, prune, outputs, doPack, noArcArrays) { - this.Outputs = outputs; + this.outputs = outputs; } internal Random rand; @@ -827,24 +827,24 @@ protected internal override Pair GetOutput(IntsRef input, int ord) { rand = new Random(17); } - return Outputs.NewPair((long)ord, (long)TestUtil.NextInt(rand, 1, 5000)); + return outputs.NewPair(ord, TestUtil.NextInt(rand, 1, 5000)); } } - private class VisitTermsAnonymousInnerClassHelper2 : VisitTerms + private class VisitTermsAnonymousInnerClassHelper2 : VisitTerms { public VisitTermsAnonymousInnerClassHelper2(string dirOut, string wordsFileIn, int inputMode, int prune, PositiveIntOutputs outputs, bool doPack, bool noArcArrays) : base(dirOut, wordsFileIn, inputMode, prune, outputs, doPack, noArcArrays) { } - protected internal override long GetOutput(IntsRef input, int ord) + protected internal override long? GetOutput(IntsRef input, int ord) { - return (long)ord; + return ord; } } - private class VisitTermsAnonymousInnerClassHelper3 : VisitTerms + private class VisitTermsAnonymousInnerClassHelper3 : VisitTerms { public VisitTermsAnonymousInnerClassHelper3(string dirOut, string wordsFileIn, int inputMode, int prune, PositiveIntOutputs outputs, bool doPack, bool noArcArrays) : base(dirOut, wordsFileIn, inputMode, prune, outputs, doPack, noArcArrays) @@ -852,7 +852,7 @@ public VisitTermsAnonymousInnerClassHelper3(string dirOut, string wordsFileIn, i } internal Random rand; - protected internal override long GetOutput(IntsRef input, int ord) + protected internal override long? GetOutput(IntsRef input, int ord) { if (ord == 0) { @@ -968,7 +968,7 @@ public virtual void TestSimple() PositiveIntOutputs outputs = PositiveIntOutputs.Singleton; // Build an FST mapping BytesRef -> Long - Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, outputs); + Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, outputs); BytesRef a = new BytesRef("a"); BytesRef b = new BytesRef("b"); @@ -978,28 +978,28 @@ public virtual void TestSimple() builder.Add(Util.ToIntsRef(b, new IntsRef()), 42L); builder.Add(Util.ToIntsRef(c, new IntsRef()), 13824324872317238L); - FST fst = builder.Finish(); + FST fst = builder.Finish(); - Assert.AreEqual(13824324872317238L, (long)Util.Get(fst, c)); - Assert.AreEqual(42, (long)Util.Get(fst, b)); - Assert.AreEqual(17, (long)Util.Get(fst, a)); + Assert.AreEqual(13824324872317238L, Util.Get(fst, c)); + Assert.AreEqual(42, Util.Get(fst, b)); + Assert.AreEqual(17, Util.Get(fst, a)); - BytesRefFSTEnum fstEnum = new BytesRefFSTEnum(fst); - BytesRefFSTEnum.InputOutput seekResult; + BytesRefFSTEnum fstEnum = new BytesRefFSTEnum(fst); + BytesRefFSTEnum.InputOutput seekResult; seekResult = fstEnum.SeekFloor(a); Assert.IsNotNull(seekResult); - Assert.AreEqual(17, (long)seekResult.Output); + Assert.AreEqual(17, seekResult.Output); // goes to a seekResult = fstEnum.SeekFloor(new BytesRef("aa")); Assert.IsNotNull(seekResult); - Assert.AreEqual(17, (long)seekResult.Output); + Assert.AreEqual(17, seekResult.Output); // goes to b seekResult = fstEnum.SeekCeil(new BytesRef("aa")); Assert.IsNotNull(seekResult); Assert.AreEqual(b, seekResult.Input); - Assert.AreEqual(42, (long)seekResult.Output); + Assert.AreEqual(42, seekResult.Output); Assert.AreEqual(Util.ToIntsRef(new BytesRef("c"), new IntsRef()), Util.GetByOutput(fst, 13824324872317238L)); Assert.IsNull(Util.GetByOutput(fst, 47)); @@ -1036,7 +1036,7 @@ public virtual void TestPrimaryKeys() if (cycle == 0) { // PKs are assigned sequentially - idString = string.Format(Locale.ROOT, "%07d", id); + idString = string.Format(CultureInfo.InvariantCulture, "{0:0000000}", id); } else { @@ -1063,7 +1063,7 @@ public virtual void TestPrimaryKeys() w.Dispose(); IList allIDsList = new List(allIDs); - IList sortedAllIDsList = new List(allIDsList); + List sortedAllIDsList = new List(allIDsList); sortedAllIDsList.Sort(); // Sprinkle in some non-existent PKs: @@ -1073,7 +1073,7 @@ public virtual void TestPrimaryKeys() string idString; if (cycle == 0) { - idString = string.Format(Locale.ROOT, "%07d", (NUM_IDS + idx)); + idString = string.Format(CultureInfo.InvariantCulture, "{0:0000000}", (NUM_IDS + idx)); } else { @@ -1127,8 +1127,8 @@ public virtual void TestPrimaryKeys() int idv = Random().Next(NUM_IDS - 1); if (cycle == 0) { - id = string.Format(Locale.ROOT, "%07da", idv); - nextID = string.Format(Locale.ROOT, "%07d", idv + 1); + id = string.Format(CultureInfo.InvariantCulture, "{0:0000000}a", idv); + nextID = string.Format(CultureInfo.InvariantCulture, "{0:0000000}", idv + 1); } else { @@ -1183,7 +1183,8 @@ public virtual void TestRandomTermLookup() { Directory dir = NewDirectory(); - RandomIndexWriter w = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(IndexWriterConfig.OpenMode.CREATE)); + RandomIndexWriter w = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())) + .SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE)); Document doc = new Document(); Field f = NewStringField("field", "", Field.Store.NO); doc.Add(f); @@ -1197,7 +1198,7 @@ public virtual void TestRandomTermLookup() HashSet allTerms = new HashSet(); while (allTerms.Count < NUM_TERMS) { - allTerms.Add(SimpleRandomString(Random())); + allTerms.Add(FSTTester.SimpleRandomString(Random())); } foreach (string term in allTerms) @@ -1255,7 +1256,7 @@ public virtual void TestExpandedCloseToRoot() StringBuilder b = new StringBuilder(); s.Generate(@out, b, 'a', 'i', 10); string[] input = @out.ToArray(); - Arrays.sort(input); + Array.Sort(input); FST fst = s.Compile(input); FST.Arc arc = fst.GetFirstArc(new FST.Arc()); s.VerifyStateAndBelow(fst, arc, 1); @@ -1310,10 +1311,9 @@ public int VerifyStateAndBelow(FST fst, FST.Arc arc, int depth) if (FST.TargetHasArcs(arc)) { int childCount = 0; - BytesReader fstReader = fst.GetBytesReader; - for (arc = fst.ReadFirstTargetArc(arc, arc, fstReader); - (arc = fst.ReadNextArc(arc, fstReader)) != null; - childCount++) + BytesReader fstReader = fst.BytesReader; + for (arc = fst.ReadFirstTargetArc(arc, arc, fstReader); ; + arc = fst.ReadNextArc(arc, fstReader), childCount++) { bool expanded = fst.IsExpandedTarget(arc, fstReader); int children = VerifyStateAndBelow(fst, new FST.Arc().CopyFrom(arc), depth + 1); @@ -1335,13 +1335,13 @@ public virtual void TestFinalOutputOnEndState() { PositiveIntOutputs outputs = PositiveIntOutputs.Singleton; - Builder builder = new Builder(FST.INPUT_TYPE.BYTE4, 2, 0, true, true, int.MaxValue, outputs, null, Random().NextBoolean(), PackedInts.DEFAULT, true, 15); + Builder builder = new Builder(FST.INPUT_TYPE.BYTE4, 2, 0, true, true, int.MaxValue, outputs, null, Random().NextBoolean(), PackedInts.DEFAULT, true, 15); builder.Add(Util.ToUTF32("stat", new IntsRef()), 17L); builder.Add(Util.ToUTF32("station", new IntsRef()), 10L); - FST fst = builder.Finish(); + FST fst = builder.Finish(); //Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp3/out.dot")); StringWriter w = new StringWriter(); - Util.toDot(fst, w, false, false); + Util.ToDot(fst, w, false, false); w.Dispose(); //System.out.println(w.toString()); Assert.IsTrue(w.ToString().IndexOf("label=\"t/[7]\"") != -1); @@ -1352,13 +1352,13 @@ public virtual void TestInternalFinalState() { PositiveIntOutputs outputs = PositiveIntOutputs.Singleton; bool willRewrite = Random().NextBoolean(); - Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, int.MaxValue, outputs, null, willRewrite, PackedInts.DEFAULT, true, 15); + Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, int.MaxValue, outputs, null, willRewrite, PackedInts.DEFAULT, true, 15); builder.Add(Util.ToIntsRef(new BytesRef("stat"), new IntsRef()), outputs.NoOutput); builder.Add(Util.ToIntsRef(new BytesRef("station"), new IntsRef()), outputs.NoOutput); - FST fst = builder.Finish(); + FST fst = builder.Finish(); StringWriter w = new StringWriter(); //Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp/out.dot")); - Util.toDot(fst, w, false, false); + Util.ToDot(fst, w, false, false); w.Dispose(); //System.out.println(w.toString()); @@ -1374,19 +1374,19 @@ public virtual void TestInternalFinalState() public virtual void TestNonFinalStopNode() { PositiveIntOutputs outputs = PositiveIntOutputs.Singleton; - long nothing = outputs.NoOutput; - Builder b = new Builder(FST.INPUT_TYPE.BYTE1, outputs); + long? nothing = outputs.NoOutput; + Builder b = new Builder(FST.INPUT_TYPE.BYTE1, outputs); - FST fst = new FST(FST.INPUT_TYPE.BYTE1, outputs, false, PackedInts.COMPACT, true, 15); + FST fst = new FST(FST.INPUT_TYPE.BYTE1, outputs, false, PackedInts.COMPACT, true, 15); - Builder.UnCompiledNode rootNode = new Builder.UnCompiledNode(b, 0); + Builder.UnCompiledNode rootNode = new Builder.UnCompiledNode(b, 0); // Add final stop node { - Builder.UnCompiledNode node = new Builder.UnCompiledNode(b, 0); + Builder.UnCompiledNode node = new Builder.UnCompiledNode(b, 0); node.IsFinal = true; rootNode.AddArc('a', node); - Builder.CompiledNode frozen = new Builder.CompiledNode(); + Builder.CompiledNode frozen = new Builder.CompiledNode(); frozen.Node = fst.AddNode(node); rootNode.Arcs[0].NextFinalOutput = 17L; rootNode.Arcs[0].IsFinal = true; @@ -1396,9 +1396,9 @@ public virtual void TestNonFinalStopNode() // Add non-final stop node { - Builder.UnCompiledNode node = new Builder.UnCompiledNode(b, 0); + Builder.UnCompiledNode node = new Builder.UnCompiledNode(b, 0); rootNode.AddArc('b', node); - Builder.CompiledNode frozen = new Builder.CompiledNode(); + Builder.CompiledNode frozen = new Builder.CompiledNode(); frozen.Node = fst.AddNode(node); rootNode.Arcs[1].NextFinalOutput = nothing; rootNode.Arcs[1].Output = 42L; @@ -1409,7 +1409,7 @@ public virtual void TestNonFinalStopNode() StringWriter w = new StringWriter(); //Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp3/out.dot")); - Util.toDot(fst, w, false, false); + Util.ToDot(fst, w, false, false); w.Dispose(); CheckStopNodes(fst, outputs); @@ -1421,41 +1421,41 @@ public virtual void TestNonFinalStopNode() @out.Dispose(); IndexInput @in = dir.OpenInput("fst", IOContext.DEFAULT); - FST fst2 = new FST(@in, outputs); + FST fst2 = new FST(@in, outputs); CheckStopNodes(fst2, outputs); @in.Dispose(); dir.Dispose(); } - private void CheckStopNodes(FST fst, PositiveIntOutputs outputs) + private void CheckStopNodes(FST fst, PositiveIntOutputs outputs) { long? nothing = outputs.NoOutput; - FST.Arc startArc = fst.GetFirstArc(new FST.Arc()); + FST.Arc startArc = fst.GetFirstArc(new FST.Arc()); Assert.AreEqual(nothing, startArc.Output); Assert.AreEqual(nothing, startArc.NextFinalOutput); - FST.Arc arc = fst.ReadFirstTargetArc(startArc, new FST.Arc(), fst.GetBytesReader); + FST.Arc arc = fst.ReadFirstTargetArc(startArc, new FST.Arc(), fst.BytesReader); Assert.AreEqual('a', arc.Label); - Assert.AreEqual(17, (long)arc.NextFinalOutput); + Assert.AreEqual(17, arc.NextFinalOutput); Assert.IsTrue(arc.Final); - arc = fst.ReadNextArc(arc, fst.GetBytesReader); + arc = fst.ReadNextArc(arc, fst.BytesReader); Assert.AreEqual('b', arc.Label); Assert.IsFalse(arc.Final); - Assert.AreEqual(42, (long)arc.Output); + Assert.AreEqual(42, arc.Output); } - internal static readonly IComparer minLongComparator = new ComparatorAnonymousInnerClassHelper(); + internal static readonly IComparer minLongComparator = new ComparatorAnonymousInnerClassHelper(); - private class ComparatorAnonymousInnerClassHelper : IComparer + private class ComparatorAnonymousInnerClassHelper : IComparer { public ComparatorAnonymousInnerClassHelper() { } - public virtual int Compare(long left, long right) + public virtual int Compare(long? left, long? right) { - return left.CompareTo(right); + return left.Value.CompareTo(right.Value); } } @@ -1463,35 +1463,35 @@ public virtual int Compare(long left, long right) public virtual void TestShortestPaths() { PositiveIntOutputs outputs = PositiveIntOutputs.Singleton; - Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, outputs); + Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, outputs); IntsRef scratch = new IntsRef(); builder.Add(Util.ToIntsRef(new BytesRef("aab"), scratch), 22L); builder.Add(Util.ToIntsRef(new BytesRef("aac"), scratch), 7L); builder.Add(Util.ToIntsRef(new BytesRef("ax"), scratch), 17L); - FST fst = builder.Finish(); + FST fst = builder.Finish(); //Writer w = new OutputStreamWriter(new FileOutputStream("out.dot")); //Util.toDot(fst, w, false, false); //w.Dispose(); - Util.TopResults res = Util.shortestPaths(fst, fst.GetFirstArc(new FST.Arc()), outputs.NoOutput, minLongComparator, 3, true); + Util.TopResults res = Util.ShortestPaths(fst, fst.GetFirstArc(new FST.Arc()), outputs.NoOutput, minLongComparator, 3, true); Assert.IsTrue(res.IsComplete); Assert.AreEqual(3, res.TopN.Count); Assert.AreEqual(Util.ToIntsRef(new BytesRef("aac"), scratch), res.TopN[0].Input); - Assert.AreEqual(7L, (long)res.TopN[0].Output); + Assert.AreEqual(7L, res.TopN[0].Output); Assert.AreEqual(Util.ToIntsRef(new BytesRef("ax"), scratch), res.TopN[1].Input); - Assert.AreEqual(17L, (long)res.TopN[1].Output); + Assert.AreEqual(17L, res.TopN[1].Output); Assert.AreEqual(Util.ToIntsRef(new BytesRef("aab"), scratch), res.TopN[2].Input); - Assert.AreEqual(22L, (long)res.TopN[2].Output); + Assert.AreEqual(22L, res.TopN[2].Output); } [Test] public virtual void TestRejectNoLimits() { PositiveIntOutputs outputs = PositiveIntOutputs.Singleton; - Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, outputs); + Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, outputs); IntsRef scratch = new IntsRef(); builder.Add(Util.ToIntsRef(new BytesRef("aab"), scratch), 22L); @@ -1500,41 +1500,41 @@ public virtual void TestRejectNoLimits() builder.Add(Util.ToIntsRef(new BytesRef("adcde"), scratch), 17L); builder.Add(Util.ToIntsRef(new BytesRef("ax"), scratch), 17L); - FST fst = builder.Finish(); + FST fst = builder.Finish(); AtomicInteger rejectCount = new AtomicInteger(); - Util.TopNSearcher searcher = new TopNSearcherAnonymousInnerClassHelper(this, fst, minLongComparator, rejectCount); + Util.TopNSearcher searcher = new TopNSearcherAnonymousInnerClassHelper(this, fst, minLongComparator, rejectCount); - searcher.AddStartPaths(fst.GetFirstArc(new FST.Arc()), outputs.NoOutput, true, new IntsRef()); - Util.TopResults res = searcher.Search(); + searcher.AddStartPaths(fst.GetFirstArc(new FST.Arc()), outputs.NoOutput, true, new IntsRef()); + Util.TopResults res = searcher.Search(); Assert.AreEqual(rejectCount.Get(), 4); Assert.IsTrue(res.IsComplete); // rejected(4) + topN(2) <= maxQueueSize(6) Assert.AreEqual(1, res.TopN.Count); Assert.AreEqual(Util.ToIntsRef(new BytesRef("aac"), scratch), res.TopN[0].Input); - Assert.AreEqual(7L, (long)res.TopN[0].Output); + Assert.AreEqual(7L, res.TopN[0].Output); rejectCount.Set(0); searcher = new TopNSearcherAnonymousInnerClassHelper2(this, fst, minLongComparator, rejectCount); - searcher.AddStartPaths(fst.GetFirstArc(new FST.Arc()), outputs.NoOutput, true, new IntsRef()); + searcher.AddStartPaths(fst.GetFirstArc(new FST.Arc()), outputs.NoOutput, true, new IntsRef()); res = searcher.Search(); Assert.AreEqual(rejectCount.Get(), 4); Assert.IsFalse(res.IsComplete); // rejected(4) + topN(2) > maxQueueSize(5) } - private class TopNSearcherAnonymousInnerClassHelper : Util.TopNSearcher + private class TopNSearcherAnonymousInnerClassHelper : Util.TopNSearcher { private readonly TestFSTs OuterInstance; private AtomicInteger RejectCount; - public TopNSearcherAnonymousInnerClassHelper(TestFSTs outerInstance, FST fst, IComparer minLongComparator, AtomicInteger rejectCount) + public TopNSearcherAnonymousInnerClassHelper(TestFSTs outerInstance, FST fst, IComparer minLongComparator, AtomicInteger rejectCount) : base(fst, 2, 6, minLongComparator) { this.OuterInstance = outerInstance; this.RejectCount = rejectCount; } - protected override bool AcceptResult(IntsRef input, long output) + protected override bool AcceptResult(IntsRef input, long? output) { bool accept = (int)output == 7; if (!accept) @@ -1545,20 +1545,20 @@ protected override bool AcceptResult(IntsRef input, long output) } } - private class TopNSearcherAnonymousInnerClassHelper2 : Util.TopNSearcher + private class TopNSearcherAnonymousInnerClassHelper2 : Util.TopNSearcher { private readonly TestFSTs OuterInstance; private AtomicInteger RejectCount; - public TopNSearcherAnonymousInnerClassHelper2(TestFSTs outerInstance, FST fst, IComparer minLongComparator, AtomicInteger rejectCount) + public TopNSearcherAnonymousInnerClassHelper2(TestFSTs outerInstance, FST fst, IComparer minLongComparator, AtomicInteger rejectCount) : base(fst, 2, 5, minLongComparator) { this.OuterInstance = outerInstance; this.RejectCount = rejectCount; } - protected override bool AcceptResult(IntsRef input, long output) + protected override bool AcceptResult(IntsRef input, long? output) { bool accept = (int)output == 7; if (!accept) @@ -1580,7 +1580,7 @@ public ComparatorAnonymousInnerClassHelper2() public virtual int Compare(Pair left, Pair right) { - return left.Output1.CompareTo(right.Output1); + return left.Output1.GetValueOrDefault().CompareTo(right.Output1.GetValueOrDefault()); } } @@ -1590,7 +1590,7 @@ public virtual int Compare(Pair left, Pair right) public virtual void TestShortestPathsWFST() { - PairOutputs outputs = new PairOutputs(PositiveIntOutputs.Singleton, PositiveIntOutputs.Singleton); // output - weight + PairOutputs outputs = new PairOutputs(PositiveIntOutputs.Singleton, PositiveIntOutputs.Singleton); // output - weight Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, outputs); @@ -1603,21 +1603,21 @@ public virtual void TestShortestPathsWFST() //Util.toDot(fst, w, false, false); //w.Dispose(); - Util.TopResults res = Util.shortestPaths(fst, fst.GetFirstArc(new FST.Arc()), outputs.NoOutput, minPairWeightComparator, 3, true); + Util.TopResults res = Util.ShortestPaths(fst, fst.GetFirstArc(new FST.Arc()), outputs.NoOutput, minPairWeightComparator, 3, true); Assert.IsTrue(res.IsComplete); Assert.AreEqual(3, res.TopN.Count); Assert.AreEqual(Util.ToIntsRef(new BytesRef("aac"), scratch), res.TopN[0].Input); - Assert.AreEqual(7L, (long)res.TopN[0].Output.Output1); // weight - Assert.AreEqual(36L, (long)res.TopN[0].Output.Output2); // output + Assert.AreEqual(7L, res.TopN[0].Output.Output1); // weight + Assert.AreEqual(36L, res.TopN[0].Output.Output2); // output Assert.AreEqual(Util.ToIntsRef(new BytesRef("ax"), scratch), res.TopN[1].Input); - Assert.AreEqual(17L, (long)res.TopN[1].Output.Output1); // weight - Assert.AreEqual(85L, (long)res.TopN[1].Output.Output2); // output + Assert.AreEqual(17L, res.TopN[1].Output.Output1); // weight + Assert.AreEqual(85L, res.TopN[1].Output.Output2); // output Assert.AreEqual(Util.ToIntsRef(new BytesRef("aab"), scratch), res.TopN[2].Input); - Assert.AreEqual(22L, (long)res.TopN[2].Output.Output1); // weight - Assert.AreEqual(57L, (long)res.TopN[2].Output.Output2); // output + Assert.AreEqual(22L, res.TopN[2].Output.Output1); // weight + Assert.AreEqual(57L, res.TopN[2].Output.Output2); // output } [Test] @@ -1630,7 +1630,7 @@ public virtual void TestShortestPathsRandom() SortedSet allPrefixes = new SortedSet(); PositiveIntOutputs outputs = PositiveIntOutputs.Singleton; - Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, outputs); + Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, outputs); IntsRef scratch = new IntsRef(); for (int i = 0; i < numWords; i++) @@ -1659,13 +1659,13 @@ public virtual void TestShortestPathsRandom() builder.Add(Util.ToIntsRef(new BytesRef(e.Key), scratch), e.Value); } - FST fst = builder.Finish(); + FST fst = builder.Finish(); //System.out.println("SAVE out.dot"); //Writer w = new OutputStreamWriter(new FileOutputStream("out.dot")); //Util.toDot(fst, w, false, false); //w.Dispose(); - BytesReader reader = fst.GetBytesReader; + BytesReader reader = fst.BytesReader; //System.out.println("testing: " + allPrefixes.Size() + " prefixes"); foreach (string prefix in allPrefixes) @@ -1673,8 +1673,8 @@ public virtual void TestShortestPathsRandom() // 1. run prefix against fst, then complete by value //System.out.println("TEST: " + prefix); - long prefixOutput = 0; - FST.Arc arc = fst.GetFirstArc(new FST.Arc()); + long? prefixOutput = 0; + FST.Arc arc = fst.GetFirstArc(new FST.Arc()); for (int idx = 0; idx < prefix.Length; idx++) { if (fst.FindTargetArc((int)prefix[idx], arc, arc, reader) == null) @@ -1686,11 +1686,11 @@ public virtual void TestShortestPathsRandom() int topN = TestUtil.NextInt(random, 1, 10); - Util.TopResults r = Util.shortestPaths(fst, arc, fst.Outputs.NoOutput, minLongComparator, topN, true); + Util.TopResults r = Util.ShortestPaths(fst, arc, fst.Outputs.NoOutput, minLongComparator, topN, true); Assert.IsTrue(r.IsComplete); // 2. go thru whole treemap (slowCompletor) and check its actually the best suggestion - IList matches = new List(); + List> matches = new List>(); // TODO: could be faster... but its slowCompletor for a reason foreach (KeyValuePair e in slowCompletor) @@ -1698,12 +1698,12 @@ public virtual void TestShortestPathsRandom() if (e.Key.StartsWith(prefix)) { //System.out.println(" consider " + e.getKey()); - matches.Add(new ResultLong(Util.ToIntsRef(new BytesRef(e.Key.Substring(prefix.Length)), new IntsRef()), e.Value - prefixOutput)); + matches.Add(new Util.Result(Util.ToIntsRef(new BytesRef(e.Key.Substring(prefix.Length)), new IntsRef()), e.Value - prefixOutput)); } } Assert.IsTrue(matches.Count > 0); - matches.Sort(new TieBreakByInputComparator(minLongComparator)); + matches.Sort(new TieBreakByInputComparator(minLongComparator)); if (matches.Count > topN) { matches.SubList(topN, matches.Count).Clear(); @@ -1720,15 +1720,15 @@ public virtual void TestShortestPathsRandom() } } - private class TieBreakByInputComparator : IComparer + private class TieBreakByInputComparator : IComparer> { - internal readonly IComparer Comparator; - public TieBreakByInputComparator(IComparer comparator) + internal readonly IComparer Comparator; + public TieBreakByInputComparator(IComparer comparator) { this.Comparator = comparator; } - public virtual int Compare(ResultLong a, ResultLong b) + public virtual int Compare(Util.Result a, Util.Result b) { int cmp = Comparator.Compare(a.Output, b.Output); if (cmp == 0) @@ -1768,7 +1768,7 @@ public virtual void TestShortestPathsWFSTRandom() SortedDictionary slowCompletor = new SortedDictionary(); SortedSet allPrefixes = new SortedSet(); - PairOutputs outputs = new PairOutputs(PositiveIntOutputs.Singleton, PositiveIntOutputs.Singleton); // output - weight + PairOutputs outputs = new PairOutputs(PositiveIntOutputs.Singleton, PositiveIntOutputs.Singleton); // output - weight Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, outputs); IntsRef scratch = new IntsRef(); @@ -1808,7 +1808,7 @@ public virtual void TestShortestPathsWFSTRandom() //Util.toDot(fst, w, false, false); //w.Dispose(); - BytesReader reader = fst.GetBytesReader; + BytesReader reader = fst.BytesReader; //System.out.println("testing: " + allPrefixes.Size() + " prefixes"); foreach (string prefix in allPrefixes) @@ -1829,10 +1829,10 @@ public virtual void TestShortestPathsWFSTRandom() int topN = TestUtil.NextInt(random, 1, 10); - Util.TopResults r = Util.shortestPaths(fst, arc, fst.Outputs.NoOutput, minPairWeightComparator, topN, true); + Util.TopResults r = Util.ShortestPaths(fst, arc, fst.Outputs.NoOutput, minPairWeightComparator, topN, true); Assert.IsTrue(r.IsComplete); // 2. go thru whole treemap (slowCompletor) and check its actually the best suggestion - IList matches = new List(); + List> matches = new List>(); // TODO: could be faster... but its slowCompletor for a reason foreach (KeyValuePair e in slowCompletor) @@ -1840,12 +1840,13 @@ public virtual void TestShortestPathsWFSTRandom() if (e.Key.StartsWith(prefix)) { //System.out.println(" consider " + e.getKey()); - matches.Add(new ResultPair(Util.ToIntsRef(new BytesRef(e.Key.Substring(prefix.Length)), new IntsRef()), outputs.NewPair(e.Value.a - prefixOutput.Output1, e.Value.b - prefixOutput.Output2))); + matches.Add(new Util.Result(Util.ToIntsRef(new BytesRef(e.Key.Substring(prefix.Length)), new IntsRef()), + outputs.NewPair(e.Value.a - prefixOutput.Output1, e.Value.b - prefixOutput.Output2))); } } Assert.IsTrue(matches.Count > 0); - matches.Sort(new TieBreakByInputComparator(minPairWeightComparator)); + matches.Sort(new TieBreakByInputComparator(minPairWeightComparator)); if (matches.Count > topN) { matches.SubList(topN, matches.Count).Clear(); @@ -1868,7 +1869,7 @@ public virtual void TestLargeOutputsOnArrayArcs() ByteSequenceOutputs outputs = ByteSequenceOutputs.Singleton; Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, outputs); - sbyte[] bytes = new sbyte[300]; + byte[] bytes = new byte[300]; IntsRef input = new IntsRef(); input.Grow(1); input.Length = 1; @@ -1876,7 +1877,7 @@ public virtual void TestLargeOutputsOnArrayArcs() for (int arc = 0; arc < 6; arc++) { input.Ints[0] = arc; - output.Bytes[0] = (sbyte)arc; + output.Bytes[0] = (byte)arc; builder.Add(input, BytesRef.DeepCopyOf(output)); } From 5effb005955f357825cb9447f186b221a8d63cb7 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Wed, 7 Sep 2016 02:01:26 +0700 Subject: [PATCH 07/47] Ported Core.Util.Packed.TestPackedInts --- src/Lucene.Net.Tests/Lucene.Net.Tests.csproj | 1 + .../core/Util/Packed/TestPackedInts.cs | 66 ++++++++++--------- 2 files changed, 35 insertions(+), 32 deletions(-) diff --git a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj index f030b0d39d..955d5bc6a8 100644 --- a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj +++ b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj @@ -469,6 +469,7 @@ Code + Code diff --git a/src/Lucene.Net.Tests/core/Util/Packed/TestPackedInts.cs b/src/Lucene.Net.Tests/core/Util/Packed/TestPackedInts.cs index 0807aa5976..3294f829a7 100644 --- a/src/Lucene.Net.Tests/core/Util/Packed/TestPackedInts.cs +++ b/src/Lucene.Net.Tests/core/Util/Packed/TestPackedInts.cs @@ -8,7 +8,6 @@ namespace Lucene.Net.Util.Packed { - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -26,7 +25,6 @@ namespace Lucene.Net.Util.Packed * limitations under the License. */ - using CodecUtil = Lucene.Net.Codecs.CodecUtil; using ByteArrayDataInput = Lucene.Net.Store.ByteArrayDataInput; using DataInput = Lucene.Net.Store.DataInput; @@ -984,6 +982,7 @@ public virtual void TestSave() } [Test] + [Ignore("LUCENENET TODO: LongBuffer and the method AsLongBuffer are not yet ported, so we cannot currently run this test without porting or finding an alternative way to compare the data.")] public virtual void TestEncodeDecode() { foreach (PackedInts.Format format in PackedInts.Format.Values()) @@ -1060,35 +1059,38 @@ public virtual void TestEncodeDecode() Assert.AreEqual(blocks2, blocks3, msg); } - // 4. byte[] decoding - sbyte[] byteBlocks = new sbyte[8 * blocks.Length]; - ByteBuffer.Wrap((byte[])(Array)byteBlocks).AsLongBuffer().Put(blocks); - long[] values2 = new long[valuesOffset + longIterations * longValueCount]; - decoder.Decode(byteBlocks, blocksOffset * 8, values2, valuesOffset, byteIterations); - foreach (long value in values2) - { - Assert.IsTrue(value <= PackedInts.MaxValue(bpv), msg); - } - Assert.AreEqual(values, values2, msg); - // test decoding to int[] - if (bpv <= 32) - { - int[] intValues2 = new int[values2.Length]; - decoder.Decode(byteBlocks, blocksOffset * 8, intValues2, valuesOffset, byteIterations); - Assert.IsTrue(Equals(intValues2, values2), msg); - } - - // 5. byte[] encoding - sbyte[] blocks3_ = new sbyte[8 * (blocksOffset2 + blocksLen)]; - encoder.Encode(values, valuesOffset, blocks3_, 8 * blocksOffset2, byteIterations); - Assert.AreEqual(msg, LongBuffer.Wrap(blocks2), ByteBuffer.Wrap((byte[])(Array)blocks3_).AsLongBuffer()); - // test encoding from int[] - if (bpv <= 32) - { - sbyte[] blocks4 = new sbyte[blocks3_.Length]; - encoder.Encode(intValues, valuesOffset, blocks4, 8 * blocksOffset2, byteIterations); - Assert.AreEqual(blocks3_, blocks4, msg); - } + // LUCENENET TODO: LongBuffer and the method AsLongBuffer are not yet ported, so we + // cannot currently run the following tests. + + //// 4. byte[] decoding + //byte[] byteBlocks = new byte[8 * blocks.Length]; + //ByteBuffer.Wrap(byteBlocks).AsLongBuffer().Put(blocks); + //long[] values2 = new long[valuesOffset + longIterations * longValueCount]; + //decoder.Decode(byteBlocks, blocksOffset * 8, values2, valuesOffset, byteIterations); + //foreach (long value in values2) + //{ + // Assert.IsTrue(value <= PackedInts.MaxValue(bpv), msg); + //} + //Assert.AreEqual(values, values2, msg); + //// test decoding to int[] + //if (bpv <= 32) + //{ + // int[] intValues2 = new int[values2.Length]; + // decoder.Decode(byteBlocks, blocksOffset * 8, intValues2, valuesOffset, byteIterations); + // Assert.IsTrue(Equals(intValues2, values2), msg); + //} + + //// 5. byte[] encoding + //byte[] blocks3_ = new byte[8 * (blocksOffset2 + blocksLen)]; + //encoder.Encode(values, valuesOffset, blocks3_, 8 * blocksOffset2, byteIterations); + //assertEquals(msg, LongBuffer.Wrap(blocks2), ByteBuffer.Wrap(blocks3_).AsLongBuffer()); + //// test encoding from int[] + //if (bpv <= 32) + //{ + // byte[] blocks4 = new byte[blocks3_.Length]; + // encoder.Encode(intValues, valuesOffset, blocks4, 8 * blocksOffset2, byteIterations); + // Assert.AreEqual(blocks3_, blocks4, msg); + //} } } } @@ -1337,7 +1339,7 @@ public virtual void TestBlockPackedReaderWriter() @out.Dispose(); IndexInput in1 = dir.OpenInput("out.bin", IOContext.DEFAULT); - sbyte[] buf = new sbyte[(int)fp]; + byte[] buf = new byte[(int)fp]; in1.ReadBytes(buf, 0, (int)fp); in1.Seek(0L); ByteArrayDataInput in2 = new ByteArrayDataInput((byte[])(Array)buf); From 510413ce4b99d15a79a70d876e5f89ca657dc193 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Wed, 7 Sep 2016 16:18:17 +0700 Subject: [PATCH 08/47] .NETify FST: Changed Final, Last, Compiled, and Reversed() to IsFinal, IsLast, IsCompiled, and IsReversed to make it clear that they are boolean states. --- .../Analysis/CharFilter/MappingCharFilter.cs | 4 +-- .../Analysis/CharFilter/NormalizeCharMap.cs | 2 +- .../Analysis/Hunspell/Dictionary.cs | 2 +- .../Miscellaneous/StemmerOverrideFilter.cs | 2 +- .../Analysis/Synonym/SynonymFilter.cs | 2 +- .../Memory/FSTOrdTermsReader.cs | 8 ++--- .../Memory/FSTTermsReader.cs | 10 +++---- .../Codecs/BlockTreeTermsReader.cs | 28 ++++++++--------- src/Lucene.Net.Core/Util/Fst/Builder.cs | 8 ++--- src/Lucene.Net.Core/Util/Fst/BytesStore.cs | 8 ++--- src/Lucene.Net.Core/Util/Fst/FST.cs | 30 +++++++++---------- src/Lucene.Net.Core/Util/Fst/FSTEnum.cs | 18 +++++------ .../Util/Fst/ForwardBytesReader.cs | 4 +-- src/Lucene.Net.Core/Util/Fst/NodeHash.cs | 8 ++--- .../Util/Fst/ReverseBytesReader.cs | 4 +-- src/Lucene.Net.Core/Util/Fst/Util.cs | 24 +++++++-------- .../Util/fst/FSTTester.cs | 2 +- .../core/Util/Fst/TestBytesStore.cs | 4 +-- .../core/Util/Fst/TestFSTs.cs | 6 ++-- 19 files changed, 87 insertions(+), 87 deletions(-) diff --git a/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/MappingCharFilter.cs b/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/MappingCharFilter.cs index f14a5e9b54..3fc3309599 100644 --- a/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/MappingCharFilter.cs +++ b/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/MappingCharFilter.cs @@ -136,7 +136,7 @@ public override int Read() if (!FST.TargetHasArcs(arc)) { // Fast pass for single character match: - Debug.Assert(arc.Final); + Debug.Assert(arc.IsFinal); lastMatchLen = 1; lastMatch = arc.Output; } @@ -148,7 +148,7 @@ public override int Read() { lookahead++; - if (arc.Final) + if (arc.IsFinal) { // Match! (to node is final) lastMatchLen = lookahead; diff --git a/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/NormalizeCharMap.cs b/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/NormalizeCharMap.cs index 6d0ed19267..9571a31c95 100644 --- a/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/NormalizeCharMap.cs +++ b/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/NormalizeCharMap.cs @@ -56,7 +56,7 @@ private NormalizeCharMap(FST map) { Debug.Assert(scratchArc.Label != FST.END_LABEL); // LUCENENET TODO END_LABEL shouldn't be under generic? cachedRootArcs[Convert.ToChar((char)scratchArc.Label)] = (new FST.Arc()).CopyFrom(scratchArc); - if (scratchArc.Last) + if (scratchArc.IsLast) { break; } diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Hunspell/Dictionary.cs b/src/Lucene.Net.Analysis.Common/Analysis/Hunspell/Dictionary.cs index e9fc1246d5..e27b43314b 100644 --- a/src/Lucene.Net.Analysis.Common/Analysis/Hunspell/Dictionary.cs +++ b/src/Lucene.Net.Analysis.Common/Analysis/Hunspell/Dictionary.cs @@ -1190,7 +1190,7 @@ internal static void ApplyMappings(FST fst, StringBuilder sb) { output = fst.Outputs.Add(output, arc.Output); } - if (arc.Final) + if (arc.IsFinal) { longestOutput = fst.Outputs.Add(output, arc.NextFinalOutput); longestMatch = j; diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/StemmerOverrideFilter.cs b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/StemmerOverrideFilter.cs index 0db4da9aac..d087551c7a 100644 --- a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/StemmerOverrideFilter.cs +++ b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/StemmerOverrideFilter.cs @@ -144,7 +144,7 @@ public BytesRef Get(char[] buffer, int bufferLen, FST.Arc scratchArc, pendingOutput = fst.Outputs.Add(pendingOutput, scratchArc.Output); bufUpto += Character.CharCount(codePoint); } - if (scratchArc.Final) + if (scratchArc.IsFinal) { matchOutput = fst.Outputs.Add(pendingOutput, scratchArc.NextFinalOutput); } diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Synonym/SynonymFilter.cs b/src/Lucene.Net.Analysis.Common/Analysis/Synonym/SynonymFilter.cs index d2fb3c8a3b..161725914c 100644 --- a/src/Lucene.Net.Analysis.Common/Analysis/Synonym/SynonymFilter.cs +++ b/src/Lucene.Net.Analysis.Common/Analysis/Synonym/SynonymFilter.cs @@ -437,7 +437,7 @@ private void Parse() // OK, entire token matched; now see if this is a final // state: - if (scratchArc.Final) + if (scratchArc.IsFinal) { matchOutput = fst.Outputs.Add(pendingOutput, scratchArc.NextFinalOutput); matchInputLength = tokenCount; diff --git a/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs b/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs index 5e082dc5c9..7f3f35b3f3 100644 --- a/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs +++ b/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs @@ -824,7 +824,7 @@ private Frame LoadNextFrame(Frame top, Frame frame) { return null; } - while (!frame.arc.Last) + while (!frame.arc.IsLast) { frame.arc = fst.ReadNextRealArc(frame.arc, fstReader); frame.state = fsa.Step(top.state, frame.arc.Label); @@ -864,7 +864,7 @@ private Frame LoadCeilFrame(int label, Frame top, Frame frame) private bool IsAccept(Frame frame) // reach a term both fst&fsa accepts { - return fsa.IsAccept(frame.state) && frame.arc.Final; + return fsa.IsAccept(frame.state) && frame.arc.IsFinal; } private bool IsValid(Frame frame) // reach a prefix both fst&fsa won't reject @@ -879,7 +879,7 @@ private bool CanGrow(Frame frame) // can walk forward on both fst&fsa private bool CanRewind(Frame frame) // can jump to sibling { - return !frame.arc.Last; + return !frame.arc.IsLast; } private void PushFrame(Frame frame) @@ -974,7 +974,7 @@ internal static void Walk(FST fst) while (true) { queue.Add((new FST.Arc()).CopyFrom(arc)); - if (arc.Last) + if (arc.IsLast) { break; } diff --git a/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs b/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs index 011eed6849..79e0365685 100644 --- a/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs +++ b/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs @@ -582,7 +582,7 @@ internal void LoadMetaData() next.Output = fstOutputs.Add(next.Output, last.Output); last = next; } - if (last.Final) + if (last.IsFinal) { meta = fstOutputs.Add(last.Output, last.NextFinalOutput); } @@ -738,7 +738,7 @@ internal Frame LoadNextFrame(Frame top, Frame frame) { return null; } - while (!frame.fstArc.Last) + while (!frame.fstArc.IsLast) { frame.fstArc = fst.ReadNextRealArc(frame.fstArc, fstReader); frame.fsaState = fsa.Step(top.fsaState, frame.fstArc.Label); @@ -778,7 +778,7 @@ internal Frame LoadCeilFrame(int label, Frame top, Frame frame) internal bool IsAccept(Frame frame) // reach a term both fst&fsa accepts { - return fsa.IsAccept(frame.fsaState) && frame.fstArc.Final; + return fsa.IsAccept(frame.fsaState) && frame.fstArc.IsFinal; } internal bool IsValid(Frame frame) // reach a prefix both fst&fsa won't reject { @@ -790,7 +790,7 @@ internal bool CanGrow(Frame frame) // can walk forward on both fst&fsa } internal bool CanRewind(Frame frame) // can jump to sibling { - return !frame.fstArc.Last; + return !frame.fstArc.IsLast; } internal void PushFrame(Frame frame) @@ -882,7 +882,7 @@ internal static void Walk(FST fst) while (true) { queue.Add((new FST.Arc()).CopyFrom(arc)); - if (arc.Last) + if (arc.IsLast) { break; } diff --git a/src/Lucene.Net.Core/Codecs/BlockTreeTermsReader.cs b/src/Lucene.Net.Core/Codecs/BlockTreeTermsReader.cs index 154d23cca7..8ae85d35e4 100644 --- a/src/Lucene.Net.Core/Codecs/BlockTreeTermsReader.cs +++ b/src/Lucene.Net.Core/Codecs/BlockTreeTermsReader.cs @@ -1073,7 +1073,7 @@ public IntersectEnum(BlockTreeTermsReader.FieldReader outerInstance, CompiledAut FST.Arc arc = outerInstance.Index.GetFirstArc(Arcs[0]); // Empty string prefix must have an output in the index! - Debug.Assert(arc.Final); + Debug.Assert(arc.IsFinal); // Special pushFrame since it's the first one: Frame f = Stack[0]; @@ -1169,7 +1169,7 @@ private Frame PushFrame(int state) f.Arc = arc; f.OutputPrefix = output; - Debug.Assert(arc.Final); + Debug.Assert(arc.IsFinal); f.Load(OuterInstance.OuterInstance.FstOutputs.Add(output, arc.NextFinalOutput)); return f; } @@ -1578,7 +1578,7 @@ public SegmentTermsEnum(BlockTreeTermsReader.FieldReader outerInstance) { arc = outerInstance.Index.GetFirstArc(Arcs[0]); // Empty string prefix must have an output in the index! - Debug.Assert(arc.Final); + Debug.Assert(arc.IsFinal); } else { @@ -1626,7 +1626,7 @@ public Stats ComputeBlockStats() { arc = OuterInstance.Index.GetFirstArc(Arcs[0]); // Empty string prefix must have an output in the index! - Debug.Assert(arc.Final); + Debug.Assert(arc.IsFinal); } else { @@ -1701,7 +1701,7 @@ public Stats ComputeBlockStats() { arc = OuterInstance.Index.GetFirstArc(Arcs[0]); // Empty string prefix must have an output in the index! - Debug.Assert(arc.Final); + Debug.Assert(arc.IsFinal); } else { @@ -1861,7 +1861,7 @@ public override bool SeekExact(BytesRef target) // } arc = Arcs[0]; - Debug.Assert(arc.Final); + Debug.Assert(arc.IsFinal); output = arc.Output; targetUpto = 0; @@ -1895,7 +1895,7 @@ public override bool SeekExact(BytesRef target) { output = OuterInstance.OuterInstance.FstOutputs.Add(output, arc.Output); } - if (arc.Final) + if (arc.IsFinal) { lastFrame = Stack[1 + lastFrame.Ord]; } @@ -1982,7 +1982,7 @@ public override bool SeekExact(BytesRef target) arc = OuterInstance.Index.GetFirstArc(Arcs[0]); // Empty string prefix must have an output (block) in the index! - Debug.Assert(arc.Final); + Debug.Assert(arc.IsFinal); Debug.Assert(arc.Output != null); // if (DEBUG) { @@ -2066,7 +2066,7 @@ public override bool SeekExact(BytesRef target) // } targetUpto++; - if (arc.Final) + if (arc.IsFinal) { //if (DEBUG) System.out.println(" arc is final!"); CurrentFrame = PushFrame(arc, OuterInstance.OuterInstance.FstOutputs.Add(output, arc.NextFinalOutput), targetUpto); @@ -2150,7 +2150,7 @@ public override SeekStatus SeekCeil(BytesRef target) //} arc = Arcs[0]; - Debug.Assert(arc.Final); + Debug.Assert(arc.IsFinal); output = arc.Output; targetUpto = 0; @@ -2186,7 +2186,7 @@ public override SeekStatus SeekCeil(BytesRef target) { output = OuterInstance.OuterInstance.FstOutputs.Add(output, arc.Output); } - if (arc.Final) + if (arc.IsFinal) { lastFrame = Stack[1 + lastFrame.Ord]; } @@ -2267,7 +2267,7 @@ public override SeekStatus SeekCeil(BytesRef target) arc = OuterInstance.Index.GetFirstArc(Arcs[0]); // Empty string prefix must have an output (block) in the index! - Debug.Assert(arc.Final); + Debug.Assert(arc.IsFinal); Debug.Assert(arc.Output != null); //if (DEBUG) { @@ -2353,7 +2353,7 @@ public override SeekStatus SeekCeil(BytesRef target) //} targetUpto++; - if (arc.Final) + if (arc.IsFinal) { //if (DEBUG) System.out.println(" arc is final!"); CurrentFrame = PushFrame(arc, OuterInstance.OuterInstance.FstOutputs.Add(output, arc.NextFinalOutput), targetUpto); @@ -2474,7 +2474,7 @@ public override BytesRef Next() { arc = OuterInstance.Index.GetFirstArc(Arcs[0]); // Empty string prefix must have an output in the index! - Debug.Assert(arc.Final); + Debug.Assert(arc.IsFinal); } else { diff --git a/src/Lucene.Net.Core/Util/Fst/Builder.cs b/src/Lucene.Net.Core/Util/Fst/Builder.cs index bfcf269af1..a566625e1a 100644 --- a/src/Lucene.Net.Core/Util/Fst/Builder.cs +++ b/src/Lucene.Net.Core/Util/Fst/Builder.cs @@ -554,7 +554,7 @@ private void CompileAllTargets(UnCompiledNode node, int tailLength) for (int arcIdx = 0; arcIdx < node.NumArcs; arcIdx++) { Arc arc = node.Arcs[arcIdx]; - if (!arc.Target.Compiled) + if (!arc.Target.IsCompiled) { // not yet compiled UnCompiledNode n = (UnCompiledNode)arc.Target; @@ -585,7 +585,7 @@ public class Arc public interface Node { - bool Compiled { get; } + bool IsCompiled { get; } } public virtual long FstSizeInBytes() @@ -597,7 +597,7 @@ public sealed class CompiledNode : Node { public long Node; - public bool Compiled + public bool IsCompiled { get { @@ -640,7 +640,7 @@ public UnCompiledNode(Builder owner, int depth) this.Depth = depth; } - public bool Compiled + public bool IsCompiled { get { diff --git a/src/Lucene.Net.Core/Util/Fst/BytesStore.cs b/src/Lucene.Net.Core/Util/Fst/BytesStore.cs index 471cd68276..b23da69f08 100644 --- a/src/Lucene.Net.Core/Util/Fst/BytesStore.cs +++ b/src/Lucene.Net.Core/Util/Fst/BytesStore.cs @@ -490,9 +490,9 @@ public override long Position } } - public override bool Reversed() + public override bool IsReversed { - return false; + get { return false; } } } @@ -572,9 +572,9 @@ public override long Position } } - public override bool Reversed() + public override bool IsReversed { - return true; + get { return true; } } } } diff --git a/src/Lucene.Net.Core/Util/Fst/FST.cs b/src/Lucene.Net.Core/Util/Fst/FST.cs index 0a21a54c30..d5f53c3438 100644 --- a/src/Lucene.Net.Core/Util/Fst/FST.cs +++ b/src/Lucene.Net.Core/Util/Fst/FST.cs @@ -409,7 +409,7 @@ public void ReadRootArcs(Arc[] arcs) { break; } - if (arc.Last) + if (arc.IsLast) { break; } @@ -935,7 +935,7 @@ public Arc ReadLastTargetArc(Arc follow, Arc arc, FST.BytesReader @in) if (!TargetHasArcs(follow)) { //System.out.println(" end node"); - Debug.Assert(follow.Final); + Debug.Assert(follow.IsFinal); arc.Label = END_LABEL; arc.Target = FINAL_END_NODE; arc.Output = follow.NextFinalOutput; @@ -969,7 +969,7 @@ public Arc ReadLastTargetArc(Arc follow, Arc arc, FST.BytesReader @in) // non-array: linear scan arc.BytesPerArc = 0; //System.out.println(" scan"); - while (!arc.Last) + while (!arc.IsLast) { // skip this arc: ReadLabel(@in); @@ -1002,7 +1002,7 @@ public Arc ReadLastTargetArc(Arc follow, Arc arc, FST.BytesReader @in) arc.NextArc = @in.Position; } ReadNextRealArc(arc, @in); - Debug.Assert(arc.Last); + Debug.Assert(arc.IsLast); return arc; } } @@ -1031,7 +1031,7 @@ public Arc ReadFirstTargetArc(Arc follow, Arc arc, FST.BytesReader @in) { //int pos = address; //System.out.println(" readFirstTarget follow.target=" + follow.Target + " isFinal=" + follow.isFinal()); - if (follow.Final) + if (follow.IsFinal) { // Insert "fake" final first arc: arc.Label = END_LABEL; @@ -1136,7 +1136,7 @@ public Arc ReadNextArc(Arc arc, FST.BytesReader @in) /// public int ReadNextArcLabel(Arc arc, BytesReader @in) { - Debug.Assert(!arc.Last); + Debug.Assert(!arc.IsLast); if (arc.Label == END_LABEL) { @@ -1317,7 +1317,7 @@ public Arc FindTargetArc(int labelToMatch, Arc follow, Arc arc, FST.Byt { if (labelToMatch == END_LABEL) { - if (follow.Final) + if (follow.IsFinal) { if (follow.Target <= 0) { @@ -1429,7 +1429,7 @@ public Arc FindTargetArc(int labelToMatch, Arc follow, Arc arc, FST.Byt { return null; } - else if (arc.Last) + else if (arc.IsLast) { return null; } @@ -1554,7 +1554,7 @@ public abstract class BytesReader : DataInput /// Returns true if this reader uses reversed bytes /// under-the-hood. /// - public abstract bool Reversed(); + public abstract bool IsReversed { get; } /// /// Skips bytes. @@ -1874,7 +1874,7 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve sbyte flags = 0; - if (arc.Last) + if (arc.IsLast) { flags += (sbyte)BIT_LAST_ARC; } @@ -1891,7 +1891,7 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve nextCount++; } } - if (arc.Final) + if (arc.IsFinal) { flags += (sbyte)BIT_FINAL_ARC; if (!arc.NextFinalOutput.Equals(NO_OUTPUT)) @@ -2021,7 +2021,7 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve writer.SkipBytes((int)(arcStartPos + bytesPerArc - writer.Position)); } - if (arc.Last) + if (arc.IsLast) { break; } @@ -2254,7 +2254,7 @@ public abstract class BytesReader : DataInput /// under-the-hood. /// /// - public abstract bool Reversed(); + public abstract bool IsReversed { get; } /// /// Skips bytes. @@ -2333,12 +2333,12 @@ internal bool Flag(int flag) return FST.Flag(Flags, flag); } - public bool Last + public bool IsLast { get { return Flag(BIT_LAST_ARC); } } - public bool Final + public bool IsFinal { get { return Flag(BIT_FINAL_ARC); } } diff --git a/src/Lucene.Net.Core/Util/Fst/FSTEnum.cs b/src/Lucene.Net.Core/Util/Fst/FSTEnum.cs index 93469ce38d..3e19974f15 100644 --- a/src/Lucene.Net.Core/Util/Fst/FSTEnum.cs +++ b/src/Lucene.Net.Core/Util/Fst/FSTEnum.cs @@ -114,7 +114,7 @@ protected internal virtual void DoNext() { // pop //System.out.println(" check pop curArc target=" + arcs[upto].target + " label=" + arcs[upto].label + " isLast?=" + arcs[upto].isLast()); - while (Arcs[Upto].Last) + while (Arcs[Upto].IsLast) { Upto--; if (Upto == 0) @@ -218,7 +218,7 @@ protected internal virtual void DoSeekCeil() // Dead end arc.ArcIdx = arc.NumArcs - 2; Fst.ReadNextRealArc(arc, @in); - Debug.Assert(arc.Last); + Debug.Assert(arc.IsLast); // Dead end (target is after the last arc); // rollback to last fork then push Upto--; @@ -230,7 +230,7 @@ protected internal virtual void DoSeekCeil() } FST.Arc prevArc = GetArc(Upto); //System.out.println(" rollback upto=" + upto + " arc.label=" + prevArc.label + " isLast?=" + prevArc.isLast()); - if (!prevArc.Last) + if (!prevArc.IsLast) { Fst.ReadNextArc(prevArc, FstReader); PushFirst(); @@ -269,7 +269,7 @@ protected internal virtual void DoSeekCeil() PushFirst(); return; } - else if (arc.Last) + else if (arc.IsLast) { // Dead end (target is after the last arc); // rollback to last fork then push @@ -282,7 +282,7 @@ protected internal virtual void DoSeekCeil() } FST.Arc prevArc = GetArc(Upto); //System.out.println(" rollback upto=" + upto + " arc.label=" + prevArc.label + " isLast?=" + prevArc.isLast()); - if (!prevArc.Last) + if (!prevArc.IsLast) { Fst.ReadNextArc(prevArc, FstReader); PushFirst(); @@ -401,7 +401,7 @@ protected internal virtual void DoSeekFloor() { // Then, scan forwards to the arc just before // the targetLabel: - while (!arc.Last && Fst.ReadNextArcLabel(arc, @in) < targetLabel) + while (!arc.IsLast && Fst.ReadNextArcLabel(arc, @in) < targetLabel) { Fst.ReadNextArc(arc, FstReader); } @@ -423,7 +423,7 @@ protected internal virtual void DoSeekFloor() arc.ArcIdx = (low > high ? high : low) - 1; //System.out.println(" hasFloor arcIdx=" + (arc.arcIdx+1)); Fst.ReadNextRealArc(arc, @in); - Debug.Assert(arc.Last || Fst.ReadNextArcLabel(arc, @in) > targetLabel); + Debug.Assert(arc.IsLast || Fst.ReadNextArcLabel(arc, @in) > targetLabel); Debug.Assert(arc.Label < targetLabel, "arc.label=" + arc.Label + " vs targetLabel=" + targetLabel); PushLast(); return; @@ -459,7 +459,7 @@ protected internal virtual void DoSeekFloor() { // Then, scan forwards to the arc just before // the targetLabel: - while (!arc.Last && Fst.ReadNextArcLabel(arc, FstReader) < targetLabel) + while (!arc.IsLast && Fst.ReadNextArcLabel(arc, FstReader) < targetLabel) { Fst.ReadNextArc(arc, FstReader); } @@ -475,7 +475,7 @@ protected internal virtual void DoSeekFloor() arc = GetArc(Upto); } } - else if (!arc.Last) + else if (!arc.IsLast) { //System.out.println(" check next label=" + fst.readNextArcLabel(arc) + " (" + (char) fst.readNextArcLabel(arc) + ")"); if (Fst.ReadNextArcLabel(arc, FstReader) > targetLabel) diff --git a/src/Lucene.Net.Core/Util/Fst/ForwardBytesReader.cs b/src/Lucene.Net.Core/Util/Fst/ForwardBytesReader.cs index b8e597baea..039e386803 100644 --- a/src/Lucene.Net.Core/Util/Fst/ForwardBytesReader.cs +++ b/src/Lucene.Net.Core/Util/Fst/ForwardBytesReader.cs @@ -62,9 +62,9 @@ public override long Position } } - public override bool Reversed() + public override bool IsReversed { - return false; + get { return false; } } } } \ No newline at end of file diff --git a/src/Lucene.Net.Core/Util/Fst/NodeHash.cs b/src/Lucene.Net.Core/Util/Fst/NodeHash.cs index ad8f2d3e91..ca92deb7dc 100644 --- a/src/Lucene.Net.Core/Util/Fst/NodeHash.cs +++ b/src/Lucene.Net.Core/Util/Fst/NodeHash.cs @@ -51,12 +51,12 @@ private bool NodesEqual(Builder.UnCompiledNode node, long address) for (int arcUpto = 0; arcUpto < node.NumArcs; arcUpto++) { Builder.Arc arc = node.Arcs[arcUpto]; - if (arc.Label != ScratchArc.Label || !arc.Output.Equals(ScratchArc.Output) || ((Builder.CompiledNode)arc.Target).Node != ScratchArc.Target || !arc.NextFinalOutput.Equals(ScratchArc.NextFinalOutput) || arc.IsFinal != ScratchArc.Final) + if (arc.Label != ScratchArc.Label || !arc.Output.Equals(ScratchArc.Output) || ((Builder.CompiledNode)arc.Target).Node != ScratchArc.Target || !arc.NextFinalOutput.Equals(ScratchArc.NextFinalOutput) || arc.IsFinal != ScratchArc.IsFinal) { return false; } - if (ScratchArc.Last) + if (ScratchArc.IsLast) { if (arcUpto == node.NumArcs - 1) { @@ -157,11 +157,11 @@ private long Hash(long node) h = PRIME * h + ScratchArc.NextFinalOutput.GetHashCode(); } - if (ScratchArc.Final) + if (ScratchArc.IsFinal) { h += 17; } - if (ScratchArc.Last) + if (ScratchArc.IsLast) { break; } diff --git a/src/Lucene.Net.Core/Util/Fst/ReverseBytesReader.cs b/src/Lucene.Net.Core/Util/Fst/ReverseBytesReader.cs index 65e60f62ae..3f68547f79 100644 --- a/src/Lucene.Net.Core/Util/Fst/ReverseBytesReader.cs +++ b/src/Lucene.Net.Core/Util/Fst/ReverseBytesReader.cs @@ -59,9 +59,9 @@ public override long Position } } - public override bool Reversed() + public override bool IsReversed { - return true; + get { return true; } } } } \ No newline at end of file diff --git a/src/Lucene.Net.Core/Util/Fst/Util.cs b/src/Lucene.Net.Core/Util/Fst/Util.cs index 00d2507f94..749f209ffa 100644 --- a/src/Lucene.Net.Core/Util/Fst/Util.cs +++ b/src/Lucene.Net.Core/Util/Fst/Util.cs @@ -62,7 +62,7 @@ public static T Get(FST fst, IntsRef input) output = fst.Outputs.Add(output, arc.Output); } - if (arc.Final) + if (arc.IsFinal) { return fst.Outputs.Add(output, arc.NextFinalOutput); } @@ -98,7 +98,7 @@ public static T Get(FST fst, BytesRef input) output = fst.Outputs.Add(output, arc.Output); } - if (arc.Final) + if (arc.IsFinal) { return fst.Outputs.Add(output, arc.NextFinalOutput); } @@ -150,7 +150,7 @@ public static IntsRef GetByOutput(FST fst, long targetOutput, FST. while (true) { //System.out.println("loop: output=" + output + " upto=" + upto + " arc=" + arc); - if (arc.Final) + if (arc.IsFinal) { long finalOutput = output + arc.NextFinalOutput.Value; //System.out.println(" isFinal finalOutput=" + finalOutput); @@ -270,7 +270,7 @@ public static IntsRef GetByOutput(FST fst, long targetOutput, FST. break; } } - else if (arc.Last) + else if (arc.IsLast) { // Recurse on this arc: output = minArcOutput; @@ -465,7 +465,7 @@ public virtual void AddStartPaths(FST.Arc node, T startOutput, bool allowE { AddIfCompetitive(path); } - if (path.Arc.Last) + if (path.Arc.IsLast) { break; } @@ -572,7 +572,7 @@ public virtual TopResults Search() { AddIfCompetitive(path); } - if (path.Arc.Last) + if (path.Arc.IsLast) { break; } @@ -777,7 +777,7 @@ public static void ToDot(FST fst, TextWriter @out, bool sameRank, bool lab bool isFinal; T finalOutput; - if (startArc.Final) + if (startArc.IsFinal) { isFinal = true; finalOutput = startArc.NextFinalOutput.Equals(NO_OUTPUT) ? default(T) : startArc.NextFinalOutput; @@ -875,7 +875,7 @@ public static void ToDot(FST fst, TextWriter @out, bool sameRank, bool lab outs = ""; } - if (!FST.TargetHasArcs(arc) && arc.Final && !arc.NextFinalOutput.Equals(NO_OUTPUT)) + if (!FST.TargetHasArcs(arc) && arc.IsFinal && !arc.NextFinalOutput.Equals(NO_OUTPUT)) { // Tricky special case: sometimes, due to // pruning, the builder can [sillily] produce @@ -897,10 +897,10 @@ public static void ToDot(FST fst, TextWriter @out, bool sameRank, bool lab } Debug.Assert(arc.Label != FST.END_LABEL); - @out.Write(" " + node + " -> " + arc.Target + " [label=\"" + PrintableLabel(arc.Label) + outs + "\"" + (arc.Final ? " style=\"bold\"" : "") + " color=\"" + arcColor + "\"]\n"); + @out.Write(" " + node + " -> " + arc.Target + " [label=\"" + PrintableLabel(arc.Label) + outs + "\"" + (arc.IsFinal ? " style=\"bold\"" : "") + " color=\"" + arcColor + "\"]\n"); // Break the loop if we're on the last arc of this state. - if (arc.Last) + if (arc.IsLast) { //System.out.println(" break"); break; @@ -1071,7 +1071,7 @@ public static FST.Arc ReadCeilArc(int label, FST fst, FST.Arc // TODO maybe this is a useful in the FST class - we could simplify some other code like FSTEnum? if (label == FST.END_LABEL) { - if (follow.Final) + if (follow.IsFinal) { if (follow.Target <= 0) { @@ -1156,7 +1156,7 @@ public static FST.Arc ReadCeilArc(int label, FST fst, FST.Arc // System.out.println(" found!"); return arc; } - else if (arc.Last) + else if (arc.IsLast) { return null; } diff --git a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs index 4a38575bea..e5d6f84658 100644 --- a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs +++ b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs @@ -274,7 +274,7 @@ private T RandomAcceptedWord(FST fst, IntsRef @in) // read all arcs: fst.ReadFirstTargetArc(arc, arc, fstReader); arcs.Add((new FST.Arc()).CopyFrom(arc)); - while (!arc.Last) + while (!arc.IsLast) { fst.ReadNextArc(arc, fstReader); arcs.Add((new FST.Arc()).CopyFrom(arc)); diff --git a/src/Lucene.Net.Tests/core/Util/Fst/TestBytesStore.cs b/src/Lucene.Net.Tests/core/Util/Fst/TestBytesStore.cs index ec5b8e6355..a07fdcfd64 100644 --- a/src/Lucene.Net.Tests/core/Util/Fst/TestBytesStore.cs +++ b/src/Lucene.Net.Tests/core/Util/Fst/TestBytesStore.cs @@ -286,7 +286,7 @@ private void Verify(BytesStore bytes, byte[] expected, int totalLength) } // reversed FST.BytesReader r2 = bytes.ReverseReader; - Assert.IsTrue(r2.Reversed()); + Assert.IsTrue(r2.IsReversed); r2.Position = totalLength - 1; r2.ReadBytes(actual, 0, actual.Length); int start = 0; @@ -308,7 +308,7 @@ private void Verify(BytesStore bytes, byte[] expected, int totalLength) Console.WriteLine(" bulk: forward"); } FST.BytesReader r3 = bytes.ForwardReader; - Assert.IsFalse(r3.Reversed()); + Assert.IsFalse(r3.IsReversed); r3.ReadBytes(actual, 0, actual.Length); } diff --git a/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs b/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs index 2cabfd2748..2e972ad2d1 100644 --- a/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs +++ b/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs @@ -1319,7 +1319,7 @@ public int VerifyStateAndBelow(FST fst, FST.Arc arc, int depth) int children = VerifyStateAndBelow(fst, new FST.Arc().CopyFrom(arc), depth + 1); Assert.AreEqual(expanded, (depth <= FST.FIXED_ARRAY_SHALLOW_DISTANCE && children >= FST.FIXED_ARRAY_NUM_ARCS_SHALLOW) || children >= FST.FIXED_ARRAY_NUM_ARCS_DEEP); - if (arc.Last) + if (arc.IsLast) { break; } @@ -1437,11 +1437,11 @@ private void CheckStopNodes(FST fst, PositiveIntOutputs outputs) FST.Arc arc = fst.ReadFirstTargetArc(startArc, new FST.Arc(), fst.BytesReader); Assert.AreEqual('a', arc.Label); Assert.AreEqual(17, arc.NextFinalOutput); - Assert.IsTrue(arc.Final); + Assert.IsTrue(arc.IsFinal); arc = fst.ReadNextArc(arc, fst.BytesReader); Assert.AreEqual('b', arc.Label); - Assert.IsFalse(arc.Final); + Assert.IsFalse(arc.IsFinal); Assert.AreEqual(42, arc.Output); } From dfa23109cbe204fa9339e7d4ab23f854c18b46c9 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Wed, 7 Sep 2016 16:49:13 +0700 Subject: [PATCH 09/47] Removed duplicate (hidden) class definitions in FST. --- src/Lucene.Net.Core/Util/Fst/FST.cs | 101 ++++++---------------------- 1 file changed, 21 insertions(+), 80 deletions(-) diff --git a/src/Lucene.Net.Core/Util/Fst/FST.cs b/src/Lucene.Net.Core/Util/Fst/FST.cs index d5f53c3438..f0d7173a65 100644 --- a/src/Lucene.Net.Core/Util/Fst/FST.cs +++ b/src/Lucene.Net.Core/Util/Fst/FST.cs @@ -1561,17 +1561,6 @@ public abstract class BytesReader : DataInput public abstract void SkipBytes(int count); }*/ - private class ArcAndState - { - internal readonly Arc Arc; - internal readonly IntsRef Chain; - - public ArcAndState(Arc arc, IntsRef chain) - { - this.Arc = arc; - this.Chain = chain; - } - } /* public void countSingleChains() throws IOException { @@ -2108,50 +2097,6 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve return fst; } - - private class NodeAndInCount : IComparable - { - internal readonly int Node; - internal readonly int Count; - - public NodeAndInCount(int node, int count) - { - this.Node = node; - this.Count = count; - } - - public virtual int CompareTo(NodeAndInCount other) - { - if (Count > other.Count) - { - return 1; - } - else if (Count < other.Count) - { - return -1; - } - else - { - // Tie-break: smaller node compares as greater than - return other.Node - Node; - } - } - } - - private class NodeQueue : PriorityQueue - { - public NodeQueue(int topN) - : base(topN, false) - { - } - - public override bool LessThan(NodeAndInCount a, NodeAndInCount b) - { - int cmp = a.CompareTo(b); - Debug.Assert(cmp != 0); - return cmp < 0; - } - } } /// @@ -2360,47 +2305,43 @@ public override string ToString() } internal class ArcAndState - where T : class { - private readonly Arc _arc; - - public Arc Arc { get { return _arc; } } - - private readonly IntsRef _chain; - - public IntsRef Chain { get { return _chain; } } + internal readonly Arc Arc; + internal readonly IntsRef Chain; public ArcAndState(Arc arc, IntsRef chain) { - _arc = arc; - _chain = chain; + this.Arc = arc; + this.Chain = chain; } } internal class NodeAndInCount : IComparable { - private readonly int _node; - - public int Node { get { return _node; } } - - private readonly int _count; - - public int Count { get { return _count; } } + internal readonly int Node; + internal readonly int Count; public NodeAndInCount(int node, int count) { - _node = node; - _count = count; + this.Node = node; + this.Count = count; } - public int CompareTo(NodeAndInCount other) + public virtual int CompareTo(NodeAndInCount other) { if (Count > other.Count) + { return 1; - if (Count < other.Count) + } + else if (Count < other.Count) + { return -1; - // Tie-break: smaller node compares as greater than - return other.Node - Node; + } + else + { + // Tie-break: smaller node compares as greater than + return other.Node - Node; + } } } @@ -2413,8 +2354,8 @@ public NodeQueue(int topN) public override bool LessThan(NodeAndInCount a, NodeAndInCount b) { - var cmp = a.CompareTo(b); - // assert cmp != 0; + int cmp = a.CompareTo(b); + Debug.Assert(cmp != 0); return cmp < 0; } } From b8db797ba917041dabe034e3ddf5960dd14fc04a Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Wed, 7 Sep 2016 17:28:33 +0700 Subject: [PATCH 10/47] .NETify FST: Private/protected fields should be camelCase. --- src/Lucene.Net.Core/Util/Fst/Builder.cs | 164 +++++++------- .../Util/Fst/ByteSequenceOutputs.cs | 4 +- .../Util/Fst/BytesRefFSTEnum.cs | 44 ++-- src/Lucene.Net.Core/Util/Fst/BytesStore.cs | 180 +++++++-------- .../Util/Fst/CharSequenceOutputs.cs | 4 +- src/Lucene.Net.Core/Util/Fst/FST.cs | 206 +++++++++--------- src/Lucene.Net.Core/Util/Fst/FSTEnum.cs | 190 ++++++++-------- .../Util/Fst/ForwardBytesReader.cs | 18 +- .../Util/Fst/IntSequenceOutputs.cs | 4 +- .../Util/Fst/IntsRefFSTEnum.cs | 44 ++-- src/Lucene.Net.Core/Util/Fst/NoOutputs.cs | 4 +- src/Lucene.Net.Core/Util/Fst/NodeHash.cs | 84 +++---- src/Lucene.Net.Core/Util/Fst/PairOutputs.cs | 44 ++-- .../Util/Fst/ReverseBytesReader.cs | 16 +- src/Lucene.Net.Core/Util/Fst/Util.cs | 54 ++--- 15 files changed, 531 insertions(+), 529 deletions(-) diff --git a/src/Lucene.Net.Core/Util/Fst/Builder.cs b/src/Lucene.Net.Core/Util/Fst/Builder.cs index a566625e1a..540b9131ac 100644 --- a/src/Lucene.Net.Core/Util/Fst/Builder.cs +++ b/src/Lucene.Net.Core/Util/Fst/Builder.cs @@ -49,36 +49,36 @@ namespace Lucene.Net.Util.Fst public class Builder { - private readonly NodeHash DedupHash; - private readonly FST Fst; + private readonly NodeHash dedupHash; + private readonly FST fst; private readonly T NO_OUTPUT; // private static final boolean DEBUG = true; // simplistic pruning: we prune node (and all following // nodes) if less than this number of terms go through it: - private readonly int MinSuffixCount1; + private readonly int minSuffixCount1; // better pruning: we prune node (and all following // nodes) if the prior node has less than this number of // terms go through it: - private readonly int MinSuffixCount2; + private readonly int minSuffixCount2; - private readonly bool DoShareNonSingletonNodes; - private readonly int ShareMaxTailLength; + private readonly bool doShareNonSingletonNodes; + private readonly int shareMaxTailLength; - private readonly IntsRef LastInput = new IntsRef(); + private readonly IntsRef lastInput = new IntsRef(); // for packing - private readonly bool DoPackFST; + private readonly bool doPackFST; - private readonly float AcceptableOverheadRatio; + private readonly float acceptableOverheadRatio; // NOTE: cutting this over to ArrayList instead loses ~6% // in build performance on 9.8M Wikipedia terms; so we // left this as an array: // current "frontier" - private UnCompiledNode[] Frontier; + private UnCompiledNode[] frontier; /// /// Expert: this is invoked by Builder whenever a suffix @@ -89,7 +89,7 @@ public abstract class FreezeTail public abstract void Freeze(UnCompiledNode[] frontier, int prefixLenPlus1, IntsRef prevInput); } - private readonly FreezeTail FreezeTail_Renamed; + private readonly FreezeTail freezeTail; /// /// Instantiates an FST/FSA builder without any pruning. A shortcut @@ -154,29 +154,29 @@ public Builder(FST.INPUT_TYPE inputType, Outputs outputs) /// bits = 32768 byte pages. public Builder(FST.INPUT_TYPE inputType, int minSuffixCount1, int minSuffixCount2, bool doShareSuffix, bool doShareNonSingletonNodes, int shareMaxTailLength, Outputs outputs, FreezeTail freezeTail, bool doPackFST, float acceptableOverheadRatio, bool allowArrayArcs, int bytesPageBits) { - this.MinSuffixCount1 = minSuffixCount1; - this.MinSuffixCount2 = minSuffixCount2; - this.FreezeTail_Renamed = freezeTail; - this.DoShareNonSingletonNodes = doShareNonSingletonNodes; - this.ShareMaxTailLength = shareMaxTailLength; - this.DoPackFST = doPackFST; - this.AcceptableOverheadRatio = acceptableOverheadRatio; - Fst = new FST(inputType, outputs, doPackFST, acceptableOverheadRatio, allowArrayArcs, bytesPageBits); + this.minSuffixCount1 = minSuffixCount1; + this.minSuffixCount2 = minSuffixCount2; + this.freezeTail = freezeTail; + this.doShareNonSingletonNodes = doShareNonSingletonNodes; + this.shareMaxTailLength = shareMaxTailLength; + this.doPackFST = doPackFST; + this.acceptableOverheadRatio = acceptableOverheadRatio; + fst = new FST(inputType, outputs, doPackFST, acceptableOverheadRatio, allowArrayArcs, bytesPageBits); if (doShareSuffix) { - DedupHash = new NodeHash(Fst, Fst.Bytes.GetReverseReader(false)); + dedupHash = new NodeHash(fst, fst.Bytes.GetReverseReader(false)); } else { - DedupHash = null; + dedupHash = null; } NO_OUTPUT = outputs.NoOutput; UnCompiledNode[] f = (UnCompiledNode[])new UnCompiledNode[10]; - Frontier = f; - for (int idx = 0; idx < Frontier.Length; idx++) + frontier = f; + for (int idx = 0; idx < frontier.Length; idx++) { - Frontier[idx] = new UnCompiledNode(this, idx); + frontier[idx] = new UnCompiledNode(this, idx); } } @@ -184,7 +184,7 @@ public virtual long TotStateCount { get { - return Fst.nodeCount; + return fst.nodeCount; } } @@ -192,7 +192,7 @@ public virtual long TermCount { get { - return Frontier[0].InputCount; + return frontier[0].InputCount; } } @@ -200,27 +200,27 @@ public virtual long MappedStateCount { get { - return DedupHash == null ? 0 : Fst.nodeCount; + return dedupHash == null ? 0 : fst.nodeCount; } } private CompiledNode CompileNode(UnCompiledNode nodeIn, int tailLength) { long node; - if (DedupHash != null && (DoShareNonSingletonNodes || nodeIn.NumArcs <= 1) && tailLength <= ShareMaxTailLength) + if (dedupHash != null && (doShareNonSingletonNodes || nodeIn.NumArcs <= 1) && tailLength <= shareMaxTailLength) { if (nodeIn.NumArcs == 0) { - node = Fst.AddNode(nodeIn); + node = fst.AddNode(nodeIn); } else { - node = DedupHash.Add(nodeIn); + node = dedupHash.Add(nodeIn); } } else { - node = Fst.AddNode(nodeIn); + node = fst.AddNode(nodeIn); } Debug.Assert(node != -2); @@ -233,24 +233,24 @@ private CompiledNode CompileNode(UnCompiledNode nodeIn, int tailLength) private void DoFreezeTail(int prefixLenPlus1) { - if (FreezeTail_Renamed != null) + if (freezeTail != null) { // Custom plugin: - FreezeTail_Renamed.Freeze(Frontier, prefixLenPlus1, LastInput); + freezeTail.Freeze(frontier, prefixLenPlus1, lastInput); } else { //System.out.println(" compileTail " + prefixLenPlus1); int downTo = Math.Max(1, prefixLenPlus1); - for (int idx = LastInput.Length; idx >= downTo; idx--) + for (int idx = lastInput.Length; idx >= downTo; idx--) { bool doPrune = false; bool doCompile = false; - UnCompiledNode node = Frontier[idx]; - UnCompiledNode parent = Frontier[idx - 1]; + UnCompiledNode node = frontier[idx]; + UnCompiledNode parent = frontier[idx - 1]; - if (node.InputCount < MinSuffixCount1) + if (node.InputCount < minSuffixCount1) { doPrune = true; doCompile = true; @@ -258,7 +258,7 @@ private void DoFreezeTail(int prefixLenPlus1) else if (idx > prefixLenPlus1) { // prune if parent's inputCount is less than suffixMinCount2 - if (parent.InputCount < MinSuffixCount2 || (MinSuffixCount2 == 1 && parent.InputCount == 1 && idx > 1)) + if (parent.InputCount < minSuffixCount2 || (minSuffixCount2 == 1 && parent.InputCount == 1 && idx > 1)) { // my parent, about to be compiled, doesn't make the cut, so // I'm definitely pruned @@ -284,12 +284,12 @@ private void DoFreezeTail(int prefixLenPlus1) { // if pruning is disabled (count is 0) we can always // compile current node - doCompile = MinSuffixCount2 == 0; + doCompile = minSuffixCount2 == 0; } //System.out.println(" label=" + ((char) lastInput.ints[lastInput.offset+idx-1]) + " idx=" + idx + " inputCount=" + frontier[idx].inputCount + " doCompile=" + doCompile + " doPrune=" + doPrune); - if (node.InputCount < MinSuffixCount2 || (MinSuffixCount2 == 1 && node.InputCount == 1 && idx > 1)) + if (node.InputCount < minSuffixCount2 || (minSuffixCount2 == 1 && node.InputCount == 1 && idx > 1)) { // drop all arcs for (int arcIdx = 0; arcIdx < node.NumArcs; arcIdx++) @@ -304,13 +304,13 @@ private void DoFreezeTail(int prefixLenPlus1) { // this node doesn't make it -- deref it node.Clear(); - parent.DeleteLast(LastInput.Ints[LastInput.Offset + idx - 1], node); + parent.DeleteLast(lastInput.Ints[lastInput.Offset + idx - 1], node); } else { - if (MinSuffixCount2 != 0) + if (minSuffixCount2 != 0) { - CompileAllTargets(node, LastInput.Length - idx); + CompileAllTargets(node, lastInput.Length - idx); } T nextFinalOutput = node.Output; @@ -326,18 +326,18 @@ private void DoFreezeTail(int prefixLenPlus1) // this node makes it and we now compile it. first, // compile any targets that were previously // undecided: - parent.ReplaceLast(LastInput.Ints[LastInput.Offset + idx - 1], CompileNode(node, 1 + LastInput.Length - idx), nextFinalOutput, isFinal); + parent.ReplaceLast(lastInput.Ints[lastInput.Offset + idx - 1], CompileNode(node, 1 + lastInput.Length - idx), nextFinalOutput, isFinal); } else { // replaceLast just to install // nextFinalOutput/isFinal onto the arc - parent.ReplaceLast(LastInput.Ints[LastInput.Offset + idx - 1], node, nextFinalOutput, isFinal); + parent.ReplaceLast(lastInput.Ints[lastInput.Offset + idx - 1], node, nextFinalOutput, isFinal); // this node will stay in play for now, since we are // undecided on whether to prune it. later, it // will be either compiled or pruned, so we must // allocate a new node: - Frontier[idx] = new UnCompiledNode(this, idx); + frontier[idx] = new UnCompiledNode(this, idx); } } } @@ -400,21 +400,21 @@ public virtual void Add(IntsRef input, T output) // format cannot represent the empty input since // 'finalness' is stored on the incoming arc, not on // the node - Frontier[0].InputCount++; - Frontier[0].IsFinal = true; - Fst.EmptyOutput = output; + frontier[0].InputCount++; + frontier[0].IsFinal = true; + fst.EmptyOutput = output; return; } // compare shared prefix length int pos1 = 0; int pos2 = input.Offset; - int pos1Stop = Math.Min(LastInput.Length, input.Length); + int pos1Stop = Math.Min(lastInput.Length, input.Length); while (true) { - Frontier[pos1].InputCount++; + frontier[pos1].InputCount++; //System.out.println(" incr " + pos1 + " ct=" + frontier[pos1].inputCount + " n=" + frontier[pos1]); - if (pos1 >= pos1Stop || LastInput.Ints[pos1] != input.Ints[pos2]) + if (pos1 >= pos1Stop || lastInput.Ints[pos1] != input.Ints[pos2]) { break; } @@ -423,15 +423,15 @@ public virtual void Add(IntsRef input, T output) } int prefixLenPlus1 = pos1 + 1; - if (Frontier.Length < input.Length + 1) + if (frontier.Length < input.Length + 1) { UnCompiledNode[] next = new UnCompiledNode[ArrayUtil.Oversize(input.Length + 1, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; - Array.Copy(Frontier, 0, next, 0, Frontier.Length); - for (int idx = Frontier.Length; idx < next.Length; idx++) + Array.Copy(frontier, 0, next, 0, frontier.Length); + for (int idx = frontier.Length; idx < next.Length; idx++) { next[idx] = new UnCompiledNode(this, idx); } - Frontier = next; + frontier = next; } // minimize/compile states from previous input's @@ -441,12 +441,12 @@ public virtual void Add(IntsRef input, T output) // init tail states for current input for (int idx = prefixLenPlus1; idx <= input.Length; idx++) { - Frontier[idx - 1].AddArc(input.Ints[input.Offset + idx - 1], Frontier[idx]); - Frontier[idx].InputCount++; + frontier[idx - 1].AddArc(input.Ints[input.Offset + idx - 1], frontier[idx]); + frontier[idx].InputCount++; } - UnCompiledNode lastNode = Frontier[input.Length]; - if (LastInput.Length != input.Length || prefixLenPlus1 != input.Length + 1) + UnCompiledNode lastNode = frontier[input.Length]; + if (lastInput.Length != input.Length || prefixLenPlus1 != input.Length + 1) { lastNode.IsFinal = true; lastNode.Output = NO_OUTPUT; @@ -456,8 +456,8 @@ public virtual void Add(IntsRef input, T output) // needed for (int idx = 1; idx < prefixLenPlus1; idx++) { - UnCompiledNode node = Frontier[idx]; - UnCompiledNode parentNode = Frontier[idx - 1]; + UnCompiledNode node = frontier[idx]; + UnCompiledNode parentNode = frontier[idx - 1]; T lastOutput = parentNode.GetLastOutput(input.Ints[input.Offset + idx - 1]); Debug.Assert(ValidOutput(lastOutput)); @@ -467,9 +467,9 @@ public virtual void Add(IntsRef input, T output) if (!lastOutput.Equals(NO_OUTPUT)) { - commonOutputPrefix = Fst.Outputs.Common(output, lastOutput); + commonOutputPrefix = fst.Outputs.Common(output, lastOutput); Debug.Assert(ValidOutput(commonOutputPrefix)); - wordSuffix = Fst.Outputs.Subtract(lastOutput, commonOutputPrefix); + wordSuffix = fst.Outputs.Subtract(lastOutput, commonOutputPrefix); Debug.Assert(ValidOutput(wordSuffix)); parentNode.SetLastOutput(input.Ints[input.Offset + idx - 1], commonOutputPrefix); node.PrependOutput(wordSuffix); @@ -479,25 +479,25 @@ public virtual void Add(IntsRef input, T output) commonOutputPrefix = wordSuffix = NO_OUTPUT; } - output = Fst.Outputs.Subtract(output, commonOutputPrefix); + output = fst.Outputs.Subtract(output, commonOutputPrefix); Debug.Assert(ValidOutput(output)); } - if (LastInput.Length == input.Length && prefixLenPlus1 == 1 + input.Length) + if (lastInput.Length == input.Length && prefixLenPlus1 == 1 + input.Length) { // same input more than 1 time in a row, mapping to // multiple outputs - lastNode.Output = Fst.Outputs.Merge(lastNode.Output, output); + lastNode.Output = fst.Outputs.Merge(lastNode.Output, output); } else { // this new arc is private to this new input; set its // arc output to the leftover output: - Frontier[prefixLenPlus1 - 1].SetLastOutput(input.Ints[input.Offset + prefixLenPlus1 - 1], output); + frontier[prefixLenPlus1 - 1].SetLastOutput(input.Ints[input.Offset + prefixLenPlus1 - 1], output); } // save last input - LastInput.CopyInts(input); + lastInput.CopyInts(input); //System.out.println(" count[0]=" + frontier[0].inputCount); } @@ -513,17 +513,17 @@ private bool ValidOutput(T output) /// public virtual FST Finish() { - UnCompiledNode root = Frontier[0]; + UnCompiledNode root = frontier[0]; // minimize nodes in the last word's suffix DoFreezeTail(0); - if (root.InputCount < MinSuffixCount1 || root.InputCount < MinSuffixCount2 || root.NumArcs == 0) + if (root.InputCount < minSuffixCount1 || root.InputCount < minSuffixCount2 || root.NumArcs == 0) { - if (Fst.emptyOutput == null) + if (fst.emptyOutput == null) { return null; } - else if (MinSuffixCount1 > 0 || MinSuffixCount2 > 0) + else if (minSuffixCount1 > 0 || minSuffixCount2 > 0) { // empty string got pruned return null; @@ -531,21 +531,21 @@ public virtual FST Finish() } else { - if (MinSuffixCount2 != 0) + if (minSuffixCount2 != 0) { - CompileAllTargets(root, LastInput.Length); + CompileAllTargets(root, lastInput.Length); } } //if (DEBUG) System.out.println(" builder.finish root.isFinal=" + root.isFinal + " root.Output=" + root.Output); - Fst.Finish(CompileNode(root, LastInput.Length).Node); + fst.Finish(CompileNode(root, lastInput.Length).Node); - if (DoPackFST) + if (doPackFST) { - return Fst.Pack(3, Math.Max(10, (int)(Fst.NodeCount / 4)), AcceptableOverheadRatio); + return fst.Pack(3, Math.Max(10, (int)(fst.NodeCount / 4)), acceptableOverheadRatio); } else { - return Fst; + return fst; } } @@ -590,7 +590,7 @@ public interface Node public virtual long FstSizeInBytes() { - return Fst.SizeInBytes(); + return fst.SizeInBytes(); } public sealed class CompiledNode : Node @@ -726,13 +726,13 @@ public void PrependOutput(S outputPrefix) for (int arcIdx = 0; arcIdx < NumArcs; arcIdx++) { - Arcs[arcIdx].Output = Owner.Fst.Outputs.Add(outputPrefix, Arcs[arcIdx].Output); + Arcs[arcIdx].Output = Owner.fst.Outputs.Add(outputPrefix, Arcs[arcIdx].Output); Debug.Assert(Owner.ValidOutput(Arcs[arcIdx].Output)); } if (IsFinal) { - Output = Owner.Fst.Outputs.Add(outputPrefix, Output); + Output = Owner.fst.Outputs.Add(outputPrefix, Output); Debug.Assert(Owner.ValidOutput(Output)); } } diff --git a/src/Lucene.Net.Core/Util/Fst/ByteSequenceOutputs.cs b/src/Lucene.Net.Core/Util/Fst/ByteSequenceOutputs.cs index 08ca319951..1d09ce2f0c 100644 --- a/src/Lucene.Net.Core/Util/Fst/ByteSequenceOutputs.cs +++ b/src/Lucene.Net.Core/Util/Fst/ByteSequenceOutputs.cs @@ -33,7 +33,7 @@ namespace Lucene.Net.Util.Fst public sealed class ByteSequenceOutputs : Outputs { private static readonly BytesRef NO_OUTPUT = new BytesRef(); - private static readonly ByteSequenceOutputs Singleton_Renamed = new ByteSequenceOutputs(); + private static readonly ByteSequenceOutputs singleton = new ByteSequenceOutputs(); private ByteSequenceOutputs() { @@ -43,7 +43,7 @@ public static ByteSequenceOutputs Singleton { get { - return Singleton_Renamed; + return singleton; } } diff --git a/src/Lucene.Net.Core/Util/Fst/BytesRefFSTEnum.cs b/src/Lucene.Net.Core/Util/Fst/BytesRefFSTEnum.cs index 9be468088b..6774b21919 100644 --- a/src/Lucene.Net.Core/Util/Fst/BytesRefFSTEnum.cs +++ b/src/Lucene.Net.Core/Util/Fst/BytesRefFSTEnum.cs @@ -28,9 +28,9 @@ namespace Lucene.Net.Util.Fst public sealed class BytesRefFSTEnum : FSTEnum { - private readonly BytesRef Current_Renamed = new BytesRef(10); - private readonly InputOutput Result = new InputOutput(); - private BytesRef Target; + private readonly BytesRef current = new BytesRef(10); + private readonly InputOutput result = new InputOutput(); + private BytesRef target; /// /// Holds a single input (BytesRef) + output pair. @@ -48,13 +48,13 @@ public class InputOutput public BytesRefFSTEnum(FST fst) : base(fst) { - Result.Input = Current_Renamed; - Current_Renamed.Offset = 1; + result.Input = current; + current.Offset = 1; } public InputOutput Current() { - return Result; + return result; } public InputOutput Next() @@ -68,8 +68,8 @@ public InputOutput Next() /// Seeks to smallest term that's >= target. public InputOutput SeekCeil(BytesRef target) { - this.Target = target; - TargetLength = target.Length; + this.target = target; + targetLength = target.Length; base.DoSeekCeil(); return SetResult(); } @@ -78,8 +78,8 @@ public InputOutput SeekCeil(BytesRef target) /// Seeks to biggest term that's <= target. public InputOutput SeekFloor(BytesRef target) { - this.Target = target; - TargetLength = target.Length; + this.target = target; + targetLength = target.Length; base.DoSeekFloor(); return SetResult(); } @@ -92,11 +92,11 @@ public InputOutput SeekFloor(BytesRef target) /// public InputOutput SeekExact(BytesRef target) { - this.Target = target; - TargetLength = target.Length; + this.target = target; + targetLength = target.Length; if (base.DoSeekExact()) { - Debug.Assert(Upto == 1 + target.Length); + Debug.Assert(upto == 1 + target.Length); return SetResult(); } else @@ -109,13 +109,13 @@ protected internal override int TargetLabel { get { - if (Upto - 1 == Target.Length) + if (upto - 1 == target.Length) { return FST.END_LABEL; } else { - return Target.Bytes[Target.Offset + Upto - 1] & 0xFF; + return target.Bytes[target.Offset + upto - 1] & 0xFF; } } } @@ -125,30 +125,30 @@ protected internal override int CurrentLabel get { // current.offset fixed at 1 - return Current_Renamed.Bytes[Upto] & 0xFF; + return current.Bytes[upto] & 0xFF; } set { - Current_Renamed.Bytes[Upto] = (byte)value; + current.Bytes[upto] = (byte)value; } } protected internal override void Grow() { - Current_Renamed.Bytes = ArrayUtil.Grow(Current_Renamed.Bytes, Upto + 1); + current.Bytes = ArrayUtil.Grow(current.Bytes, upto + 1); } private InputOutput SetResult() { - if (Upto == 0) + if (upto == 0) { return null; } else { - Current_Renamed.Length = Upto - 1; - Result.Output = Output[Upto]; - return Result; + current.Length = upto - 1; + result.Output = output[upto]; + return result; } } } diff --git a/src/Lucene.Net.Core/Util/Fst/BytesStore.cs b/src/Lucene.Net.Core/Util/Fst/BytesStore.cs index b23da69f08..26bc85a5a7 100644 --- a/src/Lucene.Net.Core/Util/Fst/BytesStore.cs +++ b/src/Lucene.Net.Core/Util/Fst/BytesStore.cs @@ -29,21 +29,21 @@ namespace Lucene.Net.Util.Fst internal class BytesStore : DataOutput { - private readonly List Blocks = new List(); + private readonly List blocks = new List(); - private readonly int BlockSize; + private readonly int blockSize; private readonly int blockBits; - private readonly int BlockMask; + private readonly int blockMask; - private byte[] Current; - private int NextWrite; + private byte[] current; + private int nextWrite; public BytesStore(int blockBits) { this.blockBits = blockBits; - BlockSize = 1 << blockBits; - BlockMask = BlockSize - 1; - NextWrite = BlockSize; + blockSize = 1 << blockBits; + blockMask = blockSize - 1; + nextWrite = blockSize; } /// @@ -58,20 +58,20 @@ public BytesStore(DataInput @in, long numBytes, int maxBlockSize) blockBits++; } this.blockBits = blockBits; - this.BlockSize = blockSize; - this.BlockMask = blockSize - 1; + this.blockSize = blockSize; + this.blockMask = blockSize - 1; long left = numBytes; while (left > 0) { int chunk = (int)Math.Min(blockSize, left); byte[] block = new byte[chunk]; @in.ReadBytes(block, 0, block.Length); - Blocks.Add(block); + blocks.Add(block); left -= chunk; } // So .getPosition still works - NextWrite = Blocks[Blocks.Count - 1].Length; + nextWrite = blocks[blocks.Count - 1].Length; } /// @@ -81,43 +81,43 @@ public BytesStore(DataInput @in, long numBytes, int maxBlockSize) public virtual void WriteByte(int dest, byte b) { int blockIndex = dest >> blockBits; - byte[] block = Blocks[blockIndex]; - block[dest & BlockMask] = b; + byte[] block = blocks[blockIndex]; + block[dest & blockMask] = b; } public override void WriteByte(byte b) { - if (NextWrite == BlockSize) + if (nextWrite == blockSize) { - Current = new byte[BlockSize]; - Blocks.Add(Current); - NextWrite = 0; + current = new byte[blockSize]; + blocks.Add(current); + nextWrite = 0; } - Current[NextWrite++] = b; + current[nextWrite++] = b; } public override void WriteBytes(byte[] b, int offset, int len) { while (len > 0) { - int chunk = BlockSize - NextWrite; + int chunk = blockSize - nextWrite; if (len <= chunk) { - System.Buffer.BlockCopy(b, offset, Current, NextWrite, len); - NextWrite += len; + System.Buffer.BlockCopy(b, offset, current, nextWrite, len); + nextWrite += len; break; } else { if (chunk > 0) { - Array.Copy(b, offset, Current, NextWrite, chunk); + Array.Copy(b, offset, current, nextWrite, chunk); offset += chunk; len -= chunk; } - Current = new byte[BlockSize]; - Blocks.Add(Current); - NextWrite = 0; + current = new byte[blockSize]; + blocks.Add(current); + nextWrite = 0; } } } @@ -168,13 +168,13 @@ internal virtual void WriteBytes(long dest, byte[] b, int offset, int len) long end = dest + len; int blockIndex = (int)(end >> blockBits); - int downTo = (int)(end & BlockMask); + int downTo = (int)(end & blockMask); if (downTo == 0) { blockIndex--; - downTo = BlockSize; + downTo = blockSize; } - byte[] block = Blocks[blockIndex]; + byte[] block = blocks[blockIndex]; while (len > 0) { @@ -191,8 +191,8 @@ internal virtual void WriteBytes(long dest, byte[] b, int offset, int len) //System.out.println(" partial: offset=" + (offset + len) + " len=" + downTo + " dest=0"); Array.Copy(b, offset + len, block, 0, downTo); blockIndex--; - block = Blocks[blockIndex]; - downTo = BlockSize; + block = blocks[blockIndex]; + downTo = blockSize; } } } @@ -236,13 +236,13 @@ public virtual void CopyBytes(long src, long dest, int len) long end = src + len; int blockIndex = (int)(end >> blockBits); - int downTo = (int)(end & BlockMask); + int downTo = (int)(end & blockMask); if (downTo == 0) { blockIndex--; - downTo = BlockSize; + downTo = blockSize; } - byte[] block = Blocks[blockIndex]; + byte[] block = blocks[blockIndex]; while (len > 0) { @@ -259,8 +259,8 @@ public virtual void CopyBytes(long src, long dest, int len) len -= downTo; WriteBytes(dest + len, block, 0, downTo); blockIndex--; - block = Blocks[blockIndex]; - downTo = BlockSize; + block = blocks[blockIndex]; + downTo = blockSize; } } } @@ -272,18 +272,18 @@ public virtual void CopyBytes(long src, long dest, int len) public virtual void WriteInt(long pos, int value) { int blockIndex = (int)(pos >> blockBits); - int upto = (int)(pos & BlockMask); - byte[] block = Blocks[blockIndex]; + int upto = (int)(pos & blockMask); + byte[] block = blocks[blockIndex]; int shift = 24; for (int i = 0; i < 4; i++) { block[upto++] = (byte)(value >> shift); shift -= 8; - if (upto == BlockSize) + if (upto == blockSize) { upto = 0; blockIndex++; - block = Blocks[blockIndex]; + block = blocks[blockIndex]; } } } @@ -297,12 +297,12 @@ public virtual void Reverse(long srcPos, long destPos) //System.out.println("reverse src=" + srcPos + " dest=" + destPos); int srcBlockIndex = (int)(srcPos >> blockBits); - int src = (int)(srcPos & BlockMask); - byte[] srcBlock = Blocks[srcBlockIndex]; + int src = (int)(srcPos & blockMask); + byte[] srcBlock = blocks[srcBlockIndex]; int destBlockIndex = (int)(destPos >> blockBits); - int dest = (int)(destPos & BlockMask); - byte[] destBlock = Blocks[destBlockIndex]; + int dest = (int)(destPos & blockMask); + byte[] destBlock = blocks[destBlockIndex]; //System.out.println(" srcBlock=" + srcBlockIndex + " destBlock=" + destBlockIndex); int limit = (int)(destPos - srcPos + 1) / 2; @@ -313,10 +313,10 @@ public virtual void Reverse(long srcPos, long destPos) srcBlock[src] = destBlock[dest]; destBlock[dest] = b; src++; - if (src == BlockSize) + if (src == blockSize) { srcBlockIndex++; - srcBlock = Blocks[srcBlockIndex]; + srcBlock = blocks[srcBlockIndex]; //System.out.println(" set destBlock=" + destBlock + " srcBlock=" + srcBlock); src = 0; } @@ -325,9 +325,9 @@ public virtual void Reverse(long srcPos, long destPos) if (dest == -1) { destBlockIndex--; - destBlock = Blocks[destBlockIndex]; + destBlock = blocks[destBlockIndex]; //System.out.println(" set destBlock=" + destBlock + " srcBlock=" + srcBlock); - dest = BlockSize - 1; + dest = blockSize - 1; } } } @@ -336,18 +336,18 @@ public virtual void SkipBytes(int len) { while (len > 0) { - int chunk = BlockSize - NextWrite; + int chunk = blockSize - nextWrite; if (len <= chunk) { - NextWrite += len; + nextWrite += len; break; } else { len -= chunk; - Current = new byte[BlockSize]; - Blocks.Add(Current); - NextWrite = 0; + current = new byte[blockSize]; + blocks.Add(current); + nextWrite = 0; } } } @@ -356,7 +356,7 @@ public virtual long Position { get { - return ((long)Blocks.Count - 1) * BlockSize + NextWrite; + return ((long)blocks.Count - 1) * blockSize + nextWrite; } } @@ -369,32 +369,32 @@ public virtual void Truncate(long newLen) Debug.Assert(newLen <= Position); Debug.Assert(newLen >= 0); int blockIndex = (int)(newLen >> blockBits); - NextWrite = (int)(newLen & BlockMask); - if (NextWrite == 0) + nextWrite = (int)(newLen & blockMask); + if (nextWrite == 0) { blockIndex--; - NextWrite = BlockSize; + nextWrite = blockSize; } - Blocks.RemoveRange(blockIndex + 1, Blocks.Count - (blockIndex + 1)); + blocks.RemoveRange(blockIndex + 1, blocks.Count - (blockIndex + 1)); if (newLen == 0) { - Current = null; + current = null; } else { - Current = Blocks[blockIndex]; + current = blocks[blockIndex]; } Debug.Assert(newLen == Position); } public virtual void Finish() { - if (Current != null) + if (current != null) { - byte[] lastBuffer = new byte[NextWrite]; - Array.Copy(Current, 0, lastBuffer, 0, NextWrite); - Blocks[Blocks.Count - 1] = lastBuffer; - Current = null; + byte[] lastBuffer = new byte[nextWrite]; + Array.Copy(current, 0, lastBuffer, 0, nextWrite); + blocks[blocks.Count - 1] = lastBuffer; + current = null; } } @@ -402,7 +402,7 @@ public virtual void Finish() /// Writes all of our bytes to the target . public virtual void WriteTo(DataOutput @out) { - foreach (byte[] block in Blocks) + foreach (byte[] block in blocks) { @out.WriteBytes(block, 0, block.Length); } @@ -412,9 +412,9 @@ public virtual FST.BytesReader ForwardReader { get { - if (Blocks.Count == 1) + if (blocks.Count == 1) { - return new ForwardBytesReader(Blocks[0]); + return new ForwardBytesReader(blocks[0]); } return new ForwardBytesReaderAnonymousInner(this); } @@ -422,12 +422,12 @@ public virtual FST.BytesReader ForwardReader private class ForwardBytesReaderAnonymousInner : FST.BytesReader { - private readonly BytesStore OuterInstance; + private readonly BytesStore outerInstance; public ForwardBytesReaderAnonymousInner(BytesStore outerInstance) { - this.OuterInstance = outerInstance; - nextRead = outerInstance.BlockSize; + this.outerInstance = outerInstance; + nextRead = outerInstance.blockSize; } private byte[] Current; @@ -436,9 +436,9 @@ public ForwardBytesReaderAnonymousInner(BytesStore outerInstance) public override byte ReadByte() { - if (nextRead == OuterInstance.BlockSize) + if (nextRead == outerInstance.blockSize) { - Current = OuterInstance.Blocks[nextBuffer++]; + Current = outerInstance.blocks[nextBuffer++]; nextRead = 0; } return Current[nextRead++]; @@ -453,7 +453,7 @@ public override void ReadBytes(byte[] b, int offset, int len) { while (len > 0) { - int chunkLeft = OuterInstance.BlockSize - nextRead; + int chunkLeft = outerInstance.blockSize - nextRead; if (len <= chunkLeft) { Array.Copy(Current, nextRead, b, offset, len); @@ -468,7 +468,7 @@ public override void ReadBytes(byte[] b, int offset, int len) offset += chunkLeft; len -= chunkLeft; } - Current = OuterInstance.Blocks[nextBuffer++]; + Current = outerInstance.blocks[nextBuffer++]; nextRead = 0; } } @@ -478,14 +478,14 @@ public override long Position { get { - return ((long)nextBuffer - 1) * OuterInstance.BlockSize + nextRead; + return ((long)nextBuffer - 1) * outerInstance.blockSize + nextRead; } set { - int bufferIndex = (int)(value >> OuterInstance.blockBits); + int bufferIndex = (int)(value >> outerInstance.blockBits); nextBuffer = bufferIndex + 1; - Current = OuterInstance.Blocks[bufferIndex]; - nextRead = (int)(value & OuterInstance.BlockMask); + Current = outerInstance.blocks[bufferIndex]; + nextRead = (int)(value & outerInstance.blockMask); Debug.Assert(this.Position == value, "pos=" + value + " getPos()=" + this.Position); } } @@ -506,21 +506,21 @@ public virtual FST.BytesReader ReverseReader internal virtual FST.BytesReader GetReverseReader(bool allowSingle) { - if (allowSingle && Blocks.Count == 1) + if (allowSingle && blocks.Count == 1) { - return new ReverseBytesReader(Blocks[0]); + return new ReverseBytesReader(blocks[0]); } return new ReverseBytesReaderAnonymousInner(this); } private class ReverseBytesReaderAnonymousInner : FST.BytesReader { - private readonly BytesStore OuterInstance; + private readonly BytesStore outerInstance; public ReverseBytesReaderAnonymousInner(BytesStore outerInstance) { - this.OuterInstance = outerInstance; - Current = outerInstance.Blocks.Count == 0 ? null : outerInstance.Blocks[0]; + this.outerInstance = outerInstance; + Current = outerInstance.blocks.Count == 0 ? null : outerInstance.blocks[0]; nextBuffer = -1; nextRead = 0; } @@ -533,8 +533,8 @@ public override byte ReadByte() { if (nextRead == -1) { - Current = OuterInstance.Blocks[nextBuffer--]; - nextRead = OuterInstance.BlockSize - 1; + Current = outerInstance.blocks[nextBuffer--]; + nextRead = outerInstance.blockSize - 1; } return Current[nextRead--]; } @@ -556,7 +556,7 @@ public override long Position { get { - return ((long)nextBuffer + 1) * OuterInstance.BlockSize + nextRead; + return ((long)nextBuffer + 1) * outerInstance.blockSize + nextRead; } set { @@ -564,10 +564,10 @@ public override long Position // setPosition(0), the next byte you read is // bytes[0] ... but I would expect bytes[-1] (ie, // EOF)...? - int bufferIndex = (int)(value >> OuterInstance.blockBits); + int bufferIndex = (int)(value >> outerInstance.blockBits); nextBuffer = bufferIndex - 1; - Current = OuterInstance.Blocks[bufferIndex]; - nextRead = (int)(value & OuterInstance.BlockMask); + Current = outerInstance.blocks[bufferIndex]; + nextRead = (int)(value & outerInstance.blockMask); Debug.Assert(this.Position == value, "value=" + value + " this.Position=" + this.Position); } } diff --git a/src/Lucene.Net.Core/Util/Fst/CharSequenceOutputs.cs b/src/Lucene.Net.Core/Util/Fst/CharSequenceOutputs.cs index b541544cc1..fc0b44f363 100644 --- a/src/Lucene.Net.Core/Util/Fst/CharSequenceOutputs.cs +++ b/src/Lucene.Net.Core/Util/Fst/CharSequenceOutputs.cs @@ -33,7 +33,7 @@ namespace Lucene.Net.Util.Fst public sealed class CharSequenceOutputs : Outputs { private static readonly CharsRef NO_OUTPUT = new CharsRef(); - private static readonly CharSequenceOutputs Singleton_Renamed = new CharSequenceOutputs(); + private static readonly CharSequenceOutputs singleton = new CharSequenceOutputs(); private CharSequenceOutputs() { @@ -43,7 +43,7 @@ public static CharSequenceOutputs Singleton { get { - return Singleton_Renamed; + return singleton; } } diff --git a/src/Lucene.Net.Core/Util/Fst/FST.cs b/src/Lucene.Net.Core/Util/Fst/FST.cs index f0d7173a65..335006883c 100644 --- a/src/Lucene.Net.Core/Util/Fst/FST.cs +++ b/src/Lucene.Net.Core/Util/Fst/FST.cs @@ -104,7 +104,7 @@ public enum INPUT_TYPE /// internal const int FIXED_ARRAY_NUM_ARCS_DEEP = 10;*/ - private int[] BytesPerArc = new int[0]; + private int[] bytesPerArc = new int[0]; /*// Increment version to change it private const string FILE_FORMAT_NAME = "FST"; @@ -144,7 +144,7 @@ public enum INPUT_TYPE internal readonly BytesStore Bytes; - private long StartNode = -1; + private long startNode = -1; public readonly Outputs Outputs; @@ -152,7 +152,7 @@ public enum INPUT_TYPE // instead of storing the address of the target node for // a given arc, we mark a single bit noting that the next // node in the byte[] is the target node): - private long LastFrozenNode; + private long lastFrozenNode; private readonly T NO_OUTPUT; @@ -160,31 +160,33 @@ public enum INPUT_TYPE public long arcCount; public long arcWithOutputCount; - private readonly bool Packed; - private PackedInts.Reader NodeRefToAddress; + private readonly bool packed; + private PackedInts.Reader nodeRefToAddress; /// /// If arc has this label then that arc is final/accepted public static readonly int END_LABEL = -1; - private readonly bool AllowArrayArcs; + private readonly bool allowArrayArcs; - private Arc[] CachedRootArcs; - private Arc[] AssertingCachedRootArcs; // only set wit assert + private Arc[] cachedRootArcs; + private Arc[] assertingCachedRootArcs; // only set wit assert + + // LUCENENET NOTE: Arc moved into FST class internal static bool Flag(int flags, int bit) { return (flags & bit) != 0; } - private GrowableWriter NodeAddress; + private GrowableWriter nodeAddress; // TODO: we could be smarter here, and prune periodically // as we go; high in-count nodes will "usually" become // clear early on: - private GrowableWriter InCounts; + private GrowableWriter inCounts; - private readonly int Version; + private readonly int version; // make a new empty FST, for building; Builder invokes // this ctor @@ -192,8 +194,8 @@ internal FST(INPUT_TYPE inputType, Outputs outputs, bool willPackFST, float a { this.inputType = inputType; this.Outputs = outputs; - this.AllowArrayArcs = allowArrayArcs; - Version = VERSION_CURRENT; + this.allowArrayArcs = allowArrayArcs; + version = VERSION_CURRENT; Bytes = new BytesStore(bytesPageBits); // pad: ensure no node gets address 0 which is reserved to mean // the stop state w/ no arcs @@ -201,18 +203,18 @@ internal FST(INPUT_TYPE inputType, Outputs outputs, bool willPackFST, float a NO_OUTPUT = outputs.NoOutput; if (willPackFST) { - NodeAddress = new GrowableWriter(15, 8, acceptableOverheadRatio); - InCounts = new GrowableWriter(1, 8, acceptableOverheadRatio); + nodeAddress = new GrowableWriter(15, 8, acceptableOverheadRatio); + inCounts = new GrowableWriter(1, 8, acceptableOverheadRatio); } else { - NodeAddress = null; - InCounts = null; + nodeAddress = null; + inCounts = null; } emptyOutput = default(T); - Packed = false; - NodeRefToAddress = null; + packed = false; + nodeRefToAddress = null; } public static readonly int DEFAULT_MAX_BLOCK_BITS = Constants.JRE_IS_64BIT ? 30 : 28; @@ -239,8 +241,8 @@ public FST(DataInput @in, Outputs outputs, int maxBlockBits) // NOTE: only reads most recent format; we don't have // back-compat promise for FSTs (they are experimental): - Version = CodecUtil.CheckHeader(@in, FILE_FORMAT_NAME, VERSION_PACKED, VERSION_VINT_TARGET); - Packed = @in.ReadByte() == 1; + version = CodecUtil.CheckHeader(@in, FILE_FORMAT_NAME, VERSION_PACKED, VERSION_VINT_TARGET); + packed = @in.ReadByte() == 1; if (@in.ReadByte() == 1) { // accepts empty string @@ -251,7 +253,7 @@ public FST(DataInput @in, Outputs outputs, int maxBlockBits) // De-serialize empty-string output: BytesReader reader; - if (Packed) + if (packed) { reader = emptyBytes.ForwardReader; } @@ -290,15 +292,15 @@ public FST(DataInput @in, Outputs outputs, int maxBlockBits) default: throw new InvalidOperationException("invalid input type " + t); } - if (Packed) + if (packed) { - NodeRefToAddress = PackedInts.GetReader(@in); + nodeRefToAddress = PackedInts.GetReader(@in); } else { - NodeRefToAddress = null; + nodeRefToAddress = null; } - StartNode = @in.ReadVLong(); + startNode = @in.ReadVLong(); nodeCount = @in.ReadVLong(); arcCount = @in.ReadVLong(); arcWithOutputCount = @in.ReadVLong(); @@ -313,7 +315,7 @@ public FST(DataInput @in, Outputs outputs, int maxBlockBits) // NOTE: bogus because this is only used during // building; we need to break out mutable FST from // immutable - AllowArrayArcs = false; + allowArrayArcs = false; /* if (bytes.length == 665) { @@ -338,21 +340,21 @@ public INPUT_TYPE InputType public long SizeInBytes() { long size = Bytes.Position; - if (Packed) + if (packed) { - size += NodeRefToAddress.RamBytesUsed(); + size += nodeRefToAddress.RamBytesUsed(); } - else if (NodeAddress != null) + else if (nodeAddress != null) { - size += NodeAddress.RamBytesUsed(); - size += InCounts.RamBytesUsed(); + size += nodeAddress.RamBytesUsed(); + size += inCounts.RamBytesUsed(); } return size; } public void Finish(long newStartNode) { - if (StartNode != -1) + if (startNode != -1) { throw new InvalidOperationException("already finished"); } @@ -360,7 +362,7 @@ public void Finish(long newStartNode) { newStartNode = 0; } - StartNode = newStartNode; + startNode = newStartNode; Bytes.Finish(); CacheRootArcs(); @@ -368,10 +370,10 @@ public void Finish(long newStartNode) private long GetNodeAddress(long node) { - if (NodeAddress != null) + if (nodeAddress != null) { // Deref - return NodeAddress.Get((int)node); + return nodeAddress.Get((int)node); } else { @@ -383,10 +385,10 @@ private long GetNodeAddress(long node) // Caches first 128 labels private void CacheRootArcs() { - CachedRootArcs = (Arc[])new Arc[0x80]; - ReadRootArcs(CachedRootArcs); + cachedRootArcs = (Arc[])new Arc[0x80]; + ReadRootArcs(cachedRootArcs); - Debug.Assert(SetAssertingRootArcs(CachedRootArcs)); + Debug.Assert(SetAssertingRootArcs(cachedRootArcs)); Debug.Assert(AssertRootArcs()); } @@ -401,7 +403,7 @@ public void ReadRootArcs(Arc[] arcs) while (true) { Debug.Assert(arc.Label != END_LABEL); - if (arc.Label < CachedRootArcs.Length) + if (arc.Label < cachedRootArcs.Length) { arcs[arc.Label] = (new Arc()).CopyFrom(arc); } @@ -420,19 +422,19 @@ public void ReadRootArcs(Arc[] arcs) private bool SetAssertingRootArcs(Arc[] arcs) { - AssertingCachedRootArcs = (Arc[])new Arc[arcs.Length]; - ReadRootArcs(AssertingCachedRootArcs); + assertingCachedRootArcs = (Arc[])new Arc[arcs.Length]; + ReadRootArcs(assertingCachedRootArcs); return true; } private bool AssertRootArcs() { - Debug.Assert(CachedRootArcs != null); - Debug.Assert(AssertingCachedRootArcs != null); - for (int i = 0; i < CachedRootArcs.Length; i++) + Debug.Assert(cachedRootArcs != null); + Debug.Assert(assertingCachedRootArcs != null); + for (int i = 0; i < cachedRootArcs.Length; i++) { - Arc root = CachedRootArcs[i]; - Arc asserting = AssertingCachedRootArcs[i]; + Arc root = cachedRootArcs[i]; + Arc asserting = assertingCachedRootArcs[i]; if (root != null) { Debug.Assert(root.ArcIdx == asserting.ArcIdx); @@ -491,20 +493,20 @@ public T EmptyOutput public void Save(DataOutput @out) { - if (StartNode == -1) + if (startNode == -1) { throw new InvalidOperationException("call finish first"); } - if (NodeAddress != null) + if (nodeAddress != null) { throw new InvalidOperationException("cannot save an FST pre-packed FST; it must first be packed"); } - if (Packed && !(NodeRefToAddress is PackedInts.Mutable)) + if (packed && !(nodeRefToAddress is PackedInts.Mutable)) { throw new InvalidOperationException("cannot save a FST which has been loaded from disk "); } CodecUtil.WriteHeader(@out, FILE_FORMAT_NAME, VERSION_CURRENT); - if (Packed) + if (packed) { @out.WriteByte(1); } @@ -526,7 +528,7 @@ public void Save(DataOutput @out) var emptyOutputBytes = new byte[(int)ros.FilePointer]; ros.WriteTo(emptyOutputBytes, 0); - if (!Packed) + if (!packed) { // reverse int stopAt = emptyOutputBytes.Length / 2; @@ -560,11 +562,11 @@ public void Save(DataOutput @out) t = 2; } @out.WriteByte((byte)t); - if (Packed) + if (packed) { - ((PackedInts.Mutable)NodeRefToAddress).Save(@out); + ((PackedInts.Mutable)nodeRefToAddress).Save(@out); } - @out.WriteVLong(StartNode); + @out.WriteVLong(startNode); @out.WriteVLong(nodeCount); @out.WriteVLong(arcCount); @out.WriteVLong(arcWithOutputCount); @@ -696,9 +698,9 @@ public long AddNode(Builder.UnCompiledNode nodeIn) if (doFixedArray) { //System.out.println(" fixedArray"); - if (BytesPerArc.Length < nodeIn.NumArcs) + if (bytesPerArc.Length < nodeIn.NumArcs) { - BytesPerArc = new int[ArrayUtil.Oversize(nodeIn.NumArcs, 1)]; + bytesPerArc = new int[ArrayUtil.Oversize(nodeIn.NumArcs, 1)]; } } @@ -720,7 +722,7 @@ public long AddNode(Builder.UnCompiledNode nodeIn) flags += BIT_LAST_ARC; } - if (LastFrozenNode == target.Node && !doFixedArray) + if (lastFrozenNode == target.Node && !doFixedArray) { // TODO: for better perf (but more RAM used) we // could avoid this except when arc is "near" the @@ -747,9 +749,9 @@ public long AddNode(Builder.UnCompiledNode nodeIn) { flags += BIT_STOP_NODE; } - else if (InCounts != null) + else if (inCounts != null) { - InCounts.Set((int)target.Node, InCounts.Get((int)target.Node) + 1); + inCounts.Set((int)target.Node, inCounts.Get((int)target.Node) + 1); } if (!arc.Output.Equals(NO_OUTPUT)) @@ -787,9 +789,9 @@ public long AddNode(Builder.UnCompiledNode nodeIn) // byte size: if (doFixedArray) { - BytesPerArc[arcIdx] = (int)(Bytes.Position - lastArcStart); + bytesPerArc[arcIdx] = (int)(Bytes.Position - lastArcStart); lastArcStart = Bytes.Position; - maxBytesPerArc = Math.Max(maxBytesPerArc, BytesPerArc[arcIdx]); + maxBytesPerArc = Math.Max(maxBytesPerArc, bytesPerArc[arcIdx]); //System.out.println(" bytes=" + bytesPerArc[arcIdx]); } } @@ -844,13 +846,13 @@ public long AddNode(Builder.UnCompiledNode nodeIn) for (int arcIdx = nodeIn.NumArcs - 1; arcIdx >= 0; arcIdx--) { destPos -= maxBytesPerArc; - srcPos -= BytesPerArc[arcIdx]; + srcPos -= bytesPerArc[arcIdx]; //System.out.println(" repack arcIdx=" + arcIdx + " srcPos=" + srcPos + " destPos=" + destPos); if (srcPos != destPos) { //System.out.println(" copy len=" + bytesPerArc[arcIdx]); - Debug.Assert(destPos > srcPos, "destPos=" + destPos + " srcPos=" + srcPos + " arcIdx=" + arcIdx + " maxBytesPerArc=" + maxBytesPerArc + " bytesPerArc[arcIdx]=" + BytesPerArc[arcIdx] + " nodeIn.numArcs=" + nodeIn.NumArcs); - Bytes.CopyBytes(srcPos, destPos, BytesPerArc[arcIdx]); + Debug.Assert(destPos > srcPos, "destPos=" + destPos + " srcPos=" + srcPos + " arcIdx=" + arcIdx + " maxBytesPerArc=" + maxBytesPerArc + " bytesPerArc[arcIdx]=" + bytesPerArc[arcIdx] + " nodeIn.numArcs=" + nodeIn.NumArcs); + Bytes.CopyBytes(srcPos, destPos, bytesPerArc[arcIdx]); } } } @@ -865,22 +867,22 @@ public long AddNode(Builder.UnCompiledNode nodeIn) // PackedInts uses int as the index, so we cannot handle // > 2.1B nodes when packing: - if (NodeAddress != null && nodeCount == int.MaxValue) + if (nodeAddress != null && nodeCount == int.MaxValue) { throw new InvalidOperationException("cannot create a packed FST with more than 2.1 billion nodes"); } nodeCount++; long node; - if (NodeAddress != null) + if (nodeAddress != null) { // Nodes are addressed by 1+ord: - if ((int)nodeCount == NodeAddress.Size()) + if ((int)nodeCount == nodeAddress.Size()) { - NodeAddress = NodeAddress.Resize(ArrayUtil.Oversize(NodeAddress.Size() + 1, NodeAddress.BitsPerValue)); - InCounts = InCounts.Resize(ArrayUtil.Oversize(InCounts.Size() + 1, InCounts.BitsPerValue)); + nodeAddress = nodeAddress.Resize(ArrayUtil.Oversize(nodeAddress.Size() + 1, nodeAddress.BitsPerValue)); + inCounts = inCounts.Resize(ArrayUtil.Oversize(inCounts.Size() + 1, inCounts.BitsPerValue)); } - NodeAddress.Set((int)nodeCount, thisNodeAddress); + nodeAddress.Set((int)nodeCount, thisNodeAddress); // System.out.println(" write nodeAddress[" + nodeCount + "] = " + endAddress); node = nodeCount; } @@ -888,7 +890,7 @@ public long AddNode(Builder.UnCompiledNode nodeIn) { node = thisNodeAddress; } - LastFrozenNode = node; + lastFrozenNode = node; //System.out.println(" ret node=" + node + " address=" + thisNodeAddress + " nodeAddress=" + nodeAddress); return node; @@ -918,7 +920,7 @@ public Arc GetFirstArc(Arc arc) // If there are no nodes, ie, the FST only accepts the // empty string, then startNode is 0 - arc.Target = StartNode; + arc.Target = startNode; return arc; } @@ -951,7 +953,7 @@ public Arc ReadLastTargetArc(Arc follow, Arc arc, FST.BytesReader @in) { // array: jump straight to end arc.NumArcs = @in.ReadVInt(); - if (Packed || Version >= VERSION_VINT_TARGET) + if (packed || version >= VERSION_VINT_TARGET) { arc.BytesPerArc = @in.ReadVInt(); } @@ -987,7 +989,7 @@ public Arc ReadLastTargetArc(Arc follow, Arc arc, FST.BytesReader @in) else if (arc.Flag(BIT_TARGET_NEXT)) { } - else if (Packed) + else if (packed) { @in.ReadVLong(); } @@ -1010,7 +1012,7 @@ public Arc ReadLastTargetArc(Arc follow, Arc arc, FST.BytesReader @in) private long ReadUnpackedNodeTarget(BytesReader @in) { long target; - if (Version < VERSION_VINT_TARGET) + if (version < VERSION_VINT_TARGET) { target = @in.ReadInt(); } @@ -1071,7 +1073,7 @@ public Arc ReadFirstRealTargetArc(long node, Arc arc, FST.BytesReader @in) //System.out.println(" fixedArray"); // this is first arc in a fixed-array arc.NumArcs = @in.ReadVInt(); - if (Packed || Version >= VERSION_VINT_TARGET) + if (packed || version >= VERSION_VINT_TARGET) { arc.BytesPerArc = @in.ReadVInt(); } @@ -1153,7 +1155,7 @@ public int ReadNextArcLabel(Arc arc, BytesReader @in) @in.ReadVInt(); // Skip bytesPerArc: - if (Packed || Version >= VERSION_VINT_TARGET) + if (packed || version >= VERSION_VINT_TARGET) { @in.ReadVInt(); } @@ -1249,7 +1251,7 @@ public Arc ReadNextRealArc(Arc arc, FST.BytesReader @in) arc.NextArc = @in.Position; // TODO: would be nice to make this lazy -- maybe // caller doesn't need the target and is scanning arcs... - if (NodeAddress == null) + if (nodeAddress == null) { if (!arc.Flag(BIT_LAST_ARC)) { @@ -1274,7 +1276,7 @@ public Arc ReadNextRealArc(Arc arc, FST.BytesReader @in) } else { - if (Packed) + if (packed) { long pos = @in.Position; long code = @in.ReadVLong(); @@ -1284,10 +1286,10 @@ public Arc ReadNextRealArc(Arc arc, FST.BytesReader @in) arc.Target = pos + code; //System.out.println(" delta pos=" + pos + " delta=" + code + " target=" + arc.target); } - else if (code < NodeRefToAddress.Size()) + else if (code < nodeRefToAddress.Size()) { // Deref - arc.Target = NodeRefToAddress.Get((int)code); + arc.Target = nodeRefToAddress.Get((int)code); //System.out.println(" deref code=" + code + " target=" + arc.target); } else @@ -1341,12 +1343,12 @@ public Arc FindTargetArc(int labelToMatch, Arc follow, Arc arc, FST.Byt } // Short-circuit if this arc is in the root arc cache: - if (follow.Target == StartNode && labelToMatch < CachedRootArcs.Length) + if (follow.Target == startNode && labelToMatch < cachedRootArcs.Length) { // LUCENE-5152: detect tricky cases where caller // modified previously returned cached root-arcs: Debug.Assert(AssertRootArcs()); - Arc result = CachedRootArcs[labelToMatch]; + Arc result = cachedRootArcs[labelToMatch]; if (result == null) { return null; @@ -1373,7 +1375,7 @@ public Arc FindTargetArc(int labelToMatch, Arc follow, Arc arc, FST.Byt { // Arcs are full array; do binary search: arc.NumArcs = @in.ReadVInt(); - if (Packed || Version >= VERSION_VINT_TARGET) + if (packed || version >= VERSION_VINT_TARGET) { arc.BytesPerArc = @in.ReadVInt(); } @@ -1459,7 +1461,7 @@ private void SeekToNextNode(FST.BytesReader @in) if (!Flag(flags, BIT_STOP_NODE) && !Flag(flags, BIT_TARGET_NEXT)) { - if (Packed) + if (packed) { @in.ReadVLong(); } @@ -1517,7 +1519,7 @@ public long ArcWithOutputCount /// private bool ShouldExpand(Builder.UnCompiledNode node) { - return AllowArrayArcs && ((node.Depth <= FIXED_ARRAY_SHALLOW_DISTANCE && node.NumArcs >= FIXED_ARRAY_NUM_ARCS_SHALLOW) || node.NumArcs >= FIXED_ARRAY_NUM_ARCS_DEEP); + return allowArrayArcs && ((node.Depth <= FIXED_ARRAY_SHALLOW_DISTANCE && node.NumArcs >= FIXED_ARRAY_NUM_ARCS_SHALLOW) || node.NumArcs >= FIXED_ARRAY_NUM_ARCS_DEEP); } /// @@ -1529,7 +1531,7 @@ public FST.BytesReader BytesReader get { FST.BytesReader @in; - if (Packed) + if (packed) { @in = Bytes.ForwardReader; } @@ -1673,8 +1675,8 @@ public void countSingleChains() throws IOException { // Creates a packed FST private FST(INPUT_TYPE inputType, Outputs outputs, int bytesPageBits) { - Version = VERSION_CURRENT; - Packed = true; + version = VERSION_CURRENT; + packed = true; this.inputType = inputType; Bytes = new BytesStore(bytesPageBits); this.Outputs = outputs; @@ -1683,7 +1685,7 @@ private FST(INPUT_TYPE inputType, Outputs outputs, int bytesPageBits) // NOTE: bogus because this is only used during // building; we need to break out mutable FST from // immutable - AllowArrayArcs = false; + allowArrayArcs = false; } /// @@ -1715,7 +1717,7 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve // - use spare bits in flags.... for top few labels / // outputs / targets - if (NodeAddress == null) + if (nodeAddress == null) { throw new System.ArgumentException("this FST was not built with willPackFST=true"); } @@ -1724,34 +1726,34 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve BytesReader r = BytesReader; - int topN = Math.Min(maxDerefNodes, InCounts.Size()); + int topN = Math.Min(maxDerefNodes, inCounts.Size()); // Find top nodes with highest number of incoming arcs: NodeQueue q = new NodeQueue(topN); // TODO: we could use more RAM efficient selection algo here... NodeAndInCount bottom = null; - for (int node = 0; node < InCounts.Size(); node++) + for (int node = 0; node < inCounts.Size(); node++) { - if (InCounts.Get(node) >= minInCountDeref) + if (inCounts.Get(node) >= minInCountDeref) { if (bottom == null) { - q.Add(new NodeAndInCount(node, (int)InCounts.Get(node))); + q.Add(new NodeAndInCount(node, (int)inCounts.Get(node))); if (q.Size() == topN) { bottom = q.Top(); } } - else if (InCounts.Get(node) > bottom.Count) + else if (inCounts.Get(node) > bottom.Count) { - q.InsertWithOverflow(new NodeAndInCount(node, (int)InCounts.Get(node))); + q.InsertWithOverflow(new NodeAndInCount(node, (int)inCounts.Get(node))); } } } // Free up RAM: - InCounts = null; + inCounts = null; IDictionary topNodeMap = new Dictionary(); for (int downTo = q.Size() - 1; downTo >= 0; downTo--) @@ -1767,7 +1769,7 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve // Fill initial coarse guess: for (int node = 1; node <= nodeCount; node++) { - newNodeAddress.Set(node, 1 + this.Bytes.Position - NodeAddress.Get(node)); + newNodeAddress.Set(node, 1 + this.Bytes.Position - nodeAddress.Get(node)); } int absCount; @@ -2075,9 +2077,9 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve { nodeRefToAddressIn.Set(ent.Value, newNodeAddress.Get(ent.Key)); } - fst.NodeRefToAddress = nodeRefToAddressIn; + fst.nodeRefToAddress = nodeRefToAddressIn; - fst.StartNode = newNodeAddress.Get((int)StartNode); + fst.startNode = newNodeAddress.Get((int)startNode); //System.out.println("new startNode=" + fst.startNode + " old startNode=" + startNode); if (emptyOutput != null) diff --git a/src/Lucene.Net.Core/Util/Fst/FSTEnum.cs b/src/Lucene.Net.Core/Util/Fst/FSTEnum.cs index 3e19974f15..dfdaf67bef 100644 --- a/src/Lucene.Net.Core/Util/Fst/FSTEnum.cs +++ b/src/Lucene.Net.Core/Util/Fst/FSTEnum.cs @@ -28,19 +28,19 @@ namespace Lucene.Net.Util.Fst public abstract class FSTEnum { - protected internal readonly FST Fst; + protected internal readonly FST fst; - protected internal FST.Arc[] Arcs = new FST.Arc[10]; + protected internal FST.Arc[] arcs = new FST.Arc[10]; // outputs are cumulative - protected internal T[] Output = new T[10]; + protected internal T[] output = new T[10]; protected internal readonly T NO_OUTPUT; - protected internal readonly FST.BytesReader FstReader; - protected internal readonly FST.Arc ScratchArc = new FST.Arc(); + protected internal readonly FST.BytesReader fstReader; + protected internal readonly FST.Arc scratchArc = new FST.Arc(); - protected internal int Upto; - protected internal int TargetLength; + protected internal int upto; + protected internal int targetLength; /// /// doFloor controls the behavior of advance: if it's true @@ -49,11 +49,11 @@ public abstract class FSTEnum /// protected internal FSTEnum(FST fst) { - this.Fst = fst; - FstReader = fst.BytesReader; + this.fst = fst; + fstReader = fst.BytesReader; NO_OUTPUT = fst.Outputs.NoOutput; fst.GetFirstArc(GetArc(0)); - Output[0] = NO_OUTPUT; + output[0] = NO_OUTPUT; } protected internal abstract int TargetLabel { get; } @@ -68,18 +68,18 @@ protected internal FSTEnum(FST fst) /// protected internal void RewindPrefix() { - if (Upto == 0) + if (upto == 0) { //System.out.println(" init"); - Upto = 1; - Fst.ReadFirstTargetArc(GetArc(0), GetArc(1), FstReader); + upto = 1; + fst.ReadFirstTargetArc(GetArc(0), GetArc(1), fstReader); return; } //System.out.println(" rewind upto=" + upto + " vs targetLength=" + targetLength); - int currentLimit = Upto; - Upto = 1; - while (Upto < currentLimit && Upto <= TargetLength + 1) + int currentLimit = upto; + upto = 1; + while (upto < currentLimit && upto <= targetLength + 1) { int cmp = CurrentLabel - TargetLabel; if (cmp < 0) @@ -91,12 +91,12 @@ protected internal void RewindPrefix() else if (cmp > 0) { // seek backwards -- reset this arc to the first arc - FST.Arc arc = GetArc(Upto); - Fst.ReadFirstTargetArc(GetArc(Upto - 1), arc, FstReader); + FST.Arc arc = GetArc(upto); + fst.ReadFirstTargetArc(GetArc(upto - 1), arc, fstReader); //System.out.println(" seek first arc"); break; } - Upto++; + upto++; } //System.out.println(" fall through upto=" + upto); } @@ -104,26 +104,26 @@ protected internal void RewindPrefix() protected internal virtual void DoNext() { //System.out.println("FE: next upto=" + upto); - if (Upto == 0) + if (upto == 0) { //System.out.println(" init"); - Upto = 1; - Fst.ReadFirstTargetArc(GetArc(0), GetArc(1), FstReader); + upto = 1; + fst.ReadFirstTargetArc(GetArc(0), GetArc(1), fstReader); } else { // pop //System.out.println(" check pop curArc target=" + arcs[upto].target + " label=" + arcs[upto].label + " isLast?=" + arcs[upto].isLast()); - while (Arcs[Upto].IsLast) + while (arcs[upto].IsLast) { - Upto--; - if (Upto == 0) + upto--; + if (upto == 0) { //System.out.println(" eof"); return; } } - Fst.ReadNextArc(Arcs[Upto], FstReader); + fst.ReadNextArc(arcs[upto], fstReader); } PushFirst(); @@ -150,7 +150,7 @@ protected internal virtual void DoSeekCeil() RewindPrefix(); //System.out.println(" after rewind upto=" + upto); - FST.Arc arc = GetArc(Upto); + FST.Arc arc = GetArc(upto); int targetLabel = TargetLabel; //System.out.println(" init targetLabel=" + targetLabel); @@ -164,7 +164,7 @@ protected internal virtual void DoSeekCeil() // Arcs are fixed array -- use binary search to find // the target. - FST.BytesReader @in = Fst.BytesReader; + FST.BytesReader @in = fst.BytesReader; int low = arc.ArcIdx; int high = arc.NumArcs - 1; int mid = 0; @@ -175,7 +175,7 @@ protected internal virtual void DoSeekCeil() mid = (int)((uint)(low + high) >> 1); @in.Position = arc.PosArcsStart; @in.SkipBytes(arc.BytesPerArc * mid + 1); - int midLabel = Fst.ReadLabel(@in); + int midLabel = fst.ReadLabel(@in); int cmp = midLabel - targetLabel; //System.out.println(" cycle low=" + low + " high=" + high + " mid=" + mid + " midLabel=" + midLabel + " cmp=" + cmp); if (cmp < 0) @@ -199,17 +199,17 @@ protected internal virtual void DoSeekCeil() { // Match arc.ArcIdx = mid - 1; - Fst.ReadNextRealArc(arc, @in); + fst.ReadNextRealArc(arc, @in); Debug.Assert(arc.ArcIdx == mid); Debug.Assert(arc.Label == targetLabel, "arc.label=" + arc.Label + " vs targetLabel=" + targetLabel + " mid=" + mid); - Output[Upto] = Fst.Outputs.Add(Output[Upto - 1], arc.Output); + output[upto] = fst.Outputs.Add(output[upto - 1], arc.Output); if (targetLabel == FST.END_LABEL) { return; } CurrentLabel = arc.Label; Incr(); - arc = Fst.ReadFirstTargetArc(arc, GetArc(Upto), FstReader); + arc = fst.ReadFirstTargetArc(arc, GetArc(upto), fstReader); targetLabel = TargetLabel; continue; } @@ -217,32 +217,32 @@ protected internal virtual void DoSeekCeil() { // Dead end arc.ArcIdx = arc.NumArcs - 2; - Fst.ReadNextRealArc(arc, @in); + fst.ReadNextRealArc(arc, @in); Debug.Assert(arc.IsLast); // Dead end (target is after the last arc); // rollback to last fork then push - Upto--; + upto--; while (true) { - if (Upto == 0) + if (upto == 0) { return; } - FST.Arc prevArc = GetArc(Upto); + FST.Arc prevArc = GetArc(upto); //System.out.println(" rollback upto=" + upto + " arc.label=" + prevArc.label + " isLast?=" + prevArc.isLast()); if (!prevArc.IsLast) { - Fst.ReadNextArc(prevArc, FstReader); + fst.ReadNextArc(prevArc, fstReader); PushFirst(); return; } - Upto--; + upto--; } } else { arc.ArcIdx = (low > high ? low : high) - 1; - Fst.ReadNextRealArc(arc, @in); + fst.ReadNextRealArc(arc, @in); Debug.Assert(arc.Label > targetLabel); PushFirst(); return; @@ -254,14 +254,14 @@ protected internal virtual void DoSeekCeil() if (arc.Label == targetLabel) { // recurse - Output[Upto] = Fst.Outputs.Add(Output[Upto - 1], arc.Output); + output[upto] = fst.Outputs.Add(output[upto - 1], arc.Output); if (targetLabel == FST.END_LABEL) { return; } CurrentLabel = arc.Label; Incr(); - arc = Fst.ReadFirstTargetArc(arc, GetArc(Upto), FstReader); + arc = fst.ReadFirstTargetArc(arc, GetArc(upto), fstReader); targetLabel = TargetLabel; } else if (arc.Label > targetLabel) @@ -273,29 +273,29 @@ protected internal virtual void DoSeekCeil() { // Dead end (target is after the last arc); // rollback to last fork then push - Upto--; + upto--; while (true) { - if (Upto == 0) + if (upto == 0) { return; } - FST.Arc prevArc = GetArc(Upto); + FST.Arc prevArc = GetArc(upto); //System.out.println(" rollback upto=" + upto + " arc.label=" + prevArc.label + " isLast?=" + prevArc.isLast()); if (!prevArc.IsLast) { - Fst.ReadNextArc(prevArc, FstReader); + fst.ReadNextArc(prevArc, fstReader); PushFirst(); return; } - Upto--; + upto--; } } else { // keep scanning //System.out.println(" next scan"); - Fst.ReadNextArc(arc, FstReader); + fst.ReadNextArc(arc, fstReader); } } } @@ -319,7 +319,7 @@ protected internal virtual void DoSeekFloor() //System.out.println("FE: after rewind upto=" + upto); - FST.Arc arc = GetArc(Upto); + FST.Arc arc = GetArc(upto); int targetLabel = TargetLabel; //System.out.println("FE: init targetLabel=" + targetLabel); @@ -334,7 +334,7 @@ protected internal virtual void DoSeekFloor() // Arcs are fixed array -- use binary search to find // the target. - FST.BytesReader @in = Fst.BytesReader; + FST.BytesReader @in = fst.BytesReader; int low = arc.ArcIdx; int high = arc.NumArcs - 1; int mid = 0; @@ -345,7 +345,7 @@ protected internal virtual void DoSeekFloor() mid = (int)((uint)(low + high) >> 1); @in.Position = arc.PosArcsStart; @in.SkipBytes(arc.BytesPerArc * mid + 1); - int midLabel = Fst.ReadLabel(@in); + int midLabel = fst.ReadLabel(@in); int cmp = midLabel - targetLabel; //System.out.println(" cycle low=" + low + " high=" + high + " mid=" + mid + " midLabel=" + midLabel + " cmp=" + cmp); if (cmp < 0) @@ -370,17 +370,17 @@ protected internal virtual void DoSeekFloor() // Match -- recurse //System.out.println(" match! arcIdx=" + mid); arc.ArcIdx = mid - 1; - Fst.ReadNextRealArc(arc, @in); + fst.ReadNextRealArc(arc, @in); Debug.Assert(arc.ArcIdx == mid); Debug.Assert(arc.Label == targetLabel, "arc.label=" + arc.Label + " vs targetLabel=" + targetLabel + " mid=" + mid); - Output[Upto] = Fst.Outputs.Add(Output[Upto - 1], arc.Output); + output[upto] = fst.Outputs.Add(output[upto - 1], arc.Output); if (targetLabel == FST.END_LABEL) { return; } CurrentLabel = arc.Label; Incr(); - arc = Fst.ReadFirstTargetArc(arc, GetArc(Upto), FstReader); + arc = fst.ReadFirstTargetArc(arc, GetArc(upto), fstReader); targetLabel = TargetLabel; continue; } @@ -396,25 +396,25 @@ protected internal virtual void DoSeekFloor() { // First, walk backwards until we find a first arc // that's before our target label: - Fst.ReadFirstTargetArc(GetArc(Upto - 1), arc, FstReader); + fst.ReadFirstTargetArc(GetArc(upto - 1), arc, fstReader); if (arc.Label < targetLabel) { // Then, scan forwards to the arc just before // the targetLabel: - while (!arc.IsLast && Fst.ReadNextArcLabel(arc, @in) < targetLabel) + while (!arc.IsLast && fst.ReadNextArcLabel(arc, @in) < targetLabel) { - Fst.ReadNextArc(arc, FstReader); + fst.ReadNextArc(arc, fstReader); } PushLast(); return; } - Upto--; - if (Upto == 0) + upto--; + if (upto == 0) { return; } targetLabel = TargetLabel; - arc = GetArc(Upto); + arc = GetArc(upto); } } else @@ -422,8 +422,8 @@ protected internal virtual void DoSeekFloor() // There is a floor arc: arc.ArcIdx = (low > high ? high : low) - 1; //System.out.println(" hasFloor arcIdx=" + (arc.arcIdx+1)); - Fst.ReadNextRealArc(arc, @in); - Debug.Assert(arc.IsLast || Fst.ReadNextArcLabel(arc, @in) > targetLabel); + fst.ReadNextRealArc(arc, @in); + Debug.Assert(arc.IsLast || fst.ReadNextArcLabel(arc, @in) > targetLabel); Debug.Assert(arc.Label < targetLabel, "arc.label=" + arc.Label + " vs targetLabel=" + targetLabel); PushLast(); return; @@ -434,14 +434,14 @@ protected internal virtual void DoSeekFloor() if (arc.Label == targetLabel) { // Match -- recurse - Output[Upto] = Fst.Outputs.Add(Output[Upto - 1], arc.Output); + output[upto] = fst.Outputs.Add(output[upto - 1], arc.Output); if (targetLabel == FST.END_LABEL) { return; } CurrentLabel = arc.Label; Incr(); - arc = Fst.ReadFirstTargetArc(arc, GetArc(Upto), FstReader); + arc = fst.ReadFirstTargetArc(arc, GetArc(upto), fstReader); targetLabel = TargetLabel; } else if (arc.Label > targetLabel) @@ -454,31 +454,31 @@ protected internal virtual void DoSeekFloor() { // First, walk backwards until we find a first arc // that's before our target label: - Fst.ReadFirstTargetArc(GetArc(Upto - 1), arc, FstReader); + fst.ReadFirstTargetArc(GetArc(upto - 1), arc, fstReader); if (arc.Label < targetLabel) { // Then, scan forwards to the arc just before // the targetLabel: - while (!arc.IsLast && Fst.ReadNextArcLabel(arc, FstReader) < targetLabel) + while (!arc.IsLast && fst.ReadNextArcLabel(arc, fstReader) < targetLabel) { - Fst.ReadNextArc(arc, FstReader); + fst.ReadNextArc(arc, fstReader); } PushLast(); return; } - Upto--; - if (Upto == 0) + upto--; + if (upto == 0) { return; } targetLabel = TargetLabel; - arc = GetArc(Upto); + arc = GetArc(upto); } } else if (!arc.IsLast) { //System.out.println(" check next label=" + fst.readNextArcLabel(arc) + " (" + (char) fst.readNextArcLabel(arc) + ")"); - if (Fst.ReadNextArcLabel(arc, FstReader) > targetLabel) + if (fst.ReadNextArcLabel(arc, fstReader) > targetLabel) { PushLast(); return; @@ -486,7 +486,7 @@ protected internal virtual void DoSeekFloor() else { // keep scanning - Fst.ReadNextArc(arc, FstReader); + fst.ReadNextArc(arc, fstReader); } } else @@ -514,26 +514,26 @@ protected internal virtual bool DoSeekExact() RewindPrefix(); //System.out.println("FE: after rewind upto=" + upto); - FST.Arc arc = GetArc(Upto - 1); + FST.Arc arc = GetArc(upto - 1); int targetLabel = TargetLabel; - FST.BytesReader fstReader = Fst.BytesReader; + FST.BytesReader fstReader = fst.BytesReader; while (true) { //System.out.println(" cycle target=" + (targetLabel == -1 ? "-1" : (char) targetLabel)); - FST.Arc nextArc = Fst.FindTargetArc(targetLabel, arc, GetArc(Upto), fstReader); + FST.Arc nextArc = fst.FindTargetArc(targetLabel, arc, GetArc(upto), fstReader); if (nextArc == null) { // short circuit //upto--; //upto = 0; - Fst.ReadFirstTargetArc(arc, GetArc(Upto), fstReader); + fst.ReadFirstTargetArc(arc, GetArc(upto), fstReader); //System.out.println(" no match upto=" + upto); return false; } // Match -- recurse: - Output[Upto] = Fst.Outputs.Add(Output[Upto - 1], nextArc.Output); + output[upto] = fst.Outputs.Add(output[upto - 1], nextArc.Output); if (targetLabel == FST.END_LABEL) { //System.out.println(" return found; upto=" + upto + " output=" + output[upto] + " nextArc=" + nextArc.isLast()); @@ -548,19 +548,19 @@ protected internal virtual bool DoSeekExact() private void Incr() { - Upto++; + upto++; Grow(); - if (Arcs.Length <= Upto) + if (arcs.Length <= upto) { - FST.Arc[] newArcs = new FST.Arc[ArrayUtil.Oversize(1 + Upto, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; - Array.Copy(Arcs, 0, newArcs, 0, Arcs.Length); - Arcs = newArcs; + FST.Arc[] newArcs = new FST.Arc[ArrayUtil.Oversize(1 + upto, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; + Array.Copy(arcs, 0, newArcs, 0, arcs.Length); + arcs = newArcs; } - if (Output.Length <= Upto) + if (output.Length <= upto) { - T[] newOutput = new T[ArrayUtil.Oversize(1 + Upto, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; - Array.Copy(Output, 0, newOutput, 0, Output.Length); - Output = newOutput; + T[] newOutput = new T[ArrayUtil.Oversize(1 + upto, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; + Array.Copy(output, 0, newOutput, 0, output.Length); + output = newOutput; } } @@ -568,12 +568,12 @@ private void Incr() // appending first arc all the way to the final node private void PushFirst() { - FST.Arc arc = Arcs[Upto]; + FST.Arc arc = arcs[upto]; Debug.Assert(arc != null); while (true) { - Output[Upto] = Fst.Outputs.Add(Output[Upto - 1], arc.Output); + output[upto] = fst.Outputs.Add(output[upto - 1], arc.Output); if (arc.Label == FST.END_LABEL) { // Final node @@ -583,8 +583,8 @@ private void PushFirst() CurrentLabel = arc.Label; Incr(); - FST.Arc nextArc = GetArc(Upto); - Fst.ReadFirstTargetArc(arc, nextArc, FstReader); + FST.Arc nextArc = GetArc(upto); + fst.ReadFirstTargetArc(arc, nextArc, fstReader); arc = nextArc; } } @@ -593,13 +593,13 @@ private void PushFirst() // way to the first final node private void PushLast() { - FST.Arc arc = Arcs[Upto]; + FST.Arc arc = arcs[upto]; Debug.Assert(arc != null); while (true) { CurrentLabel = arc.Label; - Output[Upto] = Fst.Outputs.Add(Output[Upto - 1], arc.Output); + output[upto] = fst.Outputs.Add(output[upto - 1], arc.Output); if (arc.Label == FST.END_LABEL) { // Final node @@ -607,17 +607,17 @@ private void PushLast() } Incr(); - arc = Fst.ReadLastTargetArc(arc, GetArc(Upto), FstReader); + arc = fst.ReadLastTargetArc(arc, GetArc(upto), fstReader); } } private FST.Arc GetArc(int idx) { - if (Arcs[idx] == null) + if (arcs[idx] == null) { - Arcs[idx] = new FST.Arc(); + arcs[idx] = new FST.Arc(); } - return Arcs[idx]; + return arcs[idx]; } } } \ No newline at end of file diff --git a/src/Lucene.Net.Core/Util/Fst/ForwardBytesReader.cs b/src/Lucene.Net.Core/Util/Fst/ForwardBytesReader.cs index 039e386803..694f43876a 100644 --- a/src/Lucene.Net.Core/Util/Fst/ForwardBytesReader.cs +++ b/src/Lucene.Net.Core/Util/Fst/ForwardBytesReader.cs @@ -26,39 +26,39 @@ namespace Lucene.Net.Util.Fst /// Reads from a single byte[]. internal sealed class ForwardBytesReader : FST.BytesReader { - private readonly byte[] Bytes; - private int Pos; + private readonly byte[] bytes; + private int pos; public ForwardBytesReader(byte[] bytes) { - this.Bytes = bytes; + this.bytes = bytes; } public override byte ReadByte() { - return Bytes[Pos++]; + return bytes[pos++]; } public override void ReadBytes(byte[] b, int offset, int len) { - Array.Copy(Bytes, Pos, b, offset, len); - Pos += len; + Array.Copy(bytes, pos, b, offset, len); + pos += len; } public override void SkipBytes(int count) { - Pos += count; + pos += count; } public override long Position { get { - return Pos; + return pos; } set { - this.Pos = (int)value; + this.pos = (int)value; } } diff --git a/src/Lucene.Net.Core/Util/Fst/IntSequenceOutputs.cs b/src/Lucene.Net.Core/Util/Fst/IntSequenceOutputs.cs index 8b8294c1cc..6146420a3d 100644 --- a/src/Lucene.Net.Core/Util/Fst/IntSequenceOutputs.cs +++ b/src/Lucene.Net.Core/Util/Fst/IntSequenceOutputs.cs @@ -33,7 +33,7 @@ namespace Lucene.Net.Util.Fst public sealed class IntSequenceOutputs : Outputs { private static readonly IntsRef NO_OUTPUT = new IntsRef(); - private static readonly IntSequenceOutputs Singleton_Renamed = new IntSequenceOutputs(); + private static readonly IntSequenceOutputs singleton = new IntSequenceOutputs(); private IntSequenceOutputs() { @@ -43,7 +43,7 @@ public static IntSequenceOutputs Singleton { get { - return Singleton_Renamed; + return singleton; } } diff --git a/src/Lucene.Net.Core/Util/Fst/IntsRefFSTEnum.cs b/src/Lucene.Net.Core/Util/Fst/IntsRefFSTEnum.cs index 4c71caa06a..b70d8c4797 100644 --- a/src/Lucene.Net.Core/Util/Fst/IntsRefFSTEnum.cs +++ b/src/Lucene.Net.Core/Util/Fst/IntsRefFSTEnum.cs @@ -28,9 +28,9 @@ namespace Lucene.Net.Util.Fst public sealed class IntsRefFSTEnum : FSTEnum { - private readonly IntsRef Current_Renamed = new IntsRef(10); - private readonly InputOutput Result = new InputOutput(); - private IntsRef Target; + private readonly IntsRef current = new IntsRef(10); + private readonly InputOutput result = new InputOutput(); + private IntsRef target; /// /// Holds a single input (IntsRef) + output pair. @@ -48,13 +48,13 @@ public class InputOutput public IntsRefFSTEnum(FST fst) : base(fst) { - Result.Input = Current_Renamed; - Current_Renamed.Offset = 1; + result.Input = current; + current.Offset = 1; } public InputOutput Current() { - return Result; + return result; } public InputOutput Next() @@ -68,8 +68,8 @@ public InputOutput Next() /// Seeks to smallest term that's >= target. public InputOutput SeekCeil(IntsRef target) { - this.Target = target; - TargetLength = target.Length; + this.target = target; + targetLength = target.Length; base.DoSeekCeil(); return SetResult(); } @@ -78,8 +78,8 @@ public InputOutput SeekCeil(IntsRef target) /// Seeks to biggest term that's <= target. public InputOutput SeekFloor(IntsRef target) { - this.Target = target; - TargetLength = target.Length; + this.target = target; + targetLength = target.Length; base.DoSeekFloor(); return SetResult(); } @@ -92,11 +92,11 @@ public InputOutput SeekFloor(IntsRef target) /// public InputOutput SeekExact(IntsRef target) { - this.Target = target; - TargetLength = target.Length; + this.target = target; + targetLength = target.Length; if (base.DoSeekExact()) { - Debug.Assert(Upto == 1 + target.Length); + Debug.Assert(upto == 1 + target.Length); return SetResult(); } else @@ -109,13 +109,13 @@ protected internal override int TargetLabel { get { - if (Upto - 1 == Target.Length) + if (upto - 1 == target.Length) { return FST.END_LABEL; } else { - return Target.Ints[Target.Offset + Upto - 1]; + return target.Ints[target.Offset + upto - 1]; } } } @@ -125,30 +125,30 @@ protected internal override int CurrentLabel get { // current.offset fixed at 1 - return Current_Renamed.Ints[Upto]; + return current.Ints[upto]; } set { - Current_Renamed.Ints[Upto] = value; + current.Ints[upto] = value; } } protected internal override void Grow() { - Current_Renamed.Ints = ArrayUtil.Grow(Current_Renamed.Ints, Upto + 1); + current.Ints = ArrayUtil.Grow(current.Ints, upto + 1); } private InputOutput SetResult() { - if (Upto == 0) + if (upto == 0) { return null; } else { - Current_Renamed.Length = Upto - 1; - Result.Output = Output[Upto]; - return Result; + current.Length = upto - 1; + result.Output = output[upto]; + return result; } } } diff --git a/src/Lucene.Net.Core/Util/Fst/NoOutputs.cs b/src/Lucene.Net.Core/Util/Fst/NoOutputs.cs index 248dc84cf2..77012fa0f7 100644 --- a/src/Lucene.Net.Core/Util/Fst/NoOutputs.cs +++ b/src/Lucene.Net.Core/Util/Fst/NoOutputs.cs @@ -52,7 +52,7 @@ public override bool Equals(object other) } } - private static readonly NoOutputs Singleton_Renamed = new NoOutputs(); + private static readonly NoOutputs singleton = new NoOutputs(); private NoOutputs() { @@ -62,7 +62,7 @@ public static NoOutputs Singleton { get { - return Singleton_Renamed; + return singleton; } } diff --git a/src/Lucene.Net.Core/Util/Fst/NodeHash.cs b/src/Lucene.Net.Core/Util/Fst/NodeHash.cs index ca92deb7dc..fcc9429e02 100644 --- a/src/Lucene.Net.Core/Util/Fst/NodeHash.cs +++ b/src/Lucene.Net.Core/Util/Fst/NodeHash.cs @@ -26,37 +26,37 @@ namespace Lucene.Net.Util.Fst // Used to dedup states (lookup already-frozen states) internal sealed class NodeHash { - private PagedGrowableWriter Table; - private long Count; - private long Mask; - private readonly FST Fst; - private readonly FST.Arc ScratchArc = new FST.Arc(); - private readonly FST.BytesReader @in; - - public NodeHash(FST fst, FST.BytesReader @in) + private PagedGrowableWriter table; + private long count; + private long mask; + private readonly FST fst; + private readonly FST.Arc scratchArc = new FST.Arc(); + private readonly FST.BytesReader input; + + public NodeHash(FST fst, FST.BytesReader input) { - Table = new PagedGrowableWriter(16, 1 << 30, 8, PackedInts.COMPACT); - Mask = 15; - this.Fst = fst; - this.@in = @in; + table = new PagedGrowableWriter(16, 1 << 30, 8, PackedInts.COMPACT); + mask = 15; + this.fst = fst; + this.input = input; } private bool NodesEqual(Builder.UnCompiledNode node, long address) { - Fst.ReadFirstRealTargetArc(address, ScratchArc, @in); - if (ScratchArc.BytesPerArc != 0 && node.NumArcs != ScratchArc.NumArcs) + fst.ReadFirstRealTargetArc(address, scratchArc, input); + if (scratchArc.BytesPerArc != 0 && node.NumArcs != scratchArc.NumArcs) { return false; } for (int arcUpto = 0; arcUpto < node.NumArcs; arcUpto++) { Builder.Arc arc = node.Arcs[arcUpto]; - if (arc.Label != ScratchArc.Label || !arc.Output.Equals(ScratchArc.Output) || ((Builder.CompiledNode)arc.Target).Node != ScratchArc.Target || !arc.NextFinalOutput.Equals(ScratchArc.NextFinalOutput) || arc.IsFinal != ScratchArc.IsFinal) + if (arc.Label != scratchArc.Label || !arc.Output.Equals(scratchArc.Output) || ((Builder.CompiledNode)arc.Target).Node != scratchArc.Target || !arc.NextFinalOutput.Equals(scratchArc.NextFinalOutput) || arc.IsFinal != scratchArc.IsFinal) { return false; } - if (ScratchArc.IsLast) + if (scratchArc.IsLast) { if (arcUpto == node.NumArcs - 1) { @@ -67,7 +67,7 @@ private bool NodesEqual(Builder.UnCompiledNode node, long address) return false; } } - Fst.ReadNextRealArc(ScratchArc, @in); + fst.ReadNextRealArc(scratchArc, input); } return false; @@ -126,21 +126,21 @@ private long Hash(long node) const int PRIME = 31; //System.out.println("hash frozen node=" + node); long h = 0; - Fst.ReadFirstRealTargetArc(node, ScratchArc, @in); + fst.ReadFirstRealTargetArc(node, scratchArc, input); while (true) { //System.out.println(" label=" + scratchArc.label + " target=" + scratchArc.target + " h=" + h + " output=" + fst.outputs.outputToString(scratchArc.output) + " next?=" + scratchArc.flag(4) + " final?=" + scratchArc.isFinal() + " pos=" + in.getPosition()); - h = PRIME * h + ScratchArc.Label; - h = PRIME * h + (int)(ScratchArc.Target ^ (ScratchArc.Target >> 32)); - h = PRIME * h + ScratchArc.Output.GetHashCode(); + h = PRIME * h + scratchArc.Label; + h = PRIME * h + (int)(scratchArc.Target ^ (scratchArc.Target >> 32)); + h = PRIME * h + scratchArc.Output.GetHashCode(); // LUCENENET: Since lists do not compare values by default in .NET, // we need this workaround to get the hashcode of the type + all of the // values. - if (ScratchArc.NextFinalOutput is IEnumerable) + if (scratchArc.NextFinalOutput is IEnumerable) { - h = PRIME * h + ScratchArc.NextFinalOutput.GetType().GetHashCode(); - foreach (object value in ScratchArc.NextFinalOutput as IEnumerable) + h = PRIME * h + scratchArc.NextFinalOutput.GetType().GetHashCode(); + foreach (object value in scratchArc.NextFinalOutput as IEnumerable) { if (value != null) { @@ -154,18 +154,18 @@ private long Hash(long node) } else { - h = PRIME * h + ScratchArc.NextFinalOutput.GetHashCode(); + h = PRIME * h + scratchArc.NextFinalOutput.GetHashCode(); } - if (ScratchArc.IsFinal) + if (scratchArc.IsFinal) { h += 17; } - if (ScratchArc.IsLast) + if (scratchArc.IsLast) { break; } - Fst.ReadNextRealArc(ScratchArc, @in); + fst.ReadNextRealArc(scratchArc, input); } //System.out.println(" ret " + (h&Integer.MAX_VALUE)); return h & long.MaxValue; @@ -175,22 +175,22 @@ public long Add(Builder.UnCompiledNode nodeIn) { //System.out.println("hash: add count=" + count + " vs " + table.size() + " mask=" + mask); long h = Hash(nodeIn); - long pos = h & Mask; + long pos = h & mask; int c = 0; while (true) { - long v = Table.Get(pos); + long v = table.Get(pos); if (v == 0) { // freeze & add - long node = Fst.AddNode(nodeIn); + long node = fst.AddNode(nodeIn); //System.out.println(" now freeze node=" + node); long hashNode = Hash(node); Debug.Assert(hashNode == h, "frozenHash=" + hashNode + " vs h=" + h); - Count++; - Table.Set(pos, node); + count++; + table.Set(pos, node); // Rehash at 2/3 occupancy: - if (Count > 2 * Table.Size() / 3) + if (count > 2 * table.Size() / 3) { Rehash(); } @@ -203,34 +203,34 @@ public long Add(Builder.UnCompiledNode nodeIn) } // quadratic probe - pos = (pos + (++c)) & Mask; + pos = (pos + (++c)) & mask; } } // called only by rehash private void AddNew(long address) { - long pos = Hash(address) & Mask; + long pos = Hash(address) & mask; int c = 0; while (true) { - if (Table.Get(pos) == 0) + if (table.Get(pos) == 0) { - Table.Set(pos, address); + table.Set(pos, address); break; } // quadratic probe - pos = (pos + (++c)) & Mask; + pos = (pos + (++c)) & mask; } } private void Rehash() { - PagedGrowableWriter oldTable = Table; + PagedGrowableWriter oldTable = table; - Table = new PagedGrowableWriter(2 * oldTable.Size(), 1 << 30, PackedInts.BitsRequired(Count), PackedInts.COMPACT); - Mask = Table.Size() - 1; + table = new PagedGrowableWriter(2 * oldTable.Size(), 1 << 30, PackedInts.BitsRequired(count), PackedInts.COMPACT); + mask = table.Size() - 1; for (long idx = 0; idx < oldTable.Size(); idx++) { long address = oldTable.Get(idx); diff --git a/src/Lucene.Net.Core/Util/Fst/PairOutputs.cs b/src/Lucene.Net.Core/Util/Fst/PairOutputs.cs index 2e584b50d8..b5d626f362 100644 --- a/src/Lucene.Net.Core/Util/Fst/PairOutputs.cs +++ b/src/Lucene.Net.Core/Util/Fst/PairOutputs.cs @@ -31,8 +31,8 @@ namespace Lucene.Net.Util.Fst public class PairOutputs : Outputs.Pair> { private readonly Pair NO_OUTPUT; - private readonly Outputs Outputs1; - private readonly Outputs Outputs2; + private readonly Outputs outputs1; + private readonly Outputs outputs2; /// /// Holds a single pair of two outputs. @@ -73,8 +73,8 @@ public override int GetHashCode() public PairOutputs(Outputs outputs1, Outputs outputs2) { - this.Outputs1 = outputs1; - this.Outputs2 = outputs2; + this.outputs1 = outputs1; + this.outputs2 = outputs2; NO_OUTPUT = new Pair(outputs1.NoOutput, outputs2.NoOutput); } @@ -82,16 +82,16 @@ public PairOutputs(Outputs outputs1, Outputs outputs2) /// Create a new Pair public virtual Pair NewPair(A a, B b) { - if (a.Equals(Outputs1.NoOutput)) + if (a.Equals(outputs1.NoOutput)) { - a = Outputs1.NoOutput; + a = outputs1.NoOutput; } - if (b.Equals(Outputs2.NoOutput)) + if (b.Equals(outputs2.NoOutput)) { - b = Outputs2.NoOutput; + b = outputs2.NoOutput; } - if (a.Equals(Outputs1.NoOutput) && b.Equals(Outputs2.NoOutput)) + if (a.Equals(outputs1.NoOutput) && b.Equals(outputs2.NoOutput)) { return NO_OUTPUT; } @@ -106,15 +106,15 @@ public virtual Pair NewPair(A a, B b) // for assert private bool Valid(Pair pair) { - bool noOutput1 = pair.Output1.Equals(Outputs1.NoOutput); - bool noOutput2 = pair.Output2.Equals(Outputs2.NoOutput); + bool noOutput1 = pair.Output1.Equals(outputs1.NoOutput); + bool noOutput2 = pair.Output2.Equals(outputs2.NoOutput); - if (noOutput1 && !pair.Output1.Equals(Outputs1.NoOutput)) + if (noOutput1 && !pair.Output1.Equals(outputs1.NoOutput)) { return false; } - if (noOutput2 && !pair.Output2.Equals(Outputs2.NoOutput)) + if (noOutput2 && !pair.Output2.Equals(outputs2.NoOutput)) { return false; } @@ -140,34 +140,34 @@ public override Pair Common(Pair pair1, Pair pair2) { Debug.Assert(Valid(pair1)); Debug.Assert(Valid(pair2)); - return NewPair(Outputs1.Common(pair1.Output1, pair2.Output1), Outputs2.Common(pair1.Output2, pair2.Output2)); + return NewPair(outputs1.Common(pair1.Output1, pair2.Output1), outputs2.Common(pair1.Output2, pair2.Output2)); } public override Pair Subtract(Pair output, Pair inc) { Debug.Assert(Valid(output)); Debug.Assert(Valid(inc)); - return NewPair(Outputs1.Subtract(output.Output1, inc.Output1), Outputs2.Subtract(output.Output2, inc.Output2)); + return NewPair(outputs1.Subtract(output.Output1, inc.Output1), outputs2.Subtract(output.Output2, inc.Output2)); } public override Pair Add(Pair prefix, Pair output) { Debug.Assert(Valid(prefix)); Debug.Assert(Valid(output)); - return NewPair(Outputs1.Add(prefix.Output1, output.Output1), Outputs2.Add(prefix.Output2, output.Output2)); + return NewPair(outputs1.Add(prefix.Output1, output.Output1), outputs2.Add(prefix.Output2, output.Output2)); } public override void Write(Pair output, DataOutput writer) { Debug.Assert(Valid(output)); - Outputs1.Write(output.Output1, writer); - Outputs2.Write(output.Output2, writer); + outputs1.Write(output.Output1, writer); + outputs2.Write(output.Output2, writer); } public override Pair Read(DataInput @in) { - A output1 = Outputs1.Read(@in); - B output2 = Outputs2.Read(@in); + A output1 = outputs1.Read(@in); + B output2 = outputs2.Read(@in); return NewPair(output1, output2); } @@ -182,12 +182,12 @@ public override Pair NoOutput public override string OutputToString(Pair output) { Debug.Assert(Valid(output)); - return ""; + return ""; } public override string ToString() { - return "PairOutputs<" + Outputs1 + "," + Outputs2 + ">"; + return "PairOutputs<" + outputs1 + "," + outputs2 + ">"; } } } \ No newline at end of file diff --git a/src/Lucene.Net.Core/Util/Fst/ReverseBytesReader.cs b/src/Lucene.Net.Core/Util/Fst/ReverseBytesReader.cs index 3f68547f79..3266aad281 100644 --- a/src/Lucene.Net.Core/Util/Fst/ReverseBytesReader.cs +++ b/src/Lucene.Net.Core/Util/Fst/ReverseBytesReader.cs @@ -21,41 +21,41 @@ namespace Lucene.Net.Util.Fst /// Reads in reverse from a single byte[]. internal sealed class ReverseBytesReader : FST.BytesReader { - private readonly byte[] Bytes; - private int Pos; + private readonly byte[] bytes; + private int pos; public ReverseBytesReader(byte[] bytes) { - this.Bytes = bytes; + this.bytes = bytes; } public override byte ReadByte() { - return Bytes[Pos--]; + return bytes[pos--]; } public override void ReadBytes(byte[] b, int offset, int len) { for (int i = 0; i < len; i++) { - b[offset + i] = Bytes[Pos--]; + b[offset + i] = bytes[pos--]; } } public override void SkipBytes(int count) { - Pos -= count; + pos -= count; } public override long Position { get { - return Pos; + return pos; } set { - this.Pos = (int)value; + this.pos = (int)value; } } diff --git a/src/Lucene.Net.Core/Util/Fst/Util.cs b/src/Lucene.Net.Core/Util/Fst/Util.cs index 749f209ffa..451ee51b19 100644 --- a/src/Lucene.Net.Core/Util/Fst/Util.cs +++ b/src/Lucene.Net.Core/Util/Fst/Util.cs @@ -356,12 +356,12 @@ public virtual int Compare(FSTPath a, FSTPath b) /// public class TopNSearcher { - internal readonly FST Fst; - internal readonly FST.BytesReader BytesReader; - internal readonly int TopN; - internal readonly int MaxQueueDepth; + private readonly FST fst; + private readonly FST.BytesReader bytesReader; + private readonly int topN; + private readonly int maxQueueDepth; - internal readonly FST.Arc ScratchArc = new FST.Arc(); + private readonly FST.Arc scratchArc = new FST.Arc(); internal readonly IComparer Comparator; @@ -375,10 +375,10 @@ public class TopNSearcher /// the comparator to select the top N public TopNSearcher(FST fst, int topN, int maxQueueDepth, IComparer comparator) { - this.Fst = fst; - this.BytesReader = fst.BytesReader; - this.TopN = topN; - this.MaxQueueDepth = maxQueueDepth; + this.fst = fst; + this.bytesReader = fst.BytesReader; + this.topN = topN; + this.maxQueueDepth = maxQueueDepth; this.Comparator = comparator; Queue = new SortedSet>(new TieBreakByInputComparator(comparator)); @@ -389,10 +389,10 @@ protected virtual void AddIfCompetitive(FSTPath path) { Debug.Assert(Queue != null); - T cost = Fst.Outputs.Add(path.Cost, path.Arc.Output); + T cost = fst.Outputs.Add(path.Cost, path.Arc.Output); //System.out.println(" addIfCompetitive queue.size()=" + queue.size() + " path=" + path + " + label=" + path.arc.label); - if (Queue.Count == MaxQueueDepth) + if (Queue.Count == maxQueueDepth) { FSTPath bottom = Queue.Max; int comp = Comparator.Compare(cost, bottom.Cost); @@ -435,7 +435,7 @@ protected virtual void AddIfCompetitive(FSTPath path) Queue.Add(newPath); - if (Queue.Count == MaxQueueDepth + 1) + if (Queue.Count == maxQueueDepth + 1) { Queue.Last(); } @@ -448,13 +448,13 @@ protected virtual void AddIfCompetitive(FSTPath path) public virtual void AddStartPaths(FST.Arc node, T startOutput, bool allowEmptyString, IntsRef input) { // De-dup NO_OUTPUT since it must be a singleton: - if (startOutput.Equals(Fst.Outputs.NoOutput)) + if (startOutput.Equals(fst.Outputs.NoOutput)) { - startOutput = Fst.Outputs.NoOutput; + startOutput = fst.Outputs.NoOutput; } FSTPath path = new FSTPath(startOutput, node, input); - Fst.ReadFirstTargetArc(node, path.Arc, BytesReader); + fst.ReadFirstTargetArc(node, path.Arc, bytesReader); //System.out.println("add start paths"); @@ -469,7 +469,7 @@ public virtual void AddStartPaths(FST.Arc node, T startOutput, bool allowE { break; } - Fst.ReadNextArc(path.Arc, BytesReader); + fst.ReadNextArc(path.Arc, bytesReader); } } @@ -479,8 +479,8 @@ public virtual TopResults Search() //System.out.println("search topN=" + topN); - var fstReader = Fst.BytesReader; - T NO_OUTPUT = Fst.Outputs.NoOutput; + var fstReader = fst.BytesReader; + T NO_OUTPUT = fst.Outputs.NoOutput; // TODO: we could enable FST to sorting arcs by weight // as it freezes... can easily do this on first pass @@ -491,7 +491,7 @@ public virtual TopResults Search() int rejectCount = 0; // For each top N path: - while (results.Count < TopN) + while (results.Count < topN) { //System.out.println("\nfind next path: queue.size=" + queue.size()); @@ -524,7 +524,7 @@ public virtual TopResults Search() continue; } - if (results.Count == TopN - 1 && MaxQueueDepth == TopN) + if (results.Count == topN - 1 && maxQueueDepth == topN) { // Last path -- don't bother w/ queue anymore: Queue = null; @@ -542,7 +542,7 @@ public virtual TopResults Search() while (true) { //System.out.println("\n cycle path: " + path); - Fst.ReadFirstTargetArc(path.Arc, path.Arc, fstReader); + fst.ReadFirstTargetArc(path.Arc, path.Arc, fstReader); // For each arc leaving this node: bool foundZero = false; @@ -560,7 +560,7 @@ public virtual TopResults Search() } else if (!foundZero) { - ScratchArc.CopyFrom(path.Arc); + scratchArc.CopyFrom(path.Arc); foundZero = true; } else @@ -576,7 +576,7 @@ public virtual TopResults Search() { break; } - Fst.ReadNextArc(path.Arc, fstReader); + fst.ReadNextArc(path.Arc, fstReader); } Debug.Assert(foundZero); @@ -587,14 +587,14 @@ public virtual TopResults Search() // are more clever above... eg on finding the // first NO_OUTPUT arc we'd switch to using // scratchArc - path.Arc.CopyFrom(ScratchArc); + path.Arc.CopyFrom(scratchArc); } if (path.Arc.Label == FST.END_LABEL) { // Add final output: //System.out.println(" done!: " + path); - T finalOutput = Fst.Outputs.Add(path.Cost, path.Arc.Output); + T finalOutput = fst.Outputs.Add(path.Cost, path.Arc.Output); if (AcceptResult(path.Input, finalOutput)) { //System.out.println(" add result: " + path); @@ -611,11 +611,11 @@ public virtual TopResults Search() path.Input.Grow(1 + path.Input.Length); path.Input.Ints[path.Input.Length] = path.Arc.Label; path.Input.Length++; - path.Cost = Fst.Outputs.Add(path.Cost, path.Arc.Output); + path.Cost = fst.Outputs.Add(path.Cost, path.Arc.Output); } } } - return new TopResults(rejectCount + TopN <= MaxQueueDepth, results); + return new TopResults(rejectCount + topN <= maxQueueDepth, results); } protected virtual bool AcceptResult(IntsRef input, T output) From 18f84437970c1cc0d16312952a4e6726d35f97df Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Wed, 7 Sep 2016 17:30:10 +0700 Subject: [PATCH 11/47] .NETify FST: Interfaces should start with "I" --- src/Lucene.Net.Core/Util/Fst/Builder.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Lucene.Net.Core/Util/Fst/Builder.cs b/src/Lucene.Net.Core/Util/Fst/Builder.cs index 540b9131ac..93dfbf6ec5 100644 --- a/src/Lucene.Net.Core/Util/Fst/Builder.cs +++ b/src/Lucene.Net.Core/Util/Fst/Builder.cs @@ -573,7 +573,7 @@ private void CompileAllTargets(UnCompiledNode node, int tailLength) public class Arc { public int Label; // really an "unsigned" byte - public Node Target; + public INode Target; public bool IsFinal; public S Output; public S NextFinalOutput; @@ -583,7 +583,7 @@ public class Arc // memory while the FST is being built; it's only the // current "frontier": - public interface Node + public interface INode { bool IsCompiled { get; } } @@ -593,7 +593,7 @@ public virtual long FstSizeInBytes() return fst.SizeInBytes(); } - public sealed class CompiledNode : Node + public sealed class CompiledNode : INode { public long Node; @@ -608,7 +608,7 @@ public bool IsCompiled /// /// Expert: holds a pending (seen but not yet serialized) Node. - public sealed class UnCompiledNode : Node + public sealed class UnCompiledNode : INode { internal readonly Builder Owner; public int NumArcs; @@ -666,7 +666,7 @@ public S GetLastOutput(int labelToMatch) return Arcs[NumArcs - 1].Output; } - public void AddArc(int label, Node target) + public void AddArc(int label, INode target) { Debug.Assert(label >= 0); // LUCENENET: Commented this because it makes testing difficult in Visual Studio. @@ -691,7 +691,7 @@ public void AddArc(int label, Node target) arc.IsFinal = false; } - public void ReplaceLast(int labelToMatch, Node target, S nextFinalOutput, bool isFinal) + public void ReplaceLast(int labelToMatch, INode target, S nextFinalOutput, bool isFinal) { Debug.Assert(NumArcs > 0); Arc arc = Arcs[NumArcs - 1]; @@ -702,7 +702,7 @@ public void ReplaceLast(int labelToMatch, Node target, S nextFinalOutput, bool i arc.IsFinal = isFinal; } - public void DeleteLast(int label, Node target) + public void DeleteLast(int label, INode target) { Debug.Assert(NumArcs > 0); Debug.Assert(label == Arcs[NumArcs - 1].Label); From 3994051b243f9e4a39cdda107ac8c6d1d68a78e2 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Wed, 7 Sep 2016 18:07:43 +0700 Subject: [PATCH 12/47] Moved BytesRefFSTEnum.InputOutput and IntsRefFSTEnum.InputOutput classes into non-generic classes with the same name so we don't need to specify the generic type twice. --- .../BlockTerms/VariableGapTermsIndexReader.cs | 4 +- .../Memory/FSTOrdTermsReader.cs | 2 +- .../Memory/FSTTermsReader.cs | 2 +- .../Memory/MemoryPostingsFormat.cs | 2 +- .../Codecs/BlockTreeTermsWriter.cs | 2 +- .../Lucene42/Lucene42DocValuesProducer.cs | 2 +- .../Util/Fst/BytesRefFSTEnum.cs | 41 ++++++++++++------- .../Util/Fst/IntsRefFSTEnum.cs | 41 ++++++++++++------- .../Util/fst/FSTTester.cs | 8 ++-- .../core/Util/Fst/Test2BFST.cs | 6 +-- .../core/Util/Fst/TestFSTs.cs | 6 +-- 11 files changed, 71 insertions(+), 45 deletions(-) diff --git a/src/Lucene.Net.Codecs/BlockTerms/VariableGapTermsIndexReader.cs b/src/Lucene.Net.Codecs/BlockTerms/VariableGapTermsIndexReader.cs index 654ad34a83..17b808f693 100644 --- a/src/Lucene.Net.Codecs/BlockTerms/VariableGapTermsIndexReader.cs +++ b/src/Lucene.Net.Codecs/BlockTerms/VariableGapTermsIndexReader.cs @@ -202,7 +202,7 @@ private void LoadTermsIndex() var fstEnum = new BytesRefFSTEnum(Fst); var count = _vgtir._indexDivisor; - BytesRefFSTEnum.InputOutput result; + BytesRefFSTEnum.InputOutput result; while ((result = fstEnum.Next()) != null) { if (count == _vgtir._indexDivisor) @@ -226,7 +226,7 @@ public long RamBytesUsed() protected class IndexEnum : FieldIndexEnum { private readonly BytesRefFSTEnum _fstEnum; - private BytesRefFSTEnum.InputOutput _current; + private BytesRefFSTEnum.InputOutput _current; public IndexEnum(FST fst) { diff --git a/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs b/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs index 7f3f35b3f3..86afc3e246 100644 --- a/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs +++ b/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs @@ -531,7 +531,7 @@ internal override void DecodeMetaData() } // Update current enum according to FSTEnum - private void UpdateEnum(BytesRefFSTEnum.InputOutput pair) + private void UpdateEnum(BytesRefFSTEnum.InputOutput pair) { if (pair == null) { diff --git a/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs b/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs index 79e0365685..0077054998 100644 --- a/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs +++ b/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs @@ -396,7 +396,7 @@ internal override void DecodeMetaData() } // Update current enum according to FSTEnum - internal void UpdateEnum(BytesRefFSTEnum.InputOutput pair) + internal void UpdateEnum(BytesRefFSTEnum.InputOutput pair) { if (pair == null) { diff --git a/src/Lucene.Net.Codecs/Memory/MemoryPostingsFormat.cs b/src/Lucene.Net.Codecs/Memory/MemoryPostingsFormat.cs index ebf7456047..88aea52849 100644 --- a/src/Lucene.Net.Codecs/Memory/MemoryPostingsFormat.cs +++ b/src/Lucene.Net.Codecs/Memory/MemoryPostingsFormat.cs @@ -788,7 +788,7 @@ private sealed class FSTTermsEnum : TermsEnum internal int docFreq_Renamed; internal long totalTermFreq_Renamed; - internal BytesRefFSTEnum.InputOutput current; + internal BytesRefFSTEnum.InputOutput current; internal BytesRef postingsSpare = new BytesRef(); public FSTTermsEnum(FieldInfo field, FST fst) diff --git a/src/Lucene.Net.Core/Codecs/BlockTreeTermsWriter.cs b/src/Lucene.Net.Core/Codecs/BlockTreeTermsWriter.cs index e339aa32c1..0c32f3bde4 100644 --- a/src/Lucene.Net.Core/Codecs/BlockTreeTermsWriter.cs +++ b/src/Lucene.Net.Core/Codecs/BlockTreeTermsWriter.cs @@ -501,7 +501,7 @@ public void CompileIndex(IList floorBlocks, RAMOutputStream scratc internal void Append(Builder builder, FST subIndex) { BytesRefFSTEnum subIndexEnum = new BytesRefFSTEnum(subIndex); - BytesRefFSTEnum.InputOutput indexEnt; + BytesRefFSTEnum.InputOutput indexEnt; while ((indexEnt = subIndexEnum.Next()) != null) { //if (DEBUG) { diff --git a/src/Lucene.Net.Core/Codecs/Lucene42/Lucene42DocValuesProducer.cs b/src/Lucene.Net.Core/Codecs/Lucene42/Lucene42DocValuesProducer.cs index 802ed53bb5..5977d181c2 100644 --- a/src/Lucene.Net.Core/Codecs/Lucene42/Lucene42DocValuesProducer.cs +++ b/src/Lucene.Net.Core/Codecs/Lucene42/Lucene42DocValuesProducer.cs @@ -489,7 +489,7 @@ public override int LookupTerm(BytesRef key) { try { - BytesRefFSTEnum.InputOutput o = FstEnum.SeekCeil(key); + BytesRefFSTEnum.InputOutput o = FstEnum.SeekCeil(key); if (o == null) { return -ValueCount - 1; diff --git a/src/Lucene.Net.Core/Util/Fst/BytesRefFSTEnum.cs b/src/Lucene.Net.Core/Util/Fst/BytesRefFSTEnum.cs index 6774b21919..8bd5028de9 100644 --- a/src/Lucene.Net.Core/Util/Fst/BytesRefFSTEnum.cs +++ b/src/Lucene.Net.Core/Util/Fst/BytesRefFSTEnum.cs @@ -29,16 +29,10 @@ namespace Lucene.Net.Util.Fst public sealed class BytesRefFSTEnum : FSTEnum { private readonly BytesRef current = new BytesRef(10); - private readonly InputOutput result = new InputOutput(); + private readonly BytesRefFSTEnum.InputOutput result = new BytesRefFSTEnum.InputOutput(); private BytesRef target; - /// - /// Holds a single input (BytesRef) + output pair. - public class InputOutput - { - public BytesRef Input; - public T Output; - } + // LUCENENET NOTE: InputOutput was moved to the BytesRefFSTEnum class /// /// doFloor controls the behavior of advance: if it's true @@ -52,12 +46,12 @@ public BytesRefFSTEnum(FST fst) current.Offset = 1; } - public InputOutput Current() + public BytesRefFSTEnum.InputOutput Current() { return result; } - public InputOutput Next() + public BytesRefFSTEnum.InputOutput Next() { //System.out.println(" enum.next"); DoNext(); @@ -66,7 +60,7 @@ public InputOutput Next() /// /// Seeks to smallest term that's >= target. - public InputOutput SeekCeil(BytesRef target) + public BytesRefFSTEnum.InputOutput SeekCeil(BytesRef target) { this.target = target; targetLength = target.Length; @@ -76,7 +70,7 @@ public InputOutput SeekCeil(BytesRef target) /// /// Seeks to biggest term that's <= target. - public InputOutput SeekFloor(BytesRef target) + public BytesRefFSTEnum.InputOutput SeekFloor(BytesRef target) { this.target = target; targetLength = target.Length; @@ -90,7 +84,7 @@ public InputOutput SeekFloor(BytesRef target) /// #seekFloor} or because it /// short-circuits as soon the match is not found. /// - public InputOutput SeekExact(BytesRef target) + public BytesRefFSTEnum.InputOutput SeekExact(BytesRef target) { this.target = target; targetLength = target.Length; @@ -138,7 +132,7 @@ protected internal override void Grow() current.Bytes = ArrayUtil.Grow(current.Bytes, upto + 1); } - private InputOutput SetResult() + private BytesRefFSTEnum.InputOutput SetResult() { if (upto == 0) { @@ -152,4 +146,23 @@ private InputOutput SetResult() } } } + + /// + /// LUCENENET specific. This class is to mimic Java's ability to specify + /// nested classes of Generics without having to specify the generic type + /// (i.e. BytesRefFSTEnum.InputOutput{T} rather than BytesRefFSTEnum{T}.InputOutput{T}) + /// + public sealed class BytesRefFSTEnum + { + private BytesRefFSTEnum() + { } + + /// + /// Holds a single input (BytesRef) + output pair. + public class InputOutput + { + public BytesRef Input; + public T Output; + } + } } \ No newline at end of file diff --git a/src/Lucene.Net.Core/Util/Fst/IntsRefFSTEnum.cs b/src/Lucene.Net.Core/Util/Fst/IntsRefFSTEnum.cs index b70d8c4797..f679a6f186 100644 --- a/src/Lucene.Net.Core/Util/Fst/IntsRefFSTEnum.cs +++ b/src/Lucene.Net.Core/Util/Fst/IntsRefFSTEnum.cs @@ -29,16 +29,10 @@ namespace Lucene.Net.Util.Fst public sealed class IntsRefFSTEnum : FSTEnum { private readonly IntsRef current = new IntsRef(10); - private readonly InputOutput result = new InputOutput(); + private readonly IntsRefFSTEnum.InputOutput result = new IntsRefFSTEnum.InputOutput(); private IntsRef target; - /// - /// Holds a single input (IntsRef) + output pair. - public class InputOutput - { - public IntsRef Input; - public T Output; - } + // LUCENENET NOTE: The InputOutput class was moved into the IntsRefFSTEnum class /// /// doFloor controls the behavior of advance: if it's true @@ -52,12 +46,12 @@ public IntsRefFSTEnum(FST fst) current.Offset = 1; } - public InputOutput Current() + public IntsRefFSTEnum.InputOutput Current() { return result; } - public InputOutput Next() + public IntsRefFSTEnum.InputOutput Next() { //System.out.println(" enum.next"); DoNext(); @@ -66,7 +60,7 @@ public InputOutput Next() /// /// Seeks to smallest term that's >= target. - public InputOutput SeekCeil(IntsRef target) + public IntsRefFSTEnum.InputOutput SeekCeil(IntsRef target) { this.target = target; targetLength = target.Length; @@ -76,7 +70,7 @@ public InputOutput SeekCeil(IntsRef target) /// /// Seeks to biggest term that's <= target. - public InputOutput SeekFloor(IntsRef target) + public IntsRefFSTEnum.InputOutput SeekFloor(IntsRef target) { this.target = target; targetLength = target.Length; @@ -90,7 +84,7 @@ public InputOutput SeekFloor(IntsRef target) /// #seekFloor} or because it /// short-circuits as soon the match is not found. /// - public InputOutput SeekExact(IntsRef target) + public IntsRefFSTEnum.InputOutput SeekExact(IntsRef target) { this.target = target; targetLength = target.Length; @@ -138,7 +132,7 @@ protected internal override void Grow() current.Ints = ArrayUtil.Grow(current.Ints, upto + 1); } - private InputOutput SetResult() + private IntsRefFSTEnum.InputOutput SetResult() { if (upto == 0) { @@ -152,4 +146,23 @@ private InputOutput SetResult() } } } + + /// + /// LUCENENET specific. This class is to mimic Java's ability to specify + /// nested classes of Generics without having to specify the generic type + /// (i.e. IntsRefFSTEnum.InputOutput{T} rather than IntsRefFSTEnum{T}.InputOutput{T}) + /// + public sealed class IntsRefFSTEnum + { + private IntsRefFSTEnum() + { } + + /// + /// Holds a single input (IntsRef) + output pair. + public class InputOutput + { + public IntsRef Input; + public T Output; + } + } } \ No newline at end of file diff --git a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs index e5d6f84658..cb5f44dbed 100644 --- a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs +++ b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs @@ -486,7 +486,7 @@ private void VerifyUnPruned(int inputMode, FST fst) Assert.IsTrue(OutputsEqual(pair.Output, output)); // verify enum's next - IntsRefFSTEnum.InputOutput t = fstEnum.Next(); + IntsRefFSTEnum.InputOutput t = fstEnum.Next(); Assert.IsNotNull(t); Assert.AreEqual(term, t.Input, "expected input=" + InputToString(inputMode, term) + " but fstEnum returned " + InputToString(inputMode, t.Input)); Assert.IsTrue(OutputsEqual(pair.Output, t.Output)); @@ -564,7 +564,7 @@ private void VerifyUnPruned(int inputMode, FST fst) pos = -(pos + 1); // ok doesn't exist //System.out.println(" seek " + inputToString(inputMode, term)); - IntsRefFSTEnum.InputOutput seekResult; + IntsRefFSTEnum.InputOutput seekResult; if (Random.Next(3) == 0) { if (LuceneTestCase.VERBOSE) @@ -622,7 +622,7 @@ private void VerifyUnPruned(int inputMode, FST fst) { // seek to term that does exist: InputOutput pair = Pairs[Random.Next(Pairs.Count)]; - IntsRefFSTEnum.InputOutput seekResult; + IntsRefFSTEnum.InputOutput seekResult; if (Random.Next(3) == 2) { if (LuceneTestCase.VERBOSE) @@ -960,7 +960,7 @@ private void VerifyPruned(int inputMode, FST fst, int prune1, int prune2) Console.WriteLine("TEST: check pruned enum"); } IntsRefFSTEnum fstEnum = new IntsRefFSTEnum(fst); - IntsRefFSTEnum.InputOutput current; + IntsRefFSTEnum.InputOutput current; while ((current = fstEnum.Next()) != null) { if (LuceneTestCase.VERBOSE) diff --git a/src/Lucene.Net.Tests/core/Util/Fst/Test2BFST.cs b/src/Lucene.Net.Tests/core/Util/Fst/Test2BFST.cs index ad2f968acd..7127487ab5 100644 --- a/src/Lucene.Net.Tests/core/Util/Fst/Test2BFST.cs +++ b/src/Lucene.Net.Tests/core/Util/Fst/Test2BFST.cs @@ -111,7 +111,7 @@ public virtual void Test() int upto = 0; while (true) { - IntsRefFSTEnum.InputOutput pair = fstEnum.Next(); + IntsRefFSTEnum.InputOutput pair = fstEnum.Next(); if (pair == null) { break; @@ -200,7 +200,7 @@ public virtual void Test() int upto = 0; while (true) { - IntsRefFSTEnum.InputOutput pair = fstEnum.Next(); + IntsRefFSTEnum.InputOutput pair = fstEnum.Next(); if (pair == null) { break; @@ -293,7 +293,7 @@ public virtual void Test() output = 1; while (true) { - IntsRefFSTEnum.InputOutput pair = fstEnum.Next(); + IntsRefFSTEnum.InputOutput pair = fstEnum.Next(); if (pair == null) { break; diff --git a/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs b/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs index 2e972ad2d1..bb4d206b82 100644 --- a/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs +++ b/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs @@ -438,7 +438,7 @@ public virtual void TestRealTerms() } TermsEnum.SeekStatus seekResult = termsEnum.SeekCeil(randomTerm); - BytesRefFSTEnum.InputOutput fstSeekResult = fstEnum.SeekCeil(randomTerm); + BytesRefFSTEnum.InputOutput fstSeekResult = fstEnum.SeekCeil(randomTerm); if (seekResult == TermsEnum.SeekStatus.END) { @@ -472,7 +472,7 @@ public virtual void TestRealTerms() { Console.WriteLine(" end!"); } - BytesRefFSTEnum.InputOutput nextResult = fstEnum.Next(); + BytesRefFSTEnum.InputOutput nextResult = fstEnum.Next(); if (nextResult != null) { Console.WriteLine("expected null but got: input=" + nextResult.Input.Utf8ToString() + " output=" + outputs.OutputToString(nextResult.Output)); @@ -985,7 +985,7 @@ public virtual void TestSimple() Assert.AreEqual(17, Util.Get(fst, a)); BytesRefFSTEnum fstEnum = new BytesRefFSTEnum(fst); - BytesRefFSTEnum.InputOutput seekResult; + BytesRefFSTEnum.InputOutput seekResult; seekResult = fstEnum.SeekFloor(a); Assert.IsNotNull(seekResult); Assert.AreEqual(17, seekResult.Output); From 0ccb5b65809ad6b341b3b04fcdfb43c58e53aff5 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Wed, 7 Sep 2016 18:17:58 +0700 Subject: [PATCH 13/47] Normalized FST license headers --- src/Lucene.Net.Core/Util/Fst/FST.cs | 38 +++++++++++------------- src/Lucene.Net.Core/Util/Fst/NodeHash.cs | 30 +++++++++---------- src/Lucene.Net.Core/Util/Fst/Util.cs | 4 --- 3 files changed, 33 insertions(+), 39 deletions(-) diff --git a/src/Lucene.Net.Core/Util/Fst/FST.cs b/src/Lucene.Net.Core/Util/Fst/FST.cs index 335006883c..6b4a3e23ea 100644 --- a/src/Lucene.Net.Core/Util/Fst/FST.cs +++ b/src/Lucene.Net.Core/Util/Fst/FST.cs @@ -1,32 +1,30 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Diagnostics; +using System.IO; using System.Text; namespace Lucene.Net.Util.Fst { - using Lucene.Net.Util; - using System.Collections; - using System.IO; - using ByteArrayDataOutput = Lucene.Net.Store.ByteArrayDataOutput; - /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + using ByteArrayDataOutput = Lucene.Net.Store.ByteArrayDataOutput; using CodecUtil = Lucene.Net.Codecs.CodecUtil; using DataInput = Lucene.Net.Store.DataInput; using DataOutput = Lucene.Net.Store.DataOutput; diff --git a/src/Lucene.Net.Core/Util/Fst/NodeHash.cs b/src/Lucene.Net.Core/Util/Fst/NodeHash.cs index fcc9429e02..a0fb6599ad 100644 --- a/src/Lucene.Net.Core/Util/Fst/NodeHash.cs +++ b/src/Lucene.Net.Core/Util/Fst/NodeHash.cs @@ -4,21 +4,21 @@ namespace Lucene.Net.Util.Fst { using System.Collections; /* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ using PackedInts = Lucene.Net.Util.Packed.PackedInts; using PagedGrowableWriter = Lucene.Net.Util.Packed.PagedGrowableWriter; diff --git a/src/Lucene.Net.Core/Util/Fst/Util.cs b/src/Lucene.Net.Core/Util/Fst/Util.cs index 451ee51b19..baf648f30a 100644 --- a/src/Lucene.Net.Core/Util/Fst/Util.cs +++ b/src/Lucene.Net.Core/Util/Fst/Util.cs @@ -24,10 +24,6 @@ namespace Lucene.Net.Util.Fst * See the License for the specific language governing permissions and * limitations under the License. */ - /* - using Lucene.Net.Util.Fst.FST; - using BytesReader = Lucene.Net.Util.Fst.FST.BytesReader; - */ /// /// Static helper methods. From 01e903b1022cf13d708d8c1b5502047ebf704459 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Thu, 8 Sep 2016 03:11:08 +0700 Subject: [PATCH 14/47] Refactored FST class to disallow the verbose and confusing FST.Arc syntax, favoring FST.Arc. --- .../Analysis/CharFilter/NormalizeCharMap.cs | 2 +- .../Analysis/Hunspell/Dictionary.cs | 2 +- .../Codecs/BlockTreeTermsReader.cs | 52 +-- .../Codecs/BlockTreeTermsWriter.cs | 4 +- .../Lucene42/Lucene42DocValuesProducer.cs | 30 +- src/Lucene.Net.Core/Util/Fst/Builder.cs | 7 +- .../Util/Fst/BytesRefFSTEnum.cs | 2 +- src/Lucene.Net.Core/Util/Fst/FST.cs | 305 +++++++++--------- src/Lucene.Net.Core/Util/Fst/FSTEnum.cs | 54 ++-- .../Util/Fst/IntsRefFSTEnum.cs | 2 +- src/Lucene.Net.Core/Util/Fst/NodeHash.cs | 2 +- src/Lucene.Net.Core/Util/Fst/Util.cs | 60 ++-- .../Util/fst/FSTTester.cs | 6 +- 13 files changed, 270 insertions(+), 258 deletions(-) diff --git a/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/NormalizeCharMap.cs b/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/NormalizeCharMap.cs index 9571a31c95..37129110b1 100644 --- a/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/NormalizeCharMap.cs +++ b/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/NormalizeCharMap.cs @@ -54,7 +54,7 @@ private NormalizeCharMap(FST map) map.ReadFirstRealTargetArc(scratchArc.Target, scratchArc, fstReader); while (true) { - Debug.Assert(scratchArc.Label != FST.END_LABEL); // LUCENENET TODO END_LABEL shouldn't be under generic? + Debug.Assert(scratchArc.Label != FST.END_LABEL); cachedRootArcs[Convert.ToChar((char)scratchArc.Label)] = (new FST.Arc()).CopyFrom(scratchArc); if (scratchArc.IsLast) { diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Hunspell/Dictionary.cs b/src/Lucene.Net.Analysis.Common/Analysis/Hunspell/Dictionary.cs index e27b43314b..3ce2191b55 100644 --- a/src/Lucene.Net.Analysis.Common/Analysis/Hunspell/Dictionary.cs +++ b/src/Lucene.Net.Analysis.Common/Analysis/Hunspell/Dictionary.cs @@ -233,7 +233,7 @@ internal virtual IntsRef Lookup(FST fst, char[] word, int offset, int l output = fst.Outputs.Add(output, arc.Output); } } - if (fst.FindTargetArc(FST.END_LABEL, arc, arc, bytesReader) == null) + if (fst.FindTargetArc(FST.END_LABEL, arc, arc, bytesReader) == null) { return null; } diff --git a/src/Lucene.Net.Core/Codecs/BlockTreeTermsReader.cs b/src/Lucene.Net.Core/Codecs/BlockTreeTermsReader.cs index 8ae85d35e4..a691816c77 100644 --- a/src/Lucene.Net.Core/Codecs/BlockTreeTermsReader.cs +++ b/src/Lucene.Net.Core/Codecs/BlockTreeTermsReader.cs @@ -705,7 +705,7 @@ private sealed class IntersectEnum : TermsEnum private Frame[] Stack; - internal FST.Arc[] Arcs = new FST.Arc[5]; + internal FST.Arc[] Arcs = new FST.Arc[5]; internal readonly RunAutomaton runAutomaton; internal readonly CompiledAutomaton CompiledAutomaton; @@ -714,7 +714,7 @@ private sealed class IntersectEnum : TermsEnum internal readonly BytesRef Term_Renamed = new BytesRef(); - internal readonly FST.BytesReader FstReader; + internal readonly FST.BytesReader FstReader; // TODO: can we share this with the frame in STE? private sealed class Frame @@ -764,7 +764,7 @@ private sealed class Frame internal int CurTransitionMax; internal int TransitionIndex; - internal FST.Arc Arc; + internal FST.Arc Arc; internal readonly BlockTermState TermState; @@ -1052,7 +1052,7 @@ public IntersectEnum(BlockTreeTermsReader.FieldReader outerInstance, CompiledAut } for (int arcIdx = 0; arcIdx < Arcs.Length; arcIdx++) { - Arcs[arcIdx] = new FST.Arc(); + Arcs[arcIdx] = new FST.Arc(); } if (outerInstance.Index == null) @@ -1071,7 +1071,7 @@ public IntersectEnum(BlockTreeTermsReader.FieldReader outerInstance, CompiledAut // Else the seek cost of loading the frames will be // too costly. - FST.Arc arc = outerInstance.Index.GetFirstArc(Arcs[0]); + FST.Arc arc = outerInstance.Index.GetFirstArc(Arcs[0]); // Empty string prefix must have an output in the index! Debug.Assert(arc.IsFinal); @@ -1123,15 +1123,15 @@ private Frame GetFrame(int ord) return Stack[ord]; } - internal FST.Arc GetArc(int ord) + internal FST.Arc GetArc(int ord) { if (ord >= Arcs.Length) { - FST.Arc[] next = new FST.Arc[ArrayUtil.Oversize(1 + ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; + FST.Arc[] next = new FST.Arc[ArrayUtil.Oversize(1 + ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; Array.Copy(Arcs, 0, next, 0, Arcs.Length); for (int arcOrd = Arcs.Length; arcOrd < next.Length; arcOrd++) { - next[arcOrd] = new FST.Arc(); + next[arcOrd] = new FST.Arc(); } Arcs = next; } @@ -1151,7 +1151,7 @@ private Frame PushFrame(int state) // "bother" with this so we can get the floor data // from the index and skip floor blocks when // possible: - FST.Arc arc = CurrentFrame.Arc; + FST.Arc arc = CurrentFrame.Arc; int idx = CurrentFrame.Prefix; Debug.Assert(CurrentFrame.Suffix > 0); BytesRef output = CurrentFrame.OutputPrefix; @@ -1223,7 +1223,7 @@ internal void SeekToStartTerm(BytesRef target) { Term_Renamed.Bytes = ArrayUtil.Grow(Term_Renamed.Bytes, target.Length); } - FST.Arc arc = Arcs[0]; + FST.Arc arc = Arcs[0]; Debug.Assert(arc == CurrentFrame.Arc); for (int idx = 0; idx <= target.Length; idx++) @@ -1543,9 +1543,9 @@ internal sealed class SegmentTermsEnum : TermsEnum internal bool Eof; internal readonly BytesRef Term_Renamed = new BytesRef(); - internal readonly FST.BytesReader FstReader; + internal readonly FST.BytesReader FstReader; - internal FST.Arc[] Arcs = new FST.Arc[1]; + internal FST.Arc[] Arcs = new FST.Arc[1]; public SegmentTermsEnum(BlockTreeTermsReader.FieldReader outerInstance) { @@ -1569,11 +1569,11 @@ public SegmentTermsEnum(BlockTreeTermsReader.FieldReader outerInstance) // not (and need not) have been loaded for (int arcIdx = 0; arcIdx < Arcs.Length; arcIdx++) { - Arcs[arcIdx] = new FST.Arc(); + Arcs[arcIdx] = new FST.Arc(); } CurrentFrame = StaticFrame; - FST.Arc arc; + FST.Arc arc; if (outerInstance.Index != null) { arc = outerInstance.Index.GetFirstArc(Arcs[0]); @@ -1621,7 +1621,7 @@ public Stats ComputeBlockStats() } CurrentFrame = StaticFrame; - FST.Arc arc; + FST.Arc arc; if (OuterInstance.Index != null) { arc = OuterInstance.Index.GetFirstArc(Arcs[0]); @@ -1732,15 +1732,15 @@ internal Frame GetFrame(int ord) return Stack[ord]; } - internal FST.Arc GetArc(int ord) + internal FST.Arc GetArc(int ord) { if (ord >= Arcs.Length) { - FST.Arc[] next = new FST.Arc[ArrayUtil.Oversize(1 + ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; + FST.Arc[] next = new FST.Arc[ArrayUtil.Oversize(1 + ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; Array.Copy(Arcs, 0, next, 0, Arcs.Length); for (int arcOrd = Arcs.Length; arcOrd < next.Length; arcOrd++) { - next[arcOrd] = new FST.Arc(); + next[arcOrd] = new FST.Arc(); } Arcs = next; } @@ -1756,7 +1756,7 @@ public override IComparer Comparator } // Pushes a frame we seek'd to - internal Frame PushFrame(FST.Arc arc, BytesRef frameData, int length) + internal Frame PushFrame(FST.Arc arc, BytesRef frameData, int length) { ScratchReader.Reset((byte[])(Array)frameData.Bytes, frameData.Offset, frameData.Length); long code = ScratchReader.ReadVLong(); @@ -1776,7 +1776,7 @@ internal Frame PushFrame(FST.Arc arc, BytesRef frameData, in // Pushes next'd frame or seek'd frame; we later // lazy-load the frame only when needed - internal Frame PushFrame(FST.Arc arc, long fp, int length) + internal Frame PushFrame(FST.Arc arc, long fp, int length) { Frame f = GetFrame(1 + CurrentFrame.Ord); f.Arc = arc; @@ -1841,7 +1841,7 @@ public override bool SeekExact(BytesRef target) Debug.Assert(ClearEOF()); - FST.Arc arc; + FST.Arc arc; int targetUpto; BytesRef output; @@ -2006,7 +2006,7 @@ public override bool SeekExact(BytesRef target) { int targetLabel = target.Bytes[target.Offset + targetUpto] & 0xFF; - FST.Arc nextArc = OuterInstance.Index.FindTargetArc(targetLabel, arc, GetArc(1 + targetUpto), FstReader); + FST.Arc nextArc = OuterInstance.Index.FindTargetArc(targetLabel, arc, GetArc(1 + targetUpto), FstReader); if (nextArc == null) { @@ -2130,7 +2130,7 @@ public override SeekStatus SeekCeil(BytesRef target) //printSeekState(); //} - FST.Arc arc; + FST.Arc arc; int targetUpto; BytesRef output; @@ -2291,7 +2291,7 @@ public override SeekStatus SeekCeil(BytesRef target) { int targetLabel = target.Bytes[target.Offset + targetUpto] & 0xFF; - FST.Arc nextArc = OuterInstance.Index.FindTargetArc(targetLabel, arc, GetArc(1 + targetUpto), FstReader); + FST.Arc nextArc = OuterInstance.Index.FindTargetArc(targetLabel, arc, GetArc(1 + targetUpto), FstReader); if (nextArc == null) { @@ -2469,7 +2469,7 @@ public override BytesRef Next() if (@in == null) { // Fresh TermsEnum; seek to first term: - FST.Arc arc; + FST.Arc arc; if (OuterInstance.Index != null) { arc = OuterInstance.Index.GetFirstArc(Arcs[0]); @@ -2672,7 +2672,7 @@ internal sealed class Frame internal bool HasTermsOrig; internal bool IsFloor; - internal FST.Arc Arc; + internal FST.Arc Arc; // File pointer where this block was loaded from internal long Fp; diff --git a/src/Lucene.Net.Core/Codecs/BlockTreeTermsWriter.cs b/src/Lucene.Net.Core/Codecs/BlockTreeTermsWriter.cs index 0c32f3bde4..6aa9555dbe 100644 --- a/src/Lucene.Net.Core/Codecs/BlockTreeTermsWriter.cs +++ b/src/Lucene.Net.Core/Codecs/BlockTreeTermsWriter.cs @@ -452,7 +452,7 @@ public void CompileIndex(IList floorBlocks, RAMOutputStream scratc } ByteSequenceOutputs outputs = ByteSequenceOutputs.Singleton; - Builder indexBuilder = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, true, false, int.MaxValue, outputs, null, false, PackedInts.COMPACT, true, 15); + Builder indexBuilder = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, true, false, int.MaxValue, outputs, null, false, PackedInts.COMPACT, true, 15); var bytes = new byte[(int)scratchBytes.FilePointer]; Debug.Assert(bytes.Length > 0); scratchBytes.WriteTo(bytes, 0); @@ -1100,7 +1100,7 @@ internal TermsWriter(BlockTreeTermsWriter outerInstance, FieldInfo fieldInfo) // this Builder is just used transiently to fragment // terms into "good" blocks; we don't save the // resulting FST: - BlockBuilder = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, int.MaxValue, NoOutputs, new FindBlocks(this), false, PackedInts.COMPACT, true, 15); + BlockBuilder = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, int.MaxValue, NoOutputs, new FindBlocks(this), false, PackedInts.COMPACT, true, 15); this.LongsSize = outerInstance.PostingsWriter.SetField(fieldInfo); } diff --git a/src/Lucene.Net.Core/Codecs/Lucene42/Lucene42DocValuesProducer.cs b/src/Lucene.Net.Core/Codecs/Lucene42/Lucene42DocValuesProducer.cs index 5977d181c2..b1581437a2 100644 --- a/src/Lucene.Net.Core/Codecs/Lucene42/Lucene42DocValuesProducer.cs +++ b/src/Lucene.Net.Core/Codecs/Lucene42/Lucene42DocValuesProducer.cs @@ -431,8 +431,8 @@ public override SortedDocValues GetSorted(FieldInfo field) // per-thread resources var @in = fst.BytesReader; - var firstArc = new FST.Arc(); - var scratchArc = new FST.Arc(); + var firstArc = new FST.Arc(); + var scratchArc = new FST.Arc(); var scratchInts = new IntsRef(); var fstEnum = new BytesRefFSTEnum(fst); @@ -444,13 +444,13 @@ private class SortedDocValuesAnonymousInnerClassHelper : SortedDocValues private readonly FSTEntry Entry; private readonly NumericDocValues DocToOrd; private readonly FST Fst; - private readonly FST.BytesReader @in; - private readonly FST.Arc FirstArc; - private readonly FST.Arc ScratchArc; + private readonly FST.BytesReader @in; + private readonly FST.Arc FirstArc; + private readonly FST.Arc ScratchArc; private readonly IntsRef ScratchInts; private readonly BytesRefFSTEnum FstEnum; - public SortedDocValuesAnonymousInnerClassHelper(FSTEntry entry, NumericDocValues docToOrd, FST fst, FST.BytesReader @in, FST.Arc firstArc, FST.Arc scratchArc, IntsRef scratchInts, BytesRefFSTEnum fstEnum) + public SortedDocValuesAnonymousInnerClassHelper(FSTEntry entry, NumericDocValues docToOrd, FST fst, FST.BytesReader @in, FST.Arc firstArc, FST.Arc scratchArc, IntsRef scratchInts, BytesRefFSTEnum fstEnum) { this.Entry = entry; this.DocToOrd = docToOrd; @@ -546,8 +546,8 @@ public override SortedSetDocValues GetSortedSet(FieldInfo field) // per-thread resources var @in = fst.BytesReader; - var firstArc = new FST.Arc(); - var scratchArc = new FST.Arc(); + var firstArc = new FST.Arc(); + var scratchArc = new FST.Arc(); var scratchInts = new IntsRef(); var fstEnum = new BytesRefFSTEnum(fst); var @ref = new BytesRef(); @@ -560,15 +560,15 @@ private class SortedSetDocValuesAnonymousInnerClassHelper : SortedSetDocValues private readonly FSTEntry Entry; private readonly BinaryDocValues DocToOrds; private readonly FST Fst; - private readonly FST.BytesReader @in; - private readonly FST.Arc FirstArc; - private readonly FST.Arc ScratchArc; + private readonly FST.BytesReader @in; + private readonly FST.Arc FirstArc; + private readonly FST.Arc ScratchArc; private readonly IntsRef ScratchInts; private readonly BytesRefFSTEnum FstEnum; private readonly BytesRef @ref; private readonly ByteArrayDataInput Input; - public SortedSetDocValuesAnonymousInnerClassHelper(FSTEntry entry, BinaryDocValues docToOrds, FST fst, FST.BytesReader @in, FST.Arc firstArc, FST.Arc scratchArc, IntsRef scratchInts, BytesRefFSTEnum fstEnum, BytesRef @ref, ByteArrayDataInput input) + public SortedSetDocValuesAnonymousInnerClassHelper(FSTEntry entry, BinaryDocValues docToOrds, FST fst, FST.BytesReader @in, FST.Arc firstArc, FST.Arc scratchArc, IntsRef scratchInts, BytesRefFSTEnum fstEnum, BytesRef @ref, ByteArrayDataInput input) { this.Entry = entry; this.DocToOrds = docToOrds; @@ -715,9 +715,9 @@ internal class FSTTermsEnum : TermsEnum // maybe we should add a FSTEnum that supports this operation? internal readonly FST Fst; - internal readonly FST.BytesReader BytesReader; - internal readonly FST.Arc FirstArc = new FST.Arc(); - internal readonly FST.Arc ScratchArc = new FST.Arc(); + internal readonly FST.BytesReader BytesReader; + internal readonly FST.Arc FirstArc = new FST.Arc(); + internal readonly FST.Arc ScratchArc = new FST.Arc(); internal readonly IntsRef ScratchInts = new IntsRef(); internal readonly BytesRef ScratchBytes = new BytesRef(); diff --git a/src/Lucene.Net.Core/Util/Fst/Builder.cs b/src/Lucene.Net.Core/Util/Fst/Builder.cs index 93dfbf6ec5..11cce9f82e 100644 --- a/src/Lucene.Net.Core/Util/Fst/Builder.cs +++ b/src/Lucene.Net.Core/Util/Fst/Builder.cs @@ -97,7 +97,7 @@ public abstract class FreezeTail /// boolean, int, Outputs, FreezeTail, boolean, float, /// boolean, int)} with pruning options turned off. /// - public Builder(FST.INPUT_TYPE inputType, Outputs outputs) + public Builder(FST.INPUT_TYPE inputType, Outputs outputs) : this(inputType, 0, 0, true, true, int.MaxValue, outputs, null, false, PackedInts.COMPACT, true, 15) { } @@ -152,7 +152,10 @@ public Builder(FST.INPUT_TYPE inputType, Outputs outputs) /// byte[] block in the BytesStore; if you know the FST /// will be large then make this larger. For example 15 /// bits = 32768 byte pages. - public Builder(FST.INPUT_TYPE inputType, int minSuffixCount1, int minSuffixCount2, bool doShareSuffix, bool doShareNonSingletonNodes, int shareMaxTailLength, Outputs outputs, FreezeTail freezeTail, bool doPackFST, float acceptableOverheadRatio, bool allowArrayArcs, int bytesPageBits) + public Builder(FST.INPUT_TYPE inputType, int minSuffixCount1, int minSuffixCount2, bool doShareSuffix, + bool doShareNonSingletonNodes, int shareMaxTailLength, Outputs outputs, + FreezeTail freezeTail, bool doPackFST, float acceptableOverheadRatio, bool allowArrayArcs, + int bytesPageBits) { this.minSuffixCount1 = minSuffixCount1; this.minSuffixCount2 = minSuffixCount2; diff --git a/src/Lucene.Net.Core/Util/Fst/BytesRefFSTEnum.cs b/src/Lucene.Net.Core/Util/Fst/BytesRefFSTEnum.cs index 8bd5028de9..1617f2636f 100644 --- a/src/Lucene.Net.Core/Util/Fst/BytesRefFSTEnum.cs +++ b/src/Lucene.Net.Core/Util/Fst/BytesRefFSTEnum.cs @@ -105,7 +105,7 @@ protected internal override int TargetLabel { if (upto - 1 == target.Length) { - return FST.END_LABEL; + return FST.END_LABEL; } else { diff --git a/src/Lucene.Net.Core/Util/Fst/FST.cs b/src/Lucene.Net.Core/Util/Fst/FST.cs index 6b4a3e23ea..99be83dde2 100644 --- a/src/Lucene.Net.Core/Util/Fst/FST.cs +++ b/src/Lucene.Net.Core/Util/Fst/FST.cs @@ -58,7 +58,7 @@ namespace Lucene.Net.Util.Fst /// /// @lucene.experimental /// - public sealed class FST : FST + public sealed class FST { /*/// /// Specifies allowed range of each int input label for @@ -70,7 +70,7 @@ public enum INPUT_TYPE BYTE2, BYTE4 }*/ - public readonly INPUT_TYPE inputType; + private readonly FST.INPUT_TYPE inputType; /*internal static readonly int BIT_FINAL_ARC = 1 << 0; internal static readonly int BIT_LAST_ARC = 1 << 1; @@ -154,21 +154,21 @@ public enum INPUT_TYPE private readonly T NO_OUTPUT; - public long nodeCount; - public long arcCount; - public long arcWithOutputCount; + internal long nodeCount; + private long arcCount; + private long arcWithOutputCount; private readonly bool packed; private PackedInts.Reader nodeRefToAddress; - /// - /// If arc has this label then that arc is final/accepted - public static readonly int END_LABEL = -1; + ///// + ///// If arc has this label then that arc is final/accepted + //public static readonly int END_LABEL = -1; private readonly bool allowArrayArcs; - private Arc[] cachedRootArcs; - private Arc[] assertingCachedRootArcs; // only set wit assert + private FST.Arc[] cachedRootArcs; + private FST.Arc[] assertingCachedRootArcs; // only set wit assert // LUCENENET NOTE: Arc moved into FST class @@ -188,12 +188,12 @@ internal static bool Flag(int flags, int bit) // make a new empty FST, for building; Builder invokes // this ctor - internal FST(INPUT_TYPE inputType, Outputs outputs, bool willPackFST, float acceptableOverheadRatio, bool allowArrayArcs, int bytesPageBits) + internal FST(FST.INPUT_TYPE inputType, Outputs outputs, bool willPackFST, float acceptableOverheadRatio, bool allowArrayArcs, int bytesPageBits) { this.inputType = inputType; this.Outputs = outputs; this.allowArrayArcs = allowArrayArcs; - version = VERSION_CURRENT; + version = FST.VERSION_CURRENT; Bytes = new BytesStore(bytesPageBits); // pad: ensure no node gets address 0 which is reserved to mean // the stop state w/ no arcs @@ -239,7 +239,7 @@ public FST(DataInput @in, Outputs outputs, int maxBlockBits) // NOTE: only reads most recent format; we don't have // back-compat promise for FSTs (they are experimental): - version = CodecUtil.CheckHeader(@in, FILE_FORMAT_NAME, VERSION_PACKED, VERSION_VINT_TARGET); + version = CodecUtil.CheckHeader(@in, FST.FILE_FORMAT_NAME, FST.VERSION_PACKED, FST.VERSION_VINT_TARGET); packed = @in.ReadByte() == 1; if (@in.ReadByte() == 1) { @@ -250,7 +250,7 @@ public FST(DataInput @in, Outputs outputs, int maxBlockBits) emptyBytes.CopyBytes(@in, numBytes); // De-serialize empty-string output: - BytesReader reader; + FST.BytesReader reader; if (packed) { reader = emptyBytes.ForwardReader; @@ -276,15 +276,15 @@ public FST(DataInput @in, Outputs outputs, int maxBlockBits) switch (t) { case 0: - inputType = INPUT_TYPE.BYTE1; + inputType = FST.INPUT_TYPE.BYTE1; break; case 1: - inputType = INPUT_TYPE.BYTE2; + inputType = FST.INPUT_TYPE.BYTE2; break; case 2: - inputType = INPUT_TYPE.BYTE4; + inputType = FST.INPUT_TYPE.BYTE4; break; default: @@ -325,7 +325,7 @@ public FST(DataInput @in, Outputs outputs, int maxBlockBits) */ } - public INPUT_TYPE InputType + public FST.INPUT_TYPE InputType { get { @@ -356,7 +356,7 @@ public void Finish(long newStartNode) { throw new InvalidOperationException("already finished"); } - if (newStartNode == FINAL_END_NODE && emptyOutput != null) + if (newStartNode == FST.FINAL_END_NODE && emptyOutput != null) { newStartNode = 0; } @@ -383,27 +383,27 @@ private long GetNodeAddress(long node) // Caches first 128 labels private void CacheRootArcs() { - cachedRootArcs = (Arc[])new Arc[0x80]; + cachedRootArcs = (FST.Arc[])new FST.Arc[0x80]; ReadRootArcs(cachedRootArcs); Debug.Assert(SetAssertingRootArcs(cachedRootArcs)); Debug.Assert(AssertRootArcs()); } - public void ReadRootArcs(Arc[] arcs) + public void ReadRootArcs(FST.Arc[] arcs) { - Arc arc = new Arc(); + FST.Arc arc = new FST.Arc(); GetFirstArc(arc); - BytesReader @in = BytesReader; + FST.BytesReader @in = BytesReader; if (TargetHasArcs(arc)) { ReadFirstRealTargetArc(arc.Target, arc, @in); while (true) { - Debug.Assert(arc.Label != END_LABEL); + Debug.Assert(arc.Label != FST.END_LABEL); if (arc.Label < cachedRootArcs.Length) { - arcs[arc.Label] = (new Arc()).CopyFrom(arc); + arcs[arc.Label] = (new FST.Arc()).CopyFrom(arc); } else { @@ -418,9 +418,9 @@ public void ReadRootArcs(Arc[] arcs) } } - private bool SetAssertingRootArcs(Arc[] arcs) + private bool SetAssertingRootArcs(FST.Arc[] arcs) { - assertingCachedRootArcs = (Arc[])new Arc[arcs.Length]; + assertingCachedRootArcs = (FST.Arc[])new FST.Arc[arcs.Length]; ReadRootArcs(assertingCachedRootArcs); return true; } @@ -431,8 +431,8 @@ private bool AssertRootArcs() Debug.Assert(assertingCachedRootArcs != null); for (int i = 0; i < cachedRootArcs.Length; i++) { - Arc root = cachedRootArcs[i]; - Arc asserting = assertingCachedRootArcs[i]; + FST.Arc root = cachedRootArcs[i]; + FST.Arc asserting = assertingCachedRootArcs[i]; if (root != null) { Debug.Assert(root.ArcIdx == asserting.ArcIdx); @@ -503,7 +503,7 @@ public void Save(DataOutput @out) { throw new InvalidOperationException("cannot save a FST which has been loaded from disk "); } - CodecUtil.WriteHeader(@out, FILE_FORMAT_NAME, VERSION_CURRENT); + CodecUtil.WriteHeader(@out, FST.FILE_FORMAT_NAME, FST.VERSION_CURRENT); if (packed) { @out.WriteByte(1); @@ -547,11 +547,11 @@ public void Save(DataOutput @out) @out.WriteByte(0); } sbyte t; - if (inputType == INPUT_TYPE.BYTE1) + if (inputType == FST.INPUT_TYPE.BYTE1) { t = 0; } - else if (inputType == INPUT_TYPE.BYTE2) + else if (inputType == FST.INPUT_TYPE.BYTE2) { t = 1; } @@ -598,41 +598,17 @@ public void Save(FileInfo file) } } - /// - /// Reads an automaton from a file. - /// - public static FST Read(FileInfo file, Outputs outputs) - { - var bs = new BufferedStream(file.OpenRead()); - bool success = false; - try - { - FST fst = new FST(new InputStreamDataInput(bs), outputs); - success = true; - return fst; - } - finally - { - if (success) - { - IOUtils.Close(bs); - } - else - { - IOUtils.CloseWhileHandlingException(bs); - } - } - } + // LUCENENET NOTE: static Read() was moved into the FST class private void WriteLabel(DataOutput @out, int v) { Debug.Assert(v >= 0, "v=" + v); - if (inputType == INPUT_TYPE.BYTE1) + if (inputType == FST.INPUT_TYPE.BYTE1) { Debug.Assert(v <= 255, "v=" + v); @out.WriteByte((byte)(sbyte)v); } - else if (inputType == INPUT_TYPE.BYTE2) + else if (inputType == FST.INPUT_TYPE.BYTE2) { Debug.Assert(v <= 65535, "v=" + v); @out.WriteShort((short)v); @@ -646,12 +622,12 @@ private void WriteLabel(DataOutput @out, int v) internal int ReadLabel(DataInput @in) { int v; - if (inputType == INPUT_TYPE.BYTE1) + if (inputType == FST.INPUT_TYPE.BYTE1) { // Unsigned byte: v = @in.ReadByte() & 0xFF; } - else if (inputType == INPUT_TYPE.BYTE2) + else if (inputType == FST.INPUT_TYPE.BYTE2) { // Unsigned short: v = @in.ReadShort() & 0xFFFF; @@ -667,7 +643,7 @@ internal int ReadLabel(DataInput @in) /// returns true if the node at this address has any /// outgoing arcs /// - public static bool TargetHasArcs(Arc arc) + public static bool TargetHasArcs(FST.Arc arc) { return arc.Target > 0; } @@ -681,11 +657,11 @@ public long AddNode(Builder.UnCompiledNode nodeIn) { if (nodeIn.IsFinal) { - return FINAL_END_NODE; + return FST.FINAL_END_NODE; } else { - return NON_FINAL_END_NODE; + return FST.NON_FINAL_END_NODE; } } @@ -717,7 +693,7 @@ public long AddNode(Builder.UnCompiledNode nodeIn) if (arcIdx == lastArc) { - flags += BIT_LAST_ARC; + flags += FST.BIT_LAST_ARC; } if (lastFrozenNode == target.Node && !doFixedArray) @@ -725,15 +701,15 @@ public long AddNode(Builder.UnCompiledNode nodeIn) // TODO: for better perf (but more RAM used) we // could avoid this except when arc is "near" the // last arc: - flags += BIT_TARGET_NEXT; + flags += FST.BIT_TARGET_NEXT; } if (arc.IsFinal) { - flags += BIT_FINAL_ARC; + flags += FST.BIT_FINAL_ARC; if (!arc.NextFinalOutput.Equals(NO_OUTPUT)) { - flags += BIT_ARC_HAS_FINAL_OUTPUT; + flags += FST.BIT_ARC_HAS_FINAL_OUTPUT; } } else @@ -745,7 +721,7 @@ public long AddNode(Builder.UnCompiledNode nodeIn) if (!targetHasArcs) { - flags += BIT_STOP_NODE; + flags += FST.BIT_STOP_NODE; } else if (inCounts != null) { @@ -754,7 +730,7 @@ public long AddNode(Builder.UnCompiledNode nodeIn) if (!arc.Output.Equals(NO_OUTPUT)) { - flags += BIT_ARC_HAS_OUTPUT; + flags += FST.BIT_ARC_HAS_OUTPUT; } Bytes.WriteByte((byte)(sbyte)flags); @@ -775,7 +751,7 @@ public long AddNode(Builder.UnCompiledNode nodeIn) Outputs.WriteFinalOutput(arc.NextFinalOutput, Bytes); } - if (targetHasArcs && (flags & BIT_TARGET_NEXT) == 0) + if (targetHasArcs && (flags & FST.BIT_TARGET_NEXT) == 0) { Debug.Assert(target.Node > 0); //System.out.println(" write target"); @@ -827,7 +803,7 @@ public long AddNode(Builder.UnCompiledNode nodeIn) byte[] header = new byte[MAX_HEADER_SIZE]; var bad = new ByteArrayDataOutput(header); // write a "false" first arc: - bad.WriteByte((byte)ARCS_AS_FIXED_ARRAY); + bad.WriteByte((byte)FST.ARCS_AS_FIXED_ARRAY); bad.WriteVInt(nodeIn.NumArcs); bad.WriteVInt(maxBytesPerArc); int headerLen = bad.Position; @@ -898,20 +874,20 @@ public long AddNode(Builder.UnCompiledNode nodeIn) /// Fills virtual 'start' arc, ie, an empty incoming arc to /// the FST's start node /// - public Arc GetFirstArc(Arc arc) + public FST.Arc GetFirstArc(FST.Arc arc) { if (emptyOutput != null) { - arc.Flags = BIT_FINAL_ARC | BIT_LAST_ARC; + arc.Flags = FST.BIT_FINAL_ARC | FST.BIT_LAST_ARC; arc.NextFinalOutput = emptyOutput; if (!emptyOutput.Equals(NO_OUTPUT)) { - arc.Flags |= BIT_ARC_HAS_FINAL_OUTPUT; + arc.Flags |= FST.BIT_ARC_HAS_FINAL_OUTPUT; } } else { - arc.Flags = BIT_LAST_ARC; + arc.Flags = FST.BIT_LAST_ARC; arc.NextFinalOutput = NO_OUTPUT; } arc.Output = NO_OUTPUT; @@ -929,17 +905,17 @@ public Arc GetFirstArc(Arc arc) /// /// Returns the second argument /// (arc). - public Arc ReadLastTargetArc(Arc follow, Arc arc, FST.BytesReader @in) + public FST.Arc ReadLastTargetArc(FST.Arc follow, FST.Arc arc, FST.BytesReader @in) { //System.out.println("readLast"); if (!TargetHasArcs(follow)) { //System.out.println(" end node"); Debug.Assert(follow.IsFinal); - arc.Label = END_LABEL; - arc.Target = FINAL_END_NODE; + arc.Label = FST.END_LABEL; + arc.Target = FST.FINAL_END_NODE; arc.Output = follow.NextFinalOutput; - arc.Flags = (sbyte)BIT_LAST_ARC; + arc.Flags = (sbyte)FST.BIT_LAST_ARC; return arc; } else @@ -947,11 +923,11 @@ public Arc ReadLastTargetArc(Arc follow, Arc arc, FST.BytesReader @in) @in.Position = GetNodeAddress(follow.Target); arc.Node = follow.Target; var b = (sbyte)@in.ReadByte(); - if (b == ARCS_AS_FIXED_ARRAY) + if (b == FST.ARCS_AS_FIXED_ARRAY) { // array: jump straight to end arc.NumArcs = @in.ReadVInt(); - if (packed || version >= VERSION_VINT_TARGET) + if (packed || version >= FST.VERSION_VINT_TARGET) { arc.BytesPerArc = @in.ReadVInt(); } @@ -973,18 +949,18 @@ public Arc ReadLastTargetArc(Arc follow, Arc arc, FST.BytesReader @in) { // skip this arc: ReadLabel(@in); - if (arc.Flag(BIT_ARC_HAS_OUTPUT)) + if (arc.Flag(FST.BIT_ARC_HAS_OUTPUT)) { Outputs.Read(@in); } - if (arc.Flag(BIT_ARC_HAS_FINAL_OUTPUT)) + if (arc.Flag(FST.BIT_ARC_HAS_FINAL_OUTPUT)) { Outputs.ReadFinalOutput(@in); } - if (arc.Flag(BIT_STOP_NODE)) + if (arc.Flag(FST.BIT_STOP_NODE)) { } - else if (arc.Flag(BIT_TARGET_NEXT)) + else if (arc.Flag(FST.BIT_TARGET_NEXT)) { } else if (packed) @@ -1007,10 +983,10 @@ public Arc ReadLastTargetArc(Arc follow, Arc arc, FST.BytesReader @in) } } - private long ReadUnpackedNodeTarget(BytesReader @in) + private long ReadUnpackedNodeTarget(FST.BytesReader @in) { long target; - if (version < VERSION_VINT_TARGET) + if (version < FST.VERSION_VINT_TARGET) { target = @in.ReadInt(); } @@ -1027,19 +1003,19 @@ private long ReadUnpackedNodeTarget(BytesReader @in) /// it. /// /// Returns the second argument (arc). - public Arc ReadFirstTargetArc(Arc follow, Arc arc, FST.BytesReader @in) + public FST.Arc ReadFirstTargetArc(FST.Arc follow, FST.Arc arc, FST.BytesReader @in) { //int pos = address; //System.out.println(" readFirstTarget follow.target=" + follow.Target + " isFinal=" + follow.isFinal()); if (follow.IsFinal) { // Insert "fake" final first arc: - arc.Label = END_LABEL; + arc.Label = FST.END_LABEL; arc.Output = follow.NextFinalOutput; - arc.Flags = (sbyte)BIT_FINAL_ARC; + arc.Flags = (sbyte)FST.BIT_FINAL_ARC; if (follow.Target <= 0) { - arc.Flags |= (sbyte)BIT_LAST_ARC; + arc.Flags |= (sbyte)FST.BIT_LAST_ARC; } else { @@ -1047,7 +1023,7 @@ public Arc ReadFirstTargetArc(Arc follow, Arc arc, FST.BytesReader @in) // NOTE: nextArc is a node (not an address!) in this case: arc.NextArc = follow.Target; } - arc.Target = FINAL_END_NODE; + arc.Target = FST.FINAL_END_NODE; //System.out.println(" insert isFinal; nextArc=" + follow.Target + " isLast=" + arc.isLast() + " output=" + outputs.outputToString(arc.Output)); return arc; } @@ -1057,7 +1033,7 @@ public Arc ReadFirstTargetArc(Arc follow, Arc arc, FST.BytesReader @in) } } - public Arc ReadFirstRealTargetArc(long node, Arc arc, FST.BytesReader @in) + public FST.Arc ReadFirstRealTargetArc(long node, FST.Arc arc, FST.BytesReader @in) { long address = GetNodeAddress(node); @in.Position = address; @@ -1066,12 +1042,12 @@ public Arc ReadFirstRealTargetArc(long node, Arc arc, FST.BytesReader @in) //System.out.println(" flags=" + arc.flags); arc.Node = node; - if (@in.ReadByte() == ARCS_AS_FIXED_ARRAY) + if (@in.ReadByte() == FST.ARCS_AS_FIXED_ARRAY) { //System.out.println(" fixedArray"); // this is first arc in a fixed-array arc.NumArcs = @in.ReadVInt(); - if (packed || version >= VERSION_VINT_TARGET) + if (packed || version >= FST.VERSION_VINT_TARGET) { arc.BytesPerArc = @in.ReadVInt(); } @@ -1098,7 +1074,7 @@ public Arc ReadFirstRealTargetArc(long node, Arc arc, FST.BytesReader @in) /// /// Returns true if arc points to a state in an /// expanded array format. - public bool IsExpandedTarget(Arc follow, BytesReader @in) + public bool IsExpandedTarget(FST.Arc follow, FST.BytesReader @in) { if (!TargetHasArcs(follow)) { @@ -1107,15 +1083,15 @@ public bool IsExpandedTarget(Arc follow, BytesReader @in) else { @in.Position = GetNodeAddress(follow.Target); - return @in.ReadByte() == ARCS_AS_FIXED_ARRAY; + return @in.ReadByte() == FST.ARCS_AS_FIXED_ARRAY; } } /// /// In-place read; returns the arc. - public Arc ReadNextArc(Arc arc, FST.BytesReader @in) + public FST.Arc ReadNextArc(FST.Arc arc, FST.BytesReader @in) { - if (arc.Label == END_LABEL) + if (arc.Label == FST.END_LABEL) { // this was a fake inserted "final" arc if (arc.NextArc <= 0) @@ -1134,11 +1110,11 @@ public Arc ReadNextArc(Arc arc, FST.BytesReader @in) /// Peeks at next arc's label; does not alter arc. Do /// not call this if arc.isLast()! /// - public int ReadNextArcLabel(Arc arc, BytesReader @in) + public int ReadNextArcLabel(FST.Arc arc, FST.BytesReader @in) { Debug.Assert(!arc.IsLast); - if (arc.Label == END_LABEL) + if (arc.Label == FST.END_LABEL) { //System.out.println(" nextArc fake " + //arc.nextArc); @@ -1147,13 +1123,13 @@ public int ReadNextArcLabel(Arc arc, BytesReader @in) @in.Position = pos; var b = (sbyte)@in.ReadByte(); - if (b == ARCS_AS_FIXED_ARRAY) + if (b == FST.ARCS_AS_FIXED_ARRAY) { //System.out.println(" nextArc fixed array"); @in.ReadVInt(); // Skip bytesPerArc: - if (packed || version >= VERSION_VINT_TARGET) + if (packed || version >= FST.VERSION_VINT_TARGET) { @in.ReadVInt(); } @@ -1192,7 +1168,7 @@ public int ReadNextArcLabel(Arc arc, BytesReader @in) /// Never returns null, but you should never call this if /// arc.isLast() is true. /// - public Arc ReadNextRealArc(Arc arc, FST.BytesReader @in) + public FST.Arc ReadNextRealArc(FST.Arc arc, FST.BytesReader @in) { // TODO: can't assert this because we call from readFirstArc // assert !flag(arc.flags, BIT_LAST_ARC); @@ -1214,7 +1190,7 @@ public Arc ReadNextRealArc(Arc arc, FST.BytesReader @in) arc.Flags = (sbyte)@in.ReadByte(); arc.Label = ReadLabel(@in); - if (arc.Flag(BIT_ARC_HAS_OUTPUT)) + if (arc.Flag(FST.BIT_ARC_HAS_OUTPUT)) { arc.Output = Outputs.Read(@in); } @@ -1223,7 +1199,7 @@ public Arc ReadNextRealArc(Arc arc, FST.BytesReader @in) arc.Output = Outputs.NoOutput; } - if (arc.Flag(BIT_ARC_HAS_FINAL_OUTPUT)) + if (arc.Flag(FST.BIT_ARC_HAS_FINAL_OUTPUT)) { arc.NextFinalOutput = Outputs.ReadFinalOutput(@in); } @@ -1232,26 +1208,26 @@ public Arc ReadNextRealArc(Arc arc, FST.BytesReader @in) arc.NextFinalOutput = Outputs.NoOutput; } - if (arc.Flag(BIT_STOP_NODE)) + if (arc.Flag(FST.BIT_STOP_NODE)) { - if (arc.Flag(BIT_FINAL_ARC)) + if (arc.Flag(FST.BIT_FINAL_ARC)) { - arc.Target = FINAL_END_NODE; + arc.Target = FST.FINAL_END_NODE; } else { - arc.Target = NON_FINAL_END_NODE; + arc.Target = FST.NON_FINAL_END_NODE; } arc.NextArc = @in.Position; } - else if (arc.Flag(BIT_TARGET_NEXT)) + else if (arc.Flag(FST.BIT_TARGET_NEXT)) { arc.NextArc = @in.Position; // TODO: would be nice to make this lazy -- maybe // caller doesn't need the target and is scanning arcs... if (nodeAddress == null) { - if (!arc.Flag(BIT_LAST_ARC)) + if (!arc.Flag(FST.BIT_LAST_ARC)) { if (arc.BytesPerArc == 0) { @@ -1278,7 +1254,7 @@ public Arc ReadNextRealArc(Arc arc, FST.BytesReader @in) { long pos = @in.Position; long code = @in.ReadVLong(); - if (arc.Flag(BIT_TARGET_DELTA)) + if (arc.Flag(FST.BIT_TARGET_DELTA)) { // Address is delta-coded from current address: arc.Target = pos + code; @@ -1313,15 +1289,15 @@ public Arc ReadNextRealArc(Arc arc, FST.BytesReader @in) /// Finds an arc leaving the incoming arc, replacing the arc in place. /// this returns null if the arc was not found, else the incoming arc. /// - public Arc FindTargetArc(int labelToMatch, Arc follow, Arc arc, FST.BytesReader @in) + public FST.Arc FindTargetArc(int labelToMatch, FST.Arc follow, FST.Arc arc, FST.BytesReader @in) { - if (labelToMatch == END_LABEL) + if (labelToMatch == FST.END_LABEL) { if (follow.IsFinal) { if (follow.Target <= 0) { - arc.Flags = (sbyte)BIT_LAST_ARC; + arc.Flags = (sbyte)FST.BIT_LAST_ARC; } else { @@ -1331,7 +1307,7 @@ public Arc FindTargetArc(int labelToMatch, Arc follow, Arc arc, FST.Byt arc.Node = follow.Target; } arc.Output = follow.NextFinalOutput; - arc.Label = END_LABEL; + arc.Label = FST.END_LABEL; return arc; } else @@ -1346,7 +1322,7 @@ public Arc FindTargetArc(int labelToMatch, Arc follow, Arc arc, FST.Byt // LUCENE-5152: detect tricky cases where caller // modified previously returned cached root-arcs: Debug.Assert(AssertRootArcs()); - Arc result = cachedRootArcs[labelToMatch]; + FST.Arc result = cachedRootArcs[labelToMatch]; if (result == null) { return null; @@ -1369,11 +1345,11 @@ public Arc FindTargetArc(int labelToMatch, Arc follow, Arc arc, FST.Byt // System.out.println("fta label=" + (char) labelToMatch); - if (@in.ReadByte() == ARCS_AS_FIXED_ARRAY) + if (@in.ReadByte() == FST.ARCS_AS_FIXED_ARRAY) { // Arcs are full array; do binary search: arc.NumArcs = @in.ReadVInt(); - if (packed || version >= VERSION_VINT_TARGET) + if (packed || version >= FST.VERSION_VINT_TARGET) { arc.BytesPerArc = @in.ReadVInt(); } @@ -1447,17 +1423,17 @@ private void SeekToNextNode(FST.BytesReader @in) int flags = @in.ReadByte(); ReadLabel(@in); - if (Flag(flags, BIT_ARC_HAS_OUTPUT)) + if (Flag(flags, FST.BIT_ARC_HAS_OUTPUT)) { Outputs.Read(@in); } - if (Flag(flags, BIT_ARC_HAS_FINAL_OUTPUT)) + if (Flag(flags, FST.BIT_ARC_HAS_FINAL_OUTPUT)) { Outputs.ReadFinalOutput(@in); } - if (!Flag(flags, BIT_STOP_NODE) && !Flag(flags, BIT_TARGET_NEXT)) + if (!Flag(flags, FST.BIT_STOP_NODE) && !Flag(flags, FST.BIT_TARGET_NEXT)) { if (packed) { @@ -1469,7 +1445,7 @@ private void SeekToNextNode(FST.BytesReader @in) } } - if (Flag(flags, BIT_LAST_ARC)) + if (Flag(flags, FST.BIT_LAST_ARC)) { return; } @@ -1517,7 +1493,7 @@ public long ArcWithOutputCount /// private bool ShouldExpand(Builder.UnCompiledNode node) { - return allowArrayArcs && ((node.Depth <= FIXED_ARRAY_SHALLOW_DISTANCE && node.NumArcs >= FIXED_ARRAY_NUM_ARCS_SHALLOW) || node.NumArcs >= FIXED_ARRAY_NUM_ARCS_DEEP); + return allowArrayArcs && ((node.Depth <= FST.FIXED_ARRAY_SHALLOW_DISTANCE && node.NumArcs >= FST.FIXED_ARRAY_NUM_ARCS_SHALLOW) || node.NumArcs >= FST.FIXED_ARRAY_NUM_ARCS_DEEP); } /// @@ -1671,9 +1647,9 @@ public void countSingleChains() throws IOException { */ // Creates a packed FST - private FST(INPUT_TYPE inputType, Outputs outputs, int bytesPageBits) + private FST(FST.INPUT_TYPE inputType, Outputs outputs, int bytesPageBits) { - version = VERSION_CURRENT; + version = FST.VERSION_CURRENT; packed = true; this.inputType = inputType; Bytes = new BytesStore(bytesPageBits); @@ -1720,24 +1696,24 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve throw new System.ArgumentException("this FST was not built with willPackFST=true"); } - Arc arc = new Arc(); + FST.Arc arc = new FST.Arc(); - BytesReader r = BytesReader; + FST.BytesReader r = BytesReader; int topN = Math.Min(maxDerefNodes, inCounts.Size()); // Find top nodes with highest number of incoming arcs: - NodeQueue q = new NodeQueue(topN); + FST.NodeQueue q = new FST.NodeQueue(topN); // TODO: we could use more RAM efficient selection algo here... - NodeAndInCount bottom = null; + FST.NodeAndInCount bottom = null; for (int node = 0; node < inCounts.Size(); node++) { if (inCounts.Get(node) >= minInCountDeref) { if (bottom == null) { - q.Add(new NodeAndInCount(node, (int)inCounts.Get(node))); + q.Add(new FST.NodeAndInCount(node, (int)inCounts.Get(node))); if (q.Size() == topN) { bottom = q.Top(); @@ -1745,7 +1721,7 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve } else if (inCounts.Get(node) > bottom.Count) { - q.InsertWithOverflow(new NodeAndInCount(node, (int)inCounts.Get(node))); + q.InsertWithOverflow(new FST.NodeAndInCount(node, (int)inCounts.Get(node))); } } } @@ -1756,7 +1732,7 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve IDictionary topNodeMap = new Dictionary(); for (int downTo = q.Size() - 1; downTo >= 0; downTo--) { - NodeAndInCount n = q.Pop(); + FST.NodeAndInCount n = q.Pop(); topNodeMap[n.Node] = downTo; //System.out.println("map node=" + n.Node + " inCount=" + n.count + " to newID=" + downTo); } @@ -1846,7 +1822,7 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve { bytesPerArc = arc.BytesPerArc; } - writer.WriteByte((byte)ARCS_AS_FIXED_ARRAY); + writer.WriteByte((byte)FST.ARCS_AS_FIXED_ARRAY); writer.WriteVInt(arc.NumArcs); writer.WriteVInt(bytesPerArc); //System.out.println("node " + node + ": " + arc.numArcs + " arcs"); @@ -1865,7 +1841,7 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve if (arc.IsLast) { - flags += (sbyte)BIT_LAST_ARC; + flags += (sbyte)FST.BIT_LAST_ARC; } /* if (!useArcArray && nodeUpto < nodes.length-1 && arc.Target == nodes[nodeUpto+1]) { @@ -1874,7 +1850,7 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve */ if (!useArcArray && node != 1 && arc.Target == node - 1) { - flags += (sbyte)BIT_TARGET_NEXT; + flags += (sbyte)FST.BIT_TARGET_NEXT; if (!retry) { nextCount++; @@ -1882,10 +1858,10 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve } if (arc.IsFinal) { - flags += (sbyte)BIT_FINAL_ARC; + flags += (sbyte)FST.BIT_FINAL_ARC; if (!arc.NextFinalOutput.Equals(NO_OUTPUT)) { - flags += (sbyte)BIT_ARC_HAS_FINAL_OUTPUT; + flags += (sbyte)FST.BIT_ARC_HAS_FINAL_OUTPUT; } } else @@ -1894,16 +1870,16 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve } if (!TargetHasArcs(arc)) { - flags += (sbyte)BIT_STOP_NODE; + flags += (sbyte)FST.BIT_STOP_NODE; } if (!arc.Output.Equals(NO_OUTPUT)) { - flags += (sbyte)BIT_ARC_HAS_OUTPUT; + flags += (sbyte)FST.BIT_ARC_HAS_OUTPUT; } long absPtr; - bool doWriteTarget = TargetHasArcs(arc) && (flags & BIT_TARGET_NEXT) == 0; + bool doWriteTarget = TargetHasArcs(arc) && (flags & FST.BIT_TARGET_NEXT) == 0; if (doWriteTarget) { int ptr; @@ -1926,7 +1902,7 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve if (delta < absPtr) { - flags |= BIT_TARGET_DELTA; + flags |= FST.BIT_TARGET_DELTA; } } else @@ -1934,7 +1910,7 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve absPtr = 0; } - Debug.Assert(flags != ARCS_AS_FIXED_ARRAY); + Debug.Assert(flags != FST.ARCS_AS_FIXED_ARRAY); writer.WriteByte((byte)(sbyte)flags); fst.WriteLabel(writer, arc.Label); @@ -1962,7 +1938,7 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve delta = 0; } - if (Flag(flags, BIT_TARGET_DELTA)) + if (Flag(flags, FST.BIT_TARGET_DELTA)) { //System.out.println(" delta"); writer.WriteVLong(delta); @@ -2100,11 +2076,14 @@ internal FST Pack(int minInCountDeref, int maxDerefNodes, float acceptableOve } /// - /// .NET Port: This new base class is to mimic Java's ability to use nested types without specifying + /// LUCENENET specific: This new base class is to mimic Java's ability to use nested types without specifying /// a type parameter. i.e. FST.BytesReader instead of FST<BytesRef>.BytesReader /// - public class FST + public sealed class FST { + public FST() + { } + internal const int BIT_FINAL_ARC = 1 << 0; internal const int BIT_LAST_ARC = 1 << 1; internal const int BIT_TARGET_NEXT = 1 << 2; @@ -2177,6 +2156,10 @@ public class FST // non-final node w/ no arcs: internal const long NON_FINAL_END_NODE = 0; + /// + /// If arc has this label then that arc is final/accepted + public static readonly int END_LABEL = -1; + // returns true if the node at this address has any // outgoing arcs public static bool TargetHasArcs(Arc arc) @@ -2184,6 +2167,32 @@ public static bool TargetHasArcs(Arc arc) return arc.Target > 0; } + /// + /// Reads an automaton from a file. + /// + public static FST Read(FileInfo file, Outputs outputs) + { + var bs = new BufferedStream(file.OpenRead()); + bool success = false; + try + { + FST fst = new FST(new InputStreamDataInput(bs), outputs); + success = true; + return fst; + } + finally + { + if (success) + { + IOUtils.Close(bs); + } + else + { + IOUtils.CloseWhileHandlingException(bs); + } + } + } + /// /// Reads bytes stored in an FST. /// diff --git a/src/Lucene.Net.Core/Util/Fst/FSTEnum.cs b/src/Lucene.Net.Core/Util/Fst/FSTEnum.cs index dfdaf67bef..89b36a2163 100644 --- a/src/Lucene.Net.Core/Util/Fst/FSTEnum.cs +++ b/src/Lucene.Net.Core/Util/Fst/FSTEnum.cs @@ -30,14 +30,14 @@ public abstract class FSTEnum { protected internal readonly FST fst; - protected internal FST.Arc[] arcs = new FST.Arc[10]; + protected internal FST.Arc[] arcs = new FST.Arc[10]; // outputs are cumulative protected internal T[] output = new T[10]; protected internal readonly T NO_OUTPUT; - protected internal readonly FST.BytesReader fstReader; - protected internal readonly FST.Arc scratchArc = new FST.Arc(); + protected internal readonly FST.BytesReader fstReader; + protected internal readonly FST.Arc scratchArc = new FST.Arc(); protected internal int upto; protected internal int targetLength; @@ -91,7 +91,7 @@ protected internal void RewindPrefix() else if (cmp > 0) { // seek backwards -- reset this arc to the first arc - FST.Arc arc = GetArc(upto); + FST.Arc arc = GetArc(upto); fst.ReadFirstTargetArc(GetArc(upto - 1), arc, fstReader); //System.out.println(" seek first arc"); break; @@ -150,7 +150,7 @@ protected internal virtual void DoSeekCeil() RewindPrefix(); //System.out.println(" after rewind upto=" + upto); - FST.Arc arc = GetArc(upto); + FST.Arc arc = GetArc(upto); int targetLabel = TargetLabel; //System.out.println(" init targetLabel=" + targetLabel); @@ -164,7 +164,7 @@ protected internal virtual void DoSeekCeil() // Arcs are fixed array -- use binary search to find // the target. - FST.BytesReader @in = fst.BytesReader; + FST.BytesReader @in = fst.BytesReader; int low = arc.ArcIdx; int high = arc.NumArcs - 1; int mid = 0; @@ -203,7 +203,7 @@ protected internal virtual void DoSeekCeil() Debug.Assert(arc.ArcIdx == mid); Debug.Assert(arc.Label == targetLabel, "arc.label=" + arc.Label + " vs targetLabel=" + targetLabel + " mid=" + mid); output[upto] = fst.Outputs.Add(output[upto - 1], arc.Output); - if (targetLabel == FST.END_LABEL) + if (targetLabel == FST.END_LABEL) { return; } @@ -228,7 +228,7 @@ protected internal virtual void DoSeekCeil() { return; } - FST.Arc prevArc = GetArc(upto); + FST.Arc prevArc = GetArc(upto); //System.out.println(" rollback upto=" + upto + " arc.label=" + prevArc.label + " isLast?=" + prevArc.isLast()); if (!prevArc.IsLast) { @@ -255,7 +255,7 @@ protected internal virtual void DoSeekCeil() { // recurse output[upto] = fst.Outputs.Add(output[upto - 1], arc.Output); - if (targetLabel == FST.END_LABEL) + if (targetLabel == FST.END_LABEL) { return; } @@ -280,7 +280,7 @@ protected internal virtual void DoSeekCeil() { return; } - FST.Arc prevArc = GetArc(upto); + FST.Arc prevArc = GetArc(upto); //System.out.println(" rollback upto=" + upto + " arc.label=" + prevArc.label + " isLast?=" + prevArc.isLast()); if (!prevArc.IsLast) { @@ -319,7 +319,7 @@ protected internal virtual void DoSeekFloor() //System.out.println("FE: after rewind upto=" + upto); - FST.Arc arc = GetArc(upto); + FST.Arc arc = GetArc(upto); int targetLabel = TargetLabel; //System.out.println("FE: init targetLabel=" + targetLabel); @@ -329,12 +329,12 @@ protected internal virtual void DoSeekFloor() { //System.out.println(" cycle upto=" + upto + " arc.label=" + arc.label + " (" + (char) arc.label + ") targetLabel=" + targetLabel + " isLast?=" + arc.isLast() + " bba=" + arc.bytesPerArc); - if (arc.BytesPerArc != 0 && arc.Label != FST.END_LABEL) + if (arc.BytesPerArc != 0 && arc.Label != FST.END_LABEL) { // Arcs are fixed array -- use binary search to find // the target. - FST.BytesReader @in = fst.BytesReader; + FST.BytesReader @in = fst.BytesReader; int low = arc.ArcIdx; int high = arc.NumArcs - 1; int mid = 0; @@ -374,7 +374,7 @@ protected internal virtual void DoSeekFloor() Debug.Assert(arc.ArcIdx == mid); Debug.Assert(arc.Label == targetLabel, "arc.label=" + arc.Label + " vs targetLabel=" + targetLabel + " mid=" + mid); output[upto] = fst.Outputs.Add(output[upto - 1], arc.Output); - if (targetLabel == FST.END_LABEL) + if (targetLabel == FST.END_LABEL) { return; } @@ -435,7 +435,7 @@ protected internal virtual void DoSeekFloor() { // Match -- recurse output[upto] = fst.Outputs.Add(output[upto - 1], arc.Output); - if (targetLabel == FST.END_LABEL) + if (targetLabel == FST.END_LABEL) { return; } @@ -514,15 +514,15 @@ protected internal virtual bool DoSeekExact() RewindPrefix(); //System.out.println("FE: after rewind upto=" + upto); - FST.Arc arc = GetArc(upto - 1); + FST.Arc arc = GetArc(upto - 1); int targetLabel = TargetLabel; - FST.BytesReader fstReader = fst.BytesReader; + FST.BytesReader fstReader = fst.BytesReader; while (true) { //System.out.println(" cycle target=" + (targetLabel == -1 ? "-1" : (char) targetLabel)); - FST.Arc nextArc = fst.FindTargetArc(targetLabel, arc, GetArc(upto), fstReader); + FST.Arc nextArc = fst.FindTargetArc(targetLabel, arc, GetArc(upto), fstReader); if (nextArc == null) { // short circuit @@ -534,7 +534,7 @@ protected internal virtual bool DoSeekExact() } // Match -- recurse: output[upto] = fst.Outputs.Add(output[upto - 1], nextArc.Output); - if (targetLabel == FST.END_LABEL) + if (targetLabel == FST.END_LABEL) { //System.out.println(" return found; upto=" + upto + " output=" + output[upto] + " nextArc=" + nextArc.isLast()); return true; @@ -552,7 +552,7 @@ private void Incr() Grow(); if (arcs.Length <= upto) { - FST.Arc[] newArcs = new FST.Arc[ArrayUtil.Oversize(1 + upto, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; + FST.Arc[] newArcs = new FST.Arc[ArrayUtil.Oversize(1 + upto, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; Array.Copy(arcs, 0, newArcs, 0, arcs.Length); arcs = newArcs; } @@ -568,13 +568,13 @@ private void Incr() // appending first arc all the way to the final node private void PushFirst() { - FST.Arc arc = arcs[upto]; + FST.Arc arc = arcs[upto]; Debug.Assert(arc != null); while (true) { output[upto] = fst.Outputs.Add(output[upto - 1], arc.Output); - if (arc.Label == FST.END_LABEL) + if (arc.Label == FST.END_LABEL) { // Final node break; @@ -583,7 +583,7 @@ private void PushFirst() CurrentLabel = arc.Label; Incr(); - FST.Arc nextArc = GetArc(upto); + FST.Arc nextArc = GetArc(upto); fst.ReadFirstTargetArc(arc, nextArc, fstReader); arc = nextArc; } @@ -593,14 +593,14 @@ private void PushFirst() // way to the first final node private void PushLast() { - FST.Arc arc = arcs[upto]; + FST.Arc arc = arcs[upto]; Debug.Assert(arc != null); while (true) { CurrentLabel = arc.Label; output[upto] = fst.Outputs.Add(output[upto - 1], arc.Output); - if (arc.Label == FST.END_LABEL) + if (arc.Label == FST.END_LABEL) { // Final node break; @@ -611,11 +611,11 @@ private void PushLast() } } - private FST.Arc GetArc(int idx) + private FST.Arc GetArc(int idx) { if (arcs[idx] == null) { - arcs[idx] = new FST.Arc(); + arcs[idx] = new FST.Arc(); } return arcs[idx]; } diff --git a/src/Lucene.Net.Core/Util/Fst/IntsRefFSTEnum.cs b/src/Lucene.Net.Core/Util/Fst/IntsRefFSTEnum.cs index f679a6f186..370b5a90f9 100644 --- a/src/Lucene.Net.Core/Util/Fst/IntsRefFSTEnum.cs +++ b/src/Lucene.Net.Core/Util/Fst/IntsRefFSTEnum.cs @@ -105,7 +105,7 @@ protected internal override int TargetLabel { if (upto - 1 == target.Length) { - return FST.END_LABEL; + return FST.END_LABEL; } else { diff --git a/src/Lucene.Net.Core/Util/Fst/NodeHash.cs b/src/Lucene.Net.Core/Util/Fst/NodeHash.cs index a0fb6599ad..95b6115884 100644 --- a/src/Lucene.Net.Core/Util/Fst/NodeHash.cs +++ b/src/Lucene.Net.Core/Util/Fst/NodeHash.cs @@ -30,7 +30,7 @@ internal sealed class NodeHash private long count; private long mask; private readonly FST fst; - private readonly FST.Arc scratchArc = new FST.Arc(); + private readonly FST.Arc scratchArc = new FST.Arc(); private readonly FST.BytesReader input; public NodeHash(FST fst, FST.BytesReader input) diff --git a/src/Lucene.Net.Core/Util/Fst/Util.cs b/src/Lucene.Net.Core/Util/Fst/Util.cs index baf648f30a..15a80b7d52 100644 --- a/src/Lucene.Net.Core/Util/Fst/Util.cs +++ b/src/Lucene.Net.Core/Util/Fst/Util.cs @@ -43,7 +43,7 @@ private Util() public static T Get(FST fst, IntsRef input) { // TODO: would be nice not to alloc this on every lookup - var arc = fst.GetFirstArc(new FST.Arc()); + var arc = fst.GetFirstArc(new FST.Arc()); var fstReader = fst.BytesReader; @@ -76,12 +76,12 @@ public static T Get(FST fst, IntsRef input) /// public static T Get(FST fst, BytesRef input) { - Debug.Assert(fst.inputType == FST.INPUT_TYPE.BYTE1); + Debug.Assert(fst.InputType == FST.INPUT_TYPE.BYTE1); var fstReader = fst.BytesReader; // TODO: would be nice not to alloc this on every lookup - var arc = fst.GetFirstArc(new FST.Arc()); + var arc = fst.GetFirstArc(new FST.Arc()); // Accumulate output as we go T output = fst.Outputs.NoOutput; @@ -123,9 +123,9 @@ public static IntsRef GetByOutput(FST fst, long targetOutput) var @in = fst.BytesReader; // TODO: would be nice not to alloc this on every lookup - FST.Arc arc = fst.GetFirstArc(new FST.Arc()); + FST.Arc arc = fst.GetFirstArc(new FST.Arc()); - FST.Arc scratchArc = new FST.Arc(); + FST.Arc scratchArc = new FST.Arc(); IntsRef result = new IntsRef(); @@ -136,7 +136,7 @@ public static IntsRef GetByOutput(FST fst, long targetOutput) /// Expert: like except reusing /// BytesReader, initial and scratch Arc, and result. /// - public static IntsRef GetByOutput(FST fst, long targetOutput, FST.BytesReader @in, FST.Arc arc, FST.Arc scratchArc, IntsRef result) + public static IntsRef GetByOutput(FST fst, long targetOutput, FST.BytesReader @in, FST.Arc arc, FST.Arc scratchArc, IntsRef result) { long output = arc.Output.Value; int upto = 0; @@ -188,7 +188,7 @@ public static IntsRef GetByOutput(FST fst, long targetOutput, FST. var flags = (sbyte)@in.ReadByte(); fst.ReadLabel(@in); long minArcOutput; - if ((flags & FST.BIT_ARC_HAS_OUTPUT) != 0) + if ((flags & FST.BIT_ARC_HAS_OUTPUT) != 0) { long arcOutput = fst.Outputs.Read(@in).Value; minArcOutput = output + arcOutput; @@ -231,7 +231,7 @@ public static IntsRef GetByOutput(FST fst, long targetOutput, FST. } else { - FST.Arc prevArc = null; + FST.Arc prevArc = null; while (true) { @@ -300,15 +300,15 @@ public static IntsRef GetByOutput(FST fst, long targetOutput, FST. /// public class FSTPath { - public FST.Arc Arc; + public FST.Arc Arc; public T Cost; public readonly IntsRef Input; /// /// Sole constructor - public FSTPath(T cost, FST.Arc arc, IntsRef input) + public FSTPath(T cost, FST.Arc arc, IntsRef input) { - this.Arc = (new FST.Arc()).CopyFrom(arc); + this.Arc = (new FST.Arc()).CopyFrom(arc); this.Cost = cost; this.Input = input; } @@ -353,11 +353,11 @@ public virtual int Compare(FSTPath a, FSTPath b) public class TopNSearcher { private readonly FST fst; - private readonly FST.BytesReader bytesReader; + private readonly FST.BytesReader bytesReader; private readonly int topN; private readonly int maxQueueDepth; - private readonly FST.Arc scratchArc = new FST.Arc(); + private readonly FST.Arc scratchArc = new FST.Arc(); internal readonly IComparer Comparator; @@ -441,7 +441,7 @@ protected virtual void AddIfCompetitive(FSTPath path) /// Adds all leaving arcs, including 'finished' arc, if /// the node is final, from this node into the queue. /// - public virtual void AddStartPaths(FST.Arc node, T startOutput, bool allowEmptyString, IntsRef input) + public virtual void AddStartPaths(FST.Arc node, T startOutput, bool allowEmptyString, IntsRef input) { // De-dup NO_OUTPUT since it must be a singleton: if (startOutput.Equals(fst.Outputs.NoOutput)) @@ -457,7 +457,7 @@ public virtual void AddStartPaths(FST.Arc node, T startOutput, bool allowE // Bootstrap: find the min starting arc while (true) { - if (allowEmptyString || path.Arc.Label != FST.END_LABEL) + if (allowEmptyString || path.Arc.Label != FST.END_LABEL) { AddIfCompetitive(path); } @@ -511,7 +511,7 @@ public virtual TopResults Search() break; } - if (path.Arc.Label == FST.END_LABEL) + if (path.Arc.Label == FST.END_LABEL) { //System.out.println(" empty string! cost=" + path.cost); // Empty string! @@ -586,7 +586,7 @@ public virtual TopResults Search() path.Arc.CopyFrom(scratchArc); } - if (path.Arc.Label == FST.END_LABEL) + if (path.Arc.Label == FST.END_LABEL) { // Add final output: //System.out.println(" done!: " + path); @@ -674,7 +674,7 @@ System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() /// Starting from node, find the top N min cost /// completions to a final node. /// - public static TopResults ShortestPaths(FST fst, FST.Arc fromNode, T startOutput, IComparer comparator, int topN, bool allowEmptyString) + public static TopResults ShortestPaths(FST fst, FST.Arc fromNode, T startOutput, IComparer comparator, int topN, bool allowEmptyString) { // All paths are kept, so we can pass topN for // maxQueueDepth and the pruning is admissible: @@ -723,13 +723,13 @@ public static void ToDot(FST fst, TextWriter @out, bool sameRank, bool lab // this is the start arc in the automaton (from the epsilon state to the first state // with outgoing transitions. - FST.Arc startArc = fst.GetFirstArc(new FST.Arc()); + FST.Arc startArc = fst.GetFirstArc(new FST.Arc()); // A queue of transitions to consider for the next level. - IList.Arc> thisLevelQueue = new List.Arc>(); + IList> thisLevelQueue = new List>(); // A queue of transitions to consider when processing the next level. - IList.Arc> nextLevelQueue = new List.Arc>(); + IList> nextLevelQueue = new List>(); nextLevelQueue.Add(startArc); //System.out.println("toDot: startArc: " + startArc); @@ -802,7 +802,7 @@ public static void ToDot(FST fst, TextWriter @out, bool sameRank, bool lab @out.Write("\n // Transitions and states at level: " + level + "\n"); while (thisLevelQueue.Count > 0) { - FST.Arc arc = thisLevelQueue[thisLevelQueue.Count - 1]; + FST.Arc arc = thisLevelQueue[thisLevelQueue.Count - 1]; thisLevelQueue.RemoveAt(thisLevelQueue.Count - 1); //System.out.println(" pop: " + arc); if (FST.TargetHasArcs(arc)) @@ -857,7 +857,7 @@ public static void ToDot(FST fst, TextWriter @out, bool sameRank, bool lab // To see the node address, use this instead: //emitDotState(out, Integer.toString(arc.target), stateShape, stateColor, String.valueOf(arc.target)); seen.SafeSet((int)arc.Target, true); - nextLevelQueue.Add((new FST.Arc()).CopyFrom(arc)); + nextLevelQueue.Add((new FST.Arc()).CopyFrom(arc)); sameLevelStates.Add((int)arc.Target); } @@ -883,7 +883,7 @@ public static void ToDot(FST fst, TextWriter @out, bool sameRank, bool lab } string arcColor; - if (arc.Flag(FST.BIT_TARGET_NEXT)) + if (arc.Flag(FST.BIT_TARGET_NEXT)) { arcColor = "red"; } @@ -892,7 +892,7 @@ public static void ToDot(FST fst, TextWriter @out, bool sameRank, bool lab arcColor = "black"; } - Debug.Assert(arc.Label != FST.END_LABEL); + Debug.Assert(arc.Label != FST.END_LABEL); @out.Write(" " + node + " -> " + arc.Target + " [label=\"" + PrintableLabel(arc.Label) + outs + "\"" + (arc.IsFinal ? " style=\"bold\"" : "") + " color=\"" + arcColor + "\"]\n"); // Break the loop if we're on the last arc of this state. @@ -1062,16 +1062,16 @@ public static BytesRef ToBytesRef(IntsRef input, BytesRef scratch) /// the arc to follow reading the label from /// the arc to read into in place /// the fst's - public static FST.Arc ReadCeilArc(int label, FST fst, FST.Arc follow, FST.Arc arc, FST.BytesReader @in) + public static FST.Arc ReadCeilArc(int label, FST fst, FST.Arc follow, FST.Arc arc, FST.BytesReader @in) { // TODO maybe this is a useful in the FST class - we could simplify some other code like FSTEnum? - if (label == FST.END_LABEL) + if (label == FST.END_LABEL) { if (follow.IsFinal) { if (follow.Target <= 0) { - arc.Flags = (sbyte)FST.BIT_LAST_ARC; + arc.Flags = (sbyte)FST.BIT_LAST_ARC; } else { @@ -1081,7 +1081,7 @@ public static FST.Arc ReadCeilArc(int label, FST fst, FST.Arc arc.Node = follow.Target; } arc.Output = follow.NextFinalOutput; - arc.Label = FST.END_LABEL; + arc.Label = FST.END_LABEL; return arc; } else @@ -1095,7 +1095,7 @@ public static FST.Arc ReadCeilArc(int label, FST fst, FST.Arc return null; } fst.ReadFirstTargetArc(follow, arc, @in); - if (arc.BytesPerArc != 0 && arc.Label != FST.END_LABEL) + if (arc.BytesPerArc != 0 && arc.Label != FST.END_LABEL) { // Arcs are fixed array -- use binary search to find // the target. diff --git a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs index cb5f44dbed..492dca4dd3 100644 --- a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs +++ b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs @@ -217,7 +217,7 @@ public virtual void DoTest(bool testPruning) private T Run(FST fst, IntsRef term, int[] prefixLength) { Debug.Assert(prefixLength == null || prefixLength.Length == 1); - FST.Arc arc = fst.GetFirstArc(new FST.Arc()); + FST.Arc arc = fst.GetFirstArc(new FST.Arc()); T NO_OUTPUT = fst.Outputs.NoOutput; T output = NO_OUTPUT; FST.BytesReader fstReader = fst.BytesReader; @@ -227,7 +227,7 @@ private T Run(FST fst, IntsRef term, int[] prefixLength) int label; if (i == term.Length) { - label = FST.END_LABEL; + label = FST.END_LABEL; } else { @@ -288,7 +288,7 @@ private T RandomAcceptedWord(FST fst, IntsRef @in) output = fst.Outputs.Add(output, arc.Output); // append label - if (arc.Label == FST.END_LABEL) + if (arc.Label == FST.END_LABEL) { break; } From 8617aa9997568d9fc72743f68784b60f9d2ddbd2 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Thu, 8 Sep 2016 00:06:47 +0700 Subject: [PATCH 15/47] Fixed infinite recursion issue and key not found bugs in the FSTTester --- .../Util/fst/FSTTester.cs | 107 +++++++++++++++--- 1 file changed, 93 insertions(+), 14 deletions(-) diff --git a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs index 492dca4dd3..ccd23797fe 100644 --- a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs +++ b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs @@ -1,15 +1,15 @@ +using Lucene.Net.Randomized.Generators; +using Lucene.Net.Support; +using NUnit.Framework; using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics; +using System.IO; +using System.Linq; namespace Lucene.Net.Util.Fst { - using Lucene.Net.Randomized.Generators; - using Lucene.Net.Support; - using NUnit.Framework; - using System.IO; - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -312,7 +312,17 @@ internal virtual FST DoTest(int prune1, int prune2, bool allowRandomSuffixSha bool willRewrite = Random.NextBoolean(); - Builder builder = new Builder(InputMode == 0 ? FST.INPUT_TYPE.BYTE1 : FST.INPUT_TYPE.BYTE4, prune1, prune2, prune1 == 0 && prune2 == 0, allowRandomSuffixSharing ? Random.NextBoolean() : true, allowRandomSuffixSharing ? TestUtil.NextInt(Random, 1, 10) : int.MaxValue, Outputs, null, willRewrite, PackedInts.DEFAULT, true, 15); + Builder builder = new Builder(InputMode == 0 ? FST.INPUT_TYPE.BYTE1 : FST.INPUT_TYPE.BYTE4, + prune1, prune2, + prune1 == 0 && prune2 == 0, + allowRandomSuffixSharing ? Random.NextBoolean() : true, + allowRandomSuffixSharing ? TestUtil.NextInt(Random, 1, 10) : int.MaxValue, + Outputs, + null, + willRewrite, + PackedInts.DEFAULT, + true, + 15); if (LuceneTestCase.VERBOSE) { if (willRewrite) @@ -823,7 +833,7 @@ private void VerifyPruned(int inputMode, FST fst, int prune1, int prune2) //System.out.println("TEST: tally prefixes"); // build all prefixes - IDictionary> prefixes = new Dictionary>(); + IDictionary> prefixes = new HashMap>(); IntsRef scratch = new IntsRef(10); foreach (InputOutput pair in Pairs) { @@ -867,11 +877,13 @@ private void VerifyPruned(int inputMode, FST fst, int prune1, int prune2) Console.WriteLine("TEST: now prune"); } + // prune 'em - IEnumerator>> it = prefixes.GetEnumerator(); - while (it.MoveNext()) + // LUCENENET NOTE: Altered this a bit to go in reverse rather than use an enumerator since + // in .NET you cannot delete records while enumerating forward through a dictionary. + for (int i = prefixes.Count - 1; i >= 0; i--) { - KeyValuePair> ent = it.Current; + KeyValuePair> ent = prefixes.ElementAt(i); IntsRef prefix = ent.Key; CountMinOutput cmo = ent.Value; if (LuceneTestCase.VERBOSE) @@ -895,7 +907,7 @@ private void VerifyPruned(int inputMode, FST fst, int prune1, int prune2) // consult our parent scratch.Length = prefix.Length - 1; Array.Copy(prefix.Ints, prefix.Offset, scratch.Ints, 0, scratch.Length); - CountMinOutput cmo2 = prefixes[scratch]; + CountMinOutput cmo2 = prefixes.ContainsKey(scratch) ? prefixes[scratch] : null; //System.out.println(" parent count = " + (cmo2 == null ? -1 : cmo2.count)); keep = cmo2 != null && ((prune2 > 1 && cmo2.Count >= prune2) || (prune2 == 1 && (cmo2.Count >= 2 || prefix.Length <= 1))); } @@ -911,7 +923,7 @@ private void VerifyPruned(int inputMode, FST fst, int prune1, int prune2) if (!keep) { - it.Reset(); + prefixes.Remove(prefix); //System.out.println(" remove"); } else @@ -922,7 +934,7 @@ private void VerifyPruned(int inputMode, FST fst, int prune1, int prune2) scratch.Length--; while (scratch.Length >= 0) { - CountMinOutput cmo2 = prefixes[scratch]; + CountMinOutput cmo2 = prefixes.ContainsKey(scratch) ? prefixes[scratch] : null; if (cmo2 != null) { //System.out.println(" clear isLeaf " + inputToString(inputMode, scratch)); @@ -933,6 +945,73 @@ private void VerifyPruned(int inputMode, FST fst, int prune1, int prune2) } } + + //// prune 'em + //IEnumerator>> it = prefixes.GetEnumerator(); + //while (it.MoveNext()) + //{ + // KeyValuePair> ent = it.Current; + // IntsRef prefix = ent.Key; + // CountMinOutput cmo = ent.Value; + // if (LuceneTestCase.VERBOSE) + // { + // Console.WriteLine(" term prefix=" + InputToString(inputMode, prefix, false) + " count=" + cmo.Count + " isLeaf=" + cmo.IsLeaf + " output=" + Outputs.OutputToString(cmo.Output) + " isFinal=" + cmo.IsFinal); + // } + // bool keep; + // if (prune1 > 0) + // { + // keep = cmo.Count >= prune1; + // } + // else + // { + // Debug.Assert(prune2 > 0); + // if (prune2 > 1 && cmo.Count >= prune2) + // { + // keep = true; + // } + // else if (prefix.Length > 0) + // { + // // consult our parent + // scratch.Length = prefix.Length - 1; + // Array.Copy(prefix.Ints, prefix.Offset, scratch.Ints, 0, scratch.Length); + // CountMinOutput cmo2 = prefixes.ContainsKey(scratch) ? prefixes[scratch] : null; + // //System.out.println(" parent count = " + (cmo2 == null ? -1 : cmo2.count)); + // keep = cmo2 != null && ((prune2 > 1 && cmo2.Count >= prune2) || (prune2 == 1 && (cmo2.Count >= 2 || prefix.Length <= 1))); + // } + // else if (cmo.Count >= prune2) + // { + // keep = true; + // } + // else + // { + // keep = false; + // } + // } + + // if (!keep) + // { + // it.Reset(); + // //System.out.println(" remove"); + // } + // else + // { + // // clear isLeaf for all ancestors + // //System.out.println(" keep"); + // scratch.CopyInts(prefix); + // scratch.Length--; + // while (scratch.Length >= 0) + // { + // CountMinOutput cmo2 = prefixes.ContainsKey(scratch) ? prefixes[scratch] : null; + // if (cmo2 != null) + // { + // //System.out.println(" clear isLeaf " + inputToString(inputMode, scratch)); + // cmo2.IsLeaf = false; + // } + // scratch.Length--; + // } + // } + //} + if (LuceneTestCase.VERBOSE) { Console.WriteLine("TEST: after prune"); @@ -967,7 +1046,7 @@ private void VerifyPruned(int inputMode, FST fst, int prune1, int prune2) { Console.WriteLine(" fstEnum.next prefix=" + InputToString(inputMode, current.Input, false) + " output=" + Outputs.OutputToString(current.Output)); } - CountMinOutput cmo = prefixes[current.Input]; + CountMinOutput cmo = prefixes.ContainsKey(current.Input) ? prefixes[current.Input] : null; Assert.IsNotNull(cmo); Assert.IsTrue(cmo.IsLeaf || cmo.IsFinal); //if (cmo.isFinal && !cmo.isLeaf) { From d807eeba7a8cdbb3efc54a00e12312ea0f11b308 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Thu, 8 Sep 2016 00:46:08 +0700 Subject: [PATCH 16/47] Core.Util.Fst.TestFSTs: Fixed infinite recursion bug in TestRealTerms(). Use Environment.TickCount rather than DateTime.Now.Millisecond when doing time sensitive testing. --- src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs b/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs index bb4d206b82..3ff21be2e4 100644 --- a/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs +++ b/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs @@ -333,10 +333,10 @@ public virtual void TestRealTerms() DirectoryInfo tempDir = CreateTempDir("fstlines"); Directory dir = NewFSDirectory(tempDir); IndexWriter writer = new IndexWriter(dir, conf); - long stopTime = DateTime.Now.Millisecond + RUN_TIME_MSEC; + long stopTime = Environment.TickCount + RUN_TIME_MSEC; Document doc; int docCount = 0; - while ((doc = docs.NextDoc()) != null && DateTime.Now.Millisecond < stopTime) + while ((doc = docs.NextDoc()) != null && Environment.TickCount < stopTime) { writer.AddDocument(doc); docCount++; @@ -542,7 +542,7 @@ public virtual void Run(int limit, bool verify, bool verifyByOutput) try { IntsRef intsRef = new IntsRef(10); - long tStart = DateTime.Now.Millisecond; + long tStart = Environment.TickCount; int ord = 0; while (true) { @@ -557,7 +557,7 @@ public virtual void Run(int limit, bool verify, bool verifyByOutput) ord++; if (ord % 500000 == 0) { - Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "{0:000000.000}s: {1:000000000}...", ((DateTime.Now.Millisecond - tStart) / 1000.0), ord)); + Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "{0:000000.000}s: {1:000000000}...", ((Environment.TickCount - tStart) / 1000.0), ord)); } if (ord >= limit) { @@ -565,12 +565,12 @@ public virtual void Run(int limit, bool verify, bool verifyByOutput) } } - long tMid = DateTime.Now.Millisecond; + long tMid = Environment.TickCount; Console.WriteLine(((tMid - tStart) / 1000.0) + " sec to add all terms"); Debug.Assert(Builder.TermCount == ord); FST fst = Builder.Finish(); - long tEnd = DateTime.Now.Millisecond; + long tEnd = Environment.TickCount; Console.WriteLine(((tEnd - tMid) / 1000.0) + " sec to finish/pack"); if (fst == null) { @@ -619,7 +619,7 @@ public virtual void Run(int limit, bool verify, bool verifyByOutput) @is = new StreamReader(new FileStream(WordsFileIn, FileMode.Open), Encoding.UTF8); ord = 0; - tStart = DateTime.Now.Millisecond; + tStart = Environment.TickCount; while (true) { string w = @is.ReadLine(); @@ -659,7 +659,7 @@ public virtual void Run(int limit, bool verify, bool verifyByOutput) ord++; if (ord % 500000 == 0) { - Console.WriteLine(((DateTime.Now.Millisecond - tStart) / 1000.0) + "s: " + ord + "..."); + Console.WriteLine(((Environment.TickCount - tStart) / 1000.0) + "s: " + ord + "..."); } if (ord >= limit) { @@ -667,7 +667,7 @@ public virtual void Run(int limit, bool verify, bool verifyByOutput) } } - double totSec = ((DateTime.Now.Millisecond - tStart) / 1000.0); + double totSec = ((Environment.TickCount - tStart) / 1000.0); Console.WriteLine("Verify " + (iter == 1 ? "(by output) " : "") + "took " + totSec + " sec + (" + (int)((totSec * 1000000000 / ord)) + " nsec per lookup)"); if (!verifyByOutput) From 32e2105bba349544c3166d30f664ef7edd219a3a Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Thu, 8 Sep 2016 02:19:29 +0700 Subject: [PATCH 17/47] TestFSTs: Added LongRunningTest and Ignore attributes for applicable tests. --- .../core/Util/Fst/TestFSTs.cs | 61 +++++++++---------- 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs b/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs index 3ff21be2e4..8dde68999c 100644 --- a/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs +++ b/src/Lucene.Net.Tests/core/Util/Fst/TestFSTs.cs @@ -1,10 +1,14 @@ +using Lucene.Net.Attributes; +using Lucene.Net.Randomized.Generators; +using Lucene.Net.Support; +using NUnit.Framework; using System; -using System.Diagnostics; using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; using System.IO; using System.Linq; using System.Text; -using Lucene.Net.Randomized.Generators; namespace Lucene.Net.Util.Fst { @@ -25,46 +29,38 @@ namespace Lucene.Net.Util.Fst * limitations under the License. */ - using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; + //using Slow = Lucene.Net.Util.LuceneTestCase.Slow; + using Automaton = Lucene.Net.Util.Automaton.Automaton; + //using InputOutput = Lucene.Net.Util.Fst.BytesRefFSTEnum.InputOutput; + //using Arc = Lucene.Net.Util.Fst.FST.Arc; + using BytesReader = Lucene.Net.Util.Fst.FST.BytesReader; + using CompiledAutomaton = Lucene.Net.Util.Automaton.CompiledAutomaton; + using Directory = Lucene.Net.Store.Directory; + using DirectoryReader = Lucene.Net.Index.DirectoryReader; using Document = Lucene.Net.Documents.Document; using Field = Lucene.Net.Documents.Field; - using DirectoryReader = Lucene.Net.Index.DirectoryReader; + using FSDirectory = Lucene.Net.Store.FSDirectory; + using IndexInput = Lucene.Net.Store.IndexInput; + using IndexOutput = Lucene.Net.Store.IndexOutput; using IndexReader = Lucene.Net.Index.IndexReader; + using IndexSearcher = Lucene.Net.Search.IndexSearcher; using IndexWriter = Lucene.Net.Index.IndexWriter; using IndexWriterConfig = Lucene.Net.Index.IndexWriterConfig; - using MultiFields = Lucene.Net.Index.MultiFields; - using RandomIndexWriter = Lucene.Net.Index.RandomIndexWriter; - using Term = Lucene.Net.Index.Term; - using Terms = Lucene.Net.Index.Terms; - using TermsEnum = Lucene.Net.Index.TermsEnum; - using IndexSearcher = Lucene.Net.Search.IndexSearcher; - using TermQuery = Lucene.Net.Search.TermQuery; - using Directory = Lucene.Net.Store.Directory; - using FSDirectory = Lucene.Net.Store.FSDirectory; using IOContext = Lucene.Net.Store.IOContext; - using IndexInput = Lucene.Net.Store.IndexInput; - using IndexOutput = Lucene.Net.Store.IndexOutput; + using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper; - //using Slow = Lucene.Net.Util.LuceneTestCase.Slow; - using SuppressCodecs = Lucene.Net.Util.LuceneTestCase.SuppressCodecs; - using Automaton = Lucene.Net.Util.Automaton.Automaton; - using CompiledAutomaton = Lucene.Net.Util.Automaton.CompiledAutomaton; - using RegExp = Lucene.Net.Util.Automaton.RegExp; - //using InputOutput = Lucene.Net.Util.Fst.BytesRefFSTEnum.InputOutput; - //using Arc = Lucene.Net.Util.Fst.FST.Arc; - using BytesReader = Lucene.Net.Util.Fst.FST.BytesReader; - using Pair = Lucene.Net.Util.Fst.PairOutputs.Pair; + using MultiFields = Lucene.Net.Index.MultiFields; //using ResultLong = Lucene.Net.Util.Fst.Util.Result; //using ResultPair = Lucene.Net.Util.Fst.Util.Result; using PackedInts = Lucene.Net.Util.Packed.PackedInts; - using Lucene.Net.Support; - using NUnit.Framework; - using System.Globalization; + using Pair = Lucene.Net.Util.Fst.PairOutputs.Pair; + using RandomIndexWriter = Lucene.Net.Index.RandomIndexWriter; + using RegExp = Lucene.Net.Util.Automaton.RegExp; + using Term = Lucene.Net.Index.Term; + using TermQuery = Lucene.Net.Search.TermQuery; + using Terms = Lucene.Net.Index.Terms; + using TermsEnum = Lucene.Net.Index.TermsEnum; - /* - import static Lucene.Net.Util.Fst.FSTTester.GetRandomString; - import static Lucene.Net.Util.Fst.FSTTester.SimpleRandomString; - import static Lucene.Net.Util.Fst.FSTTester.ToIntsRef;*/ [TestFixture] public class TestFSTs : LuceneTestCase @@ -269,7 +265,7 @@ private void DoTest(int inputMode, IntsRef[] terms) } - [Test] + [Test, LongRunningTest, Timeout(27000000)] // 45 minutes to be on the safe side public virtual void TestRandomWords() { TestRandomWords(1000, AtLeast(2)); @@ -313,6 +309,7 @@ private void TestRandomWords(int maxNumWords, int numIter) } [Test] + [Ignore("LUCENENET TODO: This test will take around 10-14 hours to finish. It was marked with a Nightly attribute in the original Java source, but we don't currently have a corresponding attribute")] public virtual void TestBigSet() { TestRandomWords(TestUtil.NextInt(Random(), 50000, 60000), 1); From ef889cebdeaa180d0bb232afd8e5ab3865cd41bc Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Thu, 8 Sep 2016 05:23:58 +0700 Subject: [PATCH 18/47] Fixed issue with SortedSet not being emptied in Core.Util.Fst.Util, and added thread safety, since SortedSet doesn't have the necessary atomic operations. --- src/Lucene.Net.Core/Util/Fst/Util.cs | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/Lucene.Net.Core/Util/Fst/Util.cs b/src/Lucene.Net.Core/Util/Fst/Util.cs index 15a80b7d52..0419b78819 100644 --- a/src/Lucene.Net.Core/Util/Fst/Util.cs +++ b/src/Lucene.Net.Core/Util/Fst/Util.cs @@ -363,6 +363,8 @@ public class TopNSearcher internal SortedSet> Queue = null; + private object syncLock = new object(); + /// /// Creates an unbounded TopNSearcher /// the to search on @@ -433,7 +435,14 @@ protected virtual void AddIfCompetitive(FSTPath path) if (Queue.Count == maxQueueDepth + 1) { - Queue.Last(); + // LUCENENET NOTE: SortedSet doesn't have atomic operations, + // so we need to add some thread safety just in case. + // Perhaps it might make sense to wrap SortedSet into a type + // that provides thread safety. + lock (syncLock) + { + Queue.Remove(Queue.Max); + } } } @@ -502,7 +511,19 @@ public virtual TopResults Search() // Remove top path since we are now going to // pursue it: - path = Queue.First(); + + // LUCENENET NOTE: SortedSet doesn't have atomic operations, + // so we need to add some thread safety just in case. + // Perhaps it might make sense to wrap SortedSet into a type + // that provides thread safety. + lock (syncLock) + { + path = Queue.Min; + if (path != null) + { + Queue.Remove(path); + } + } if (path == null) { From b1203e6b4cb9551fe09999114890e74e77cb6937 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sat, 10 Sep 2016 17:53:32 +0700 Subject: [PATCH 19/47] Ported StressRamUsageEstimator & TestRamUsageEstimator + fixed bugs in RamUsageEstimator --- src/Lucene.Net.Core/Util/RamUsageEstimator.cs | 53 +++++++----- src/Lucene.Net.Tests/Lucene.Net.Tests.csproj | 2 + .../core/Util/StressRamUsageEstimator.cs | 62 +++++--------- .../core/Util/TestRamUsageEstimator.cs | 85 +++++++++---------- 4 files changed, 98 insertions(+), 104 deletions(-) diff --git a/src/Lucene.Net.Core/Util/RamUsageEstimator.cs b/src/Lucene.Net.Core/Util/RamUsageEstimator.cs index d3f289bea3..07f603743e 100644 --- a/src/Lucene.Net.Core/Util/RamUsageEstimator.cs +++ b/src/Lucene.Net.Core/Util/RamUsageEstimator.cs @@ -111,7 +111,8 @@ static RamUsageEstimator() int arrayHeader = Constants.JRE_IS_64BIT ? 24 : 12; int objectAlignment = Constants.JRE_IS_64BIT ? 8 : 4; - /* LUCENE-TODO + + /* LUCENENET TODO Type unsafeClass = null; object tempTheUnsafe = null; @@ -138,12 +139,15 @@ static RamUsageEstimator() { // ignore. } + */ - // "best guess" based on reference size. We will attempt to modify - // these to exact values if there is supported infrastructure. - objectHeader = Constants.JRE_IS_64BIT ? (8 + referenceSize) : 8; + // "best guess" based on reference size. We will attempt to modify + // these to exact values if there is supported infrastructure. + objectHeader = Constants.JRE_IS_64BIT ? (8 + referenceSize) : 8; arrayHeader = Constants.JRE_IS_64BIT ? (8 + 2 * referenceSize) : 12; + /* LUCENENET TODO + // get the object header size: // - first try out if the field offsets are not scaled (see warning in Unsafe docs) // - get the object header size by getting the field offset of the first field of a dummy object @@ -187,7 +191,7 @@ static RamUsageEstimator() NUM_BYTES_OBJECT_HEADER = objectHeader; NUM_BYTES_ARRAY_HEADER = arrayHeader; - /* LUCENE-TODO + /* LUCENENET TODO // Try to get the object alignment (the default seems to be 8 on Hotspot, // regardless of the architecture). int objectAlignment = 8; @@ -207,6 +211,7 @@ static RamUsageEstimator() // Ignore. } */ + NUM_BYTES_OBJECT_ALIGNMENT = objectAlignment; JVM_INFO_STRING = "[JVM: " + Constants.JVM_NAME + ", " + Constants.JVM_VERSION + ", " + Constants.JVM_VENDOR + ", " + Constants.JAVA_VENDOR + ", " + Constants.JAVA_VERSION + "]"; @@ -215,7 +220,7 @@ static RamUsageEstimator() /// /// A handle to sun.misc.Unsafe. /// - private static readonly object TheUnsafe; + //private static readonly object TheUnsafe; /// /// A handle to sun.misc.Unsafe#fieldOffset(Field). @@ -237,18 +242,18 @@ public ClassCache(long alignedShallowInstanceSize, FieldInfo[] referenceFields) } } - // Object with just one field to determine the object header size by getting the offset of the dummy field: - private sealed class DummyOneFieldObject - { - public sbyte @base; - } + //// Object with just one field to determine the object header size by getting the offset of the dummy field: + //private sealed class DummyOneFieldObject + //{ + // public sbyte @base; + //} - // Another test object for checking, if the difference in offsets of dummy1 and dummy2 is 8 bytes. - // Only then we can be sure that those are real, unscaled offsets: - private sealed class DummyTwoLongObject - { - public long Dummy1, Dummy2; - } + //// Another test object for checking, if the difference in offsets of dummy1 and dummy2 is 8 bytes. + //// Only then we can be sure that those are real, unscaled offsets: + //private sealed class DummyTwoLongObject + //{ + // public long Dummy1, Dummy2; + //} /// /// Aligns an object size to be the next multiple of . @@ -376,7 +381,12 @@ public static long ShallowSizeOfInstance(Type clazz) // Walk type hierarchy for (; clazz != null; clazz = clazz.BaseType) { - FieldInfo[] fields = clazz.GetFields(BindingFlags.Public); + FieldInfo[] fields = clazz.GetFields( + BindingFlags.Instance | + BindingFlags.NonPublic | + BindingFlags.Public | + BindingFlags.DeclaredOnly | + BindingFlags.Static); foreach (FieldInfo f in fields) { if (!f.IsStatic) @@ -525,7 +535,12 @@ private static ClassCache CreateCacheEntry(Type clazz) List referenceFields = new List(32); for (Type c = clazz; c != null; c = c.BaseType) { - FieldInfo[] fields = c.GetFields(BindingFlags.Public | BindingFlags.Instance); + FieldInfo[] fields = c.GetFields( + BindingFlags.Instance | + BindingFlags.NonPublic | + BindingFlags.Public | + BindingFlags.DeclaredOnly | + BindingFlags.Static); foreach (FieldInfo f in fields) { if (!f.IsStatic) diff --git a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj index 955d5bc6a8..03f336c318 100644 --- a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj +++ b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj @@ -470,6 +470,7 @@ Code + Code @@ -562,6 +563,7 @@ Code + Code diff --git a/src/Lucene.Net.Tests/core/Util/StressRamUsageEstimator.cs b/src/Lucene.Net.Tests/core/Util/StressRamUsageEstimator.cs index 0bc9f75799..c0b8533d4f 100644 --- a/src/Lucene.Net.Tests/core/Util/StressRamUsageEstimator.cs +++ b/src/Lucene.Net.Tests/core/Util/StressRamUsageEstimator.cs @@ -1,10 +1,10 @@ -using System; -using System.Collections.Generic; +using Lucene.Net.Support; using NUnit.Framework; +using System; +using System.Globalization; namespace Lucene.Net.Util { - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -50,8 +50,6 @@ public virtual Entry CreateNext(object o) [Test] public virtual void TestChainedEstimation() { - MemoryMXBean memoryMXBean = ManagementFactory.MemoryMXBean; - Random rnd = Random(); Entry first = new Entry(); try @@ -59,9 +57,9 @@ public virtual void TestChainedEstimation() while (true) { // Check the current memory consumption and provide the estimate. - long jvmUsed = memoryMXBean.HeapMemoryUsage.Used; - long estimated = RamUsageEstimator.sizeOf(first); - Console.WriteLine(string.format(Locale.ROOT, "%10d, %10d", jvmUsed, estimated)); + long jvmUsed = GC.GetTotalMemory(false); + long estimated = RamUsageEstimator.SizeOf(first); + Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "{0:0000000000}, {1:0000000000}", jvmUsed, estimated)); // Make a batch of objects. for (int i = 0; i < 5000; i++) @@ -82,52 +80,48 @@ public virtual void TestChainedEstimation() [Test] public virtual void TestLargeSetOfByteArrays() { - MemoryMXBean memoryMXBean = ManagementFactory.MemoryMXBean; - CauseGc(); - long before = memoryMXBean.HeapMemoryUsage.Used; + long before = GC.GetTotalMemory(false); object[] all = new object[1000000]; for (int i = 0; i < all.Length; i++) { all[i] = new sbyte[Random().Next(3)]; } CauseGc(); - long after = memoryMXBean.HeapMemoryUsage.Used; - Console.WriteLine("mx: " + RamUsageEstimator.humanReadableUnits(after - before)); - Console.WriteLine("rue: " + RamUsageEstimator.humanReadableUnits(ShallowSizeOf(all))); + long after = GC.GetTotalMemory(false); + Console.WriteLine("mx: " + RamUsageEstimator.HumanReadableUnits(after - before)); + Console.WriteLine("rue: " + RamUsageEstimator.HumanReadableUnits(ShallowSizeOf(all))); Guard = all; } private long ShallowSizeOf(object[] all) { - long s = RamUsageEstimator.shallowSizeOf(all); + long s = RamUsageEstimator.ShallowSizeOf(all); foreach (object o in all) { - s += RamUsageEstimator.shallowSizeOf(o); + s += RamUsageEstimator.ShallowSizeOf(o); } return s; } private long ShallowSizeOf(object[][] all) { - long s = RamUsageEstimator.shallowSizeOf(all); + long s = RamUsageEstimator.ShallowSizeOf(all); foreach (object[] o in all) { - s += RamUsageEstimator.shallowSizeOf(o); + s += RamUsageEstimator.ShallowSizeOf(o); foreach (object o2 in o) { - s += RamUsageEstimator.shallowSizeOf(o2); + s += RamUsageEstimator.ShallowSizeOf(o2); } } return s; } - [Test] + [Test, Timeout(180000)] public virtual void TestSimpleByteArrays() { - MemoryMXBean memoryMXBean = ManagementFactory.MemoryMXBean; - object[][] all = new object[0][]; try { @@ -135,18 +129,18 @@ public virtual void TestSimpleByteArrays() { // Check the current memory consumption and provide the estimate. CauseGc(); - MemoryUsage mu = memoryMXBean.HeapMemoryUsage; + long estimated = ShallowSizeOf(all); if (estimated > 50 * RamUsageEstimator.ONE_MB) { break; } - Console.WriteLine(string.format(Locale.ROOT, "%10s\t%10s\t%10s", RamUsageEstimator.humanReadableUnits(mu.Used), RamUsageEstimator.humanReadableUnits(mu.Max), RamUsageEstimator.humanReadableUnits(estimated))); + Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "{0}\t{1}\t{2}", RamUsageEstimator.HumanReadableUnits(GC.GetTotalMemory(false)).PadLeft(10, ' '), RamUsageEstimator.HumanReadableUnits(GC.MaxGeneration).PadLeft(10, ' '), RamUsageEstimator.HumanReadableUnits(estimated).PadLeft(10, ' '))); // Make another batch of objects. object[] seg = new object[10000]; - all = Arrays.copyOf(all, all.Length + 1); + all = Arrays.CopyOf(all, all.Length + 1); all[all.Length - 1] = seg; for (int i = 0; i < seg.Length; i++) { @@ -166,23 +160,7 @@ public virtual void TestSimpleByteArrays() /// private void CauseGc() { - IList garbageCollectorMXBeans = ManagementFactory.GarbageCollectorMXBeans; - IList ccounts = new List(); - foreach (GarbageCollectorMXBean g in garbageCollectorMXBeans) - { - ccounts.Add(g.CollectionCount); - } - IList ccounts2 = new List(); - do - { - System.gc(); - ccounts.Clear(); - foreach (GarbageCollectorMXBean g in garbageCollectorMXBeans) - { - ccounts2.Add(g.CollectionCount); - } - } while (ccounts2.Equals(ccounts)); + GC.Collect(); } } - } \ No newline at end of file diff --git a/src/Lucene.Net.Tests/core/Util/TestRamUsageEstimator.cs b/src/Lucene.Net.Tests/core/Util/TestRamUsageEstimator.cs index 113e730ace..b08739dc77 100644 --- a/src/Lucene.Net.Tests/core/Util/TestRamUsageEstimator.cs +++ b/src/Lucene.Net.Tests/core/Util/TestRamUsageEstimator.cs @@ -1,10 +1,8 @@ -using System; -using Lucene.Net.Util; using NUnit.Framework; +using System; namespace Lucene.Net.Util { - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -25,22 +23,22 @@ namespace Lucene.Net.Util [TestFixture] public class TestRamUsageEstimator : LuceneTestCase { - /*[Test] + [Test] public virtual void TestSanity() { - Assert.IsTrue(sizeof(new string("test string")) > ShallowSizeOfInstance(typeof(string))); + Assert.IsTrue(RamUsageEstimator.SizeOf("test string") > RamUsageEstimator.ShallowSizeOfInstance(typeof(string))); Holder holder = new Holder(); holder.Holder_Renamed = new Holder("string2", 5000L); - Assert.IsTrue(sizeof(holder) > shallowSizeOfInstance(typeof(Holder))); - Assert.IsTrue(sizeof(holder) > sizeof(holder.Holder_Renamed)); + Assert.IsTrue(RamUsageEstimator.SizeOf(holder) > RamUsageEstimator.ShallowSizeOfInstance(typeof(Holder))); + Assert.IsTrue(RamUsageEstimator.SizeOf(holder) > RamUsageEstimator.SizeOf(holder.Holder_Renamed)); - Assert.IsTrue(shallowSizeOfInstance(typeof(HolderSubclass)) >= shallowSizeOfInstance(typeof(Holder))); - Assert.IsTrue(shallowSizeOfInstance(typeof(Holder)) == shallowSizeOfInstance(typeof(HolderSubclass2))); + Assert.IsTrue(RamUsageEstimator.ShallowSizeOfInstance(typeof(HolderSubclass)) >= RamUsageEstimator.ShallowSizeOfInstance(typeof(Holder))); + Assert.IsTrue(RamUsageEstimator.ShallowSizeOfInstance(typeof(Holder)) == RamUsageEstimator.ShallowSizeOfInstance(typeof(HolderSubclass2))); - string[] strings = new string[] { new string("test string"), new string("hollow"), new string("catchmaster") }; - Assert.IsTrue(sizeof(strings) > shallowSizeOf(strings)); - }*/ + string[] strings = new string[] { "test string", "hollow", "catchmaster" }; + Assert.IsTrue(RamUsageEstimator.SizeOf(strings) > RamUsageEstimator.ShallowSizeOf(strings)); + } [Test] public virtual void TestStaticOverloads() @@ -48,73 +46,74 @@ public virtual void TestStaticOverloads() Random rnd = Random(); { sbyte[] array = new sbyte[rnd.Next(1024)]; - Assert.AreEqual(sizeof(array), sizeof((object)array)); + assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array)); } { bool[] array = new bool[rnd.Next(1024)]; - Assert.AreEqual(sizeof(array), sizeof((object)array)); + assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array)); } { char[] array = new char[rnd.Next(1024)]; - Assert.AreEqual(sizeof(array), sizeof((object)array)); + assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array)); } { short[] array = new short[rnd.Next(1024)]; - Assert.AreEqual(sizeof(array), sizeof((object)array)); + assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array)); } { int[] array = new int[rnd.Next(1024)]; - Assert.AreEqual(sizeof(array), sizeof((object)array)); + assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array)); } { float[] array = new float[rnd.Next(1024)]; - Assert.AreEqual(sizeof(array), sizeof((object)array)); + assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array)); } { long[] array = new long[rnd.Next(1024)]; - Assert.AreEqual(sizeof(array), sizeof((object)array)); + assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array)); } { double[] array = new double[rnd.Next(1024)]; - Assert.AreEqual(sizeof(array), sizeof((object)array)); + assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array)); } } - /* LUCENE PORT: The JVM is obviously not relevant to our purposes - [Test] + + [Test] public virtual void TestReferenceSize() { - if (!SupportedJVM) - { - Console.Error.WriteLine("WARN: Your JVM does not support certain Oracle/Sun extensions."); - Console.Error.WriteLine(" Memory estimates may be inaccurate."); - Console.Error.WriteLine(" Please report this to the Lucene mailing list."); - Console.Error.WriteLine("JVM version: " + RamUsageEstimator.JVM_INFO_STRING); - Console.Error.WriteLine("UnsupportedFeatures:"); - foreach (JvmFeature f in RamUsageEstimator.UnsupportedFeatures) - { - Console.Error.Write(" - " + f.ToString()); - if (f == RamUsageEstimator.JvmFeature.OBJECT_ALIGNMENT) - { - Console.Error.Write("; Please note: 32bit Oracle/Sun VMs don't allow exact OBJECT_ALIGNMENT retrieval, this is a known issue."); - } - Console.Error.WriteLine(); - } - } - - Assert.IsTrue(NUM_BYTES_OBJECT_REF == 4 || NUM_BYTES_OBJECT_REF == 8); + //LUCENENET: The JVM is obviously not relevant to our purposes + //if (!SupportedJVM) + //{ + // Console.Error.WriteLine("WARN: Your JVM does not support certain Oracle/Sun extensions."); + // Console.Error.WriteLine(" Memory estimates may be inaccurate."); + // Console.Error.WriteLine(" Please report this to the Lucene mailing list."); + // Console.Error.WriteLine("JVM version: " + RamUsageEstimator.JVM_INFO_STRING); + // Console.Error.WriteLine("UnsupportedFeatures:"); + // foreach (JvmFeature f in RamUsageEstimator.UnsupportedFeatures) + // { + // Console.Error.Write(" - " + f.ToString()); + // if (f == RamUsageEstimator.JvmFeature.OBJECT_ALIGNMENT) + // { + // Console.Error.Write("; Please note: 32bit Oracle/Sun VMs don't allow exact OBJECT_ALIGNMENT retrieval, this is a known issue."); + // } + // Console.Error.WriteLine(); + // } + //} + + Assert.IsTrue(RamUsageEstimator.NUM_BYTES_OBJECT_REF == 4 || RamUsageEstimator.NUM_BYTES_OBJECT_REF == 8); if (!Constants.JRE_IS_64BIT) { - Assert.AreEqual("For 32bit JVMs, reference size must always be 4?", 4, NUM_BYTES_OBJECT_REF); + assertEquals("For 32bit JVMs, reference size must always be 4?", 4, RamUsageEstimator.NUM_BYTES_OBJECT_REF); } - }*/ + } private class Holder { From 878080dcd9d7aa5d705e8fce6d3d6cdad20dfeff Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sat, 10 Sep 2016 17:55:00 +0700 Subject: [PATCH 20/47] Added timeout to long running test (~45 sec) in Core.Util.Packed.TestPackedInts --- src/Lucene.Net.Tests/core/Util/Packed/TestPackedInts.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Lucene.Net.Tests/core/Util/Packed/TestPackedInts.cs b/src/Lucene.Net.Tests/core/Util/Packed/TestPackedInts.cs index 3294f829a7..d1057a9bf3 100644 --- a/src/Lucene.Net.Tests/core/Util/Packed/TestPackedInts.cs +++ b/src/Lucene.Net.Tests/core/Util/Packed/TestPackedInts.cs @@ -1472,7 +1472,7 @@ public virtual void TestMonotonicBlockPackedReaderWriter() } } - [Test] + [Test, Timeout(180000)] public virtual void TestBlockReaderOverflow() { long valueCount = TestUtil.NextLong(Random(), 1L + int.MaxValue, (long)int.MaxValue * 2); From 0c2f5ae9a89eb29ce35835773e308235a5a91579 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 00:02:22 +0700 Subject: [PATCH 21/47] Ported Core.Util.TestIdentityHashSet + fixed bugs in RamUsageEstimator.IdentityHashSet. --- .../Support/Compatibility/Collections.cs | 195 +++++++++++++++++- src/Lucene.Net.Core/Util/RamUsageEstimator.cs | 9 +- src/Lucene.Net.Tests/Lucene.Net.Tests.csproj | 1 + .../core/Util/TestIdentityHashSet.cs | 10 +- 4 files changed, 205 insertions(+), 10 deletions(-) diff --git a/src/Lucene.Net.Core/Support/Compatibility/Collections.cs b/src/Lucene.Net.Core/Support/Compatibility/Collections.cs index 8421b39227..61651733b3 100644 --- a/src/Lucene.Net.Core/Support/Compatibility/Collections.cs +++ b/src/Lucene.Net.Core/Support/Compatibility/Collections.cs @@ -1,6 +1,9 @@ -using System.Collections.Generic; +using System; +using System.Collections; +using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using System.Runtime.Serialization; namespace Lucene.Net { @@ -42,5 +45,195 @@ public static IDictionary SingletonMap(T key, S value) { return new Dictionary {{key, value}}; } + + public static ISet NewSetFromMap(IDictionary map) + { + return new SetFromMap(map); + } + + internal class SetFromMap : ICollection, IEnumerable, IEnumerable, ISerializable, IDeserializationCallback, ISet, IReadOnlyCollection + { + private readonly IDictionary m; // The backing map + [NonSerialized] + private ICollection s; + + internal SetFromMap(IDictionary map) + { + if (map.Any()) + throw new ArgumentException("Map is not empty"); + m = map; + s = map.Keys; + } + + public void Clear() + { + m.Clear(); + } + + public int Count + { + get + { + return m.Count; + } + } + + // LUCENENET: IsEmpty doesn't exist here + + public bool Contains(T item) + { + return m.ContainsKey(item); + } + + public bool Remove(T item) + { + return m.Remove(item); + } + + public bool Add(T item) + { + m.Add(item, true); + return m.ContainsKey(item); + } + + void ICollection.Add(T item) + { + m.Add(item, true); + } + + public IEnumerator GetEnumerator() + { + return s.GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return s.GetEnumerator(); + } + + // LUCENENET: ToArray() is part of LINQ + + public override string ToString() + { + return s.ToString(); + } + + public override int GetHashCode() + { + return s.GetHashCode(); + } + + public override bool Equals(object obj) + { + return obj == this || s.Equals(obj); + } + + public virtual bool ContainsAll(IEnumerable other) + { + // we don't care about order, so sort both sequences before comparing + return this.OrderBy(x => x).SequenceEqual(other.OrderBy(x => x)); + } + + public void CopyTo(T[] array, int arrayIndex) + { + m.Keys.CopyTo(array, arrayIndex); + } + + + public bool IsReadOnly + { + get + { + return false; + } + } + + public bool SetEquals(IEnumerable other) + { + if (other == null) + { + throw new ArgumentNullException("other"); + } + SetFromMap set = other as SetFromMap; + if (set != null) + { + if (this.m.Count != set.Count) + { + return false; + } + return this.ContainsAll(set); + } + ICollection is2 = other as ICollection; + if (((is2 != null) && (this.m.Count == 0)) && (is2.Count > 0)) + { + return false; + } + foreach (var item in this) + { + if (!is2.Contains(item)) + { + return false; + } + } + return true; + } + + #region Not Implemented Members + public void ExceptWith(IEnumerable other) + { + throw new NotImplementedException(); + } + + public void IntersectWith(IEnumerable other) + { + throw new NotImplementedException(); + } + + public bool IsProperSubsetOf(IEnumerable other) + { + throw new NotImplementedException(); + } + + public bool IsProperSupersetOf(IEnumerable other) + { + throw new NotImplementedException(); + } + + public bool IsSubsetOf(IEnumerable other) + { + throw new NotImplementedException(); + } + + public bool IsSupersetOf(IEnumerable other) + { + throw new NotImplementedException(); + } + + public bool Overlaps(IEnumerable other) + { + throw new NotImplementedException(); + } + + public void SymmetricExceptWith(IEnumerable other) + { + throw new NotImplementedException(); + } + + public void UnionWith(IEnumerable other) + { + throw new NotImplementedException(); + } + + public void GetObjectData(SerializationInfo info, StreamingContext context) + { + throw new NotImplementedException(); + } + + public void OnDeserialization(object sender) + { + throw new NotImplementedException(); + } + #endregion + } } } diff --git a/src/Lucene.Net.Core/Util/RamUsageEstimator.cs b/src/Lucene.Net.Core/Util/RamUsageEstimator.cs index 07f603743e..de039bf8e5 100644 --- a/src/Lucene.Net.Core/Util/RamUsageEstimator.cs +++ b/src/Lucene.Net.Core/Util/RamUsageEstimator.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Globalization; using System.Reflection; +using System.Runtime.CompilerServices; namespace Lucene.Net.Util { @@ -717,7 +718,7 @@ public bool Add(KType e) object existing; while ((existing = Keys[slot]) != null) { - if (Object.ReferenceEquals(e, existing)) + if (object.ReferenceEquals(e, existing)) { return false; // already found. } @@ -738,7 +739,7 @@ public bool Contains(KType e) object existing; while ((existing = Keys[slot]) != null) { - if (Object.ReferenceEquals(e, existing)) + if (object.ReferenceEquals(e, existing)) { return true; } @@ -757,7 +758,7 @@ public bool Contains(KType e) /// internal static int Rehash(object o) { - int k = o.GetHashCode(); + int k = RuntimeHelpers.GetHashCode(o); k ^= (int)((uint)k >> 16); k *= unchecked((int)0x85ebca6b); k ^= (int)((uint)k >> 13); @@ -811,7 +812,7 @@ internal void AllocateBuffers(int capacity) /// /// Return the next possible capacity, counting from the current buffers' size. /// - protected internal int NextCapacity(int current) + internal int NextCapacity(int current) { Debug.Assert(current > 0 && ((current & (current - 1)) == 0), "Capacity must be a power of two."); Debug.Assert((current << 1) > 0, "Maximum capacity exceeded (" + ((int)((uint)0x80000000 >> 1)) + ")."); diff --git a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj index 03f336c318..dc54705ff5 100644 --- a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj +++ b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj @@ -520,6 +520,7 @@ Code + Code diff --git a/src/Lucene.Net.Tests/core/Util/TestIdentityHashSet.cs b/src/Lucene.Net.Tests/core/Util/TestIdentityHashSet.cs index b3e420e0bf..bc3fee4247 100644 --- a/src/Lucene.Net.Tests/core/Util/TestIdentityHashSet.cs +++ b/src/Lucene.Net.Tests/core/Util/TestIdentityHashSet.cs @@ -2,10 +2,10 @@ using System.Collections.Generic; using Lucene.Net.Support; using NUnit.Framework; +using System.Linq; namespace Lucene.Net.Util { - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -31,7 +31,8 @@ public class TestIdentityHashSet : LuceneTestCase public virtual void TestCheck() { Random rnd = Random(); - ISet jdk = CollectionsHelper.NewSetFromMap(new IdentityHashMap()); + + ISet jdk = Collections.NewSetFromMap(new IdentityHashMap()); RamUsageEstimator.IdentityHashSet us = new RamUsageEstimator.IdentityHashSet(); int max = 100000; @@ -50,14 +51,13 @@ public virtual void TestCheck() Assert.AreEqual(e1, e2); } - ISet collected = CollectionsHelper.NewSetFromMap(new IdentityHashMap()); + ISet collected = Collections.NewSetFromMap(new IdentityHashMap()); foreach (object o in us) { collected.Add(o); } - Assert.AreEqual(collected, jdk); + assertEquals(collected, jdk); } } - } \ No newline at end of file From 3b18a74a95c21ee4174fcd87c16bea5fe05d6752 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 02:22:30 +0700 Subject: [PATCH 22/47] Ported Core.Util.TestWeakIdentityMap (partially) --- src/Lucene.Net.Tests/Lucene.Net.Tests.csproj | 1 + .../core/Util/TestWeakIdentityMap.cs | 83 ++++++++++++------- 2 files changed, 52 insertions(+), 32 deletions(-) diff --git a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj index dc54705ff5..32dea94cf2 100644 --- a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj +++ b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj @@ -608,6 +608,7 @@ Code + diff --git a/src/Lucene.Net.Tests/core/Util/TestWeakIdentityMap.cs b/src/Lucene.Net.Tests/core/Util/TestWeakIdentityMap.cs index 19939c321a..82c5c4c66e 100644 --- a/src/Lucene.Net.Tests/core/Util/TestWeakIdentityMap.cs +++ b/src/Lucene.Net.Tests/core/Util/TestWeakIdentityMap.cs @@ -1,28 +1,28 @@ -using System; -using System.Collections.Generic; -using System.Threading; -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ using Lucene.Net.Randomized.Generators; using Lucene.Net.Support; using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Threading; namespace Lucene.Net.Util { + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ [TestFixture] public class TestWeakIdentityMap : LuceneTestCase @@ -35,8 +35,14 @@ public virtual void TestSimpleHashMap() // we keep strong references to the keys, // so WeakIdentityMap will not forget about them: string key1 = "foo"; - string key2 = "foo"; - string key3 = "foo"; + string key2 = "test1 foo".Split(' ')[1]; + string key3 = "test2 foo".Split(' ')[1]; + + // LUCENENET NOTE: As per http://stackoverflow.com/a/543329/181087, + // the above hack is required in order to ensure the AreNotSame + // check will work. If you assign the same string to 3 different variables + // without doing some kind of manipulation from the original string, the + // AreNotSame test will fail because the references will be the same. Assert.AreNotSame(key1, key2); Assert.AreEqual(key1, key2); @@ -102,7 +108,8 @@ public virtual void TestSimpleHashMap() //Assert.IsTrue(iter.hasNext()); // try again, should return same result! string k = iter.Current; Assert.IsTrue(k == key1 || k == key2 | k == key3); - keysAssigned += (k == key1) ? 1 : ((k == key2) ? 2 : 4); + // LUCENENET NOTE: Need object.ReferenceEquals here because the == operator does more than check reference equality + keysAssigned += object.ReferenceEquals(k, key1) ? 1 : (object.ReferenceEquals(k, key2) ? 2 : 4); c++; } Assert.AreEqual(3, c); @@ -126,12 +133,13 @@ public virtual void TestSimpleHashMap() { try { - System.RunFinalization(); - System.gc(); + //System.RunFinalization(); + //System.gc(); + GC.Collect(); int newSize = map.Size(); Assert.IsTrue(size >= newSize, "previousSize(" + size + ")>=newSize(" + newSize + ")"); size = newSize; - Thread.Sleep(new TimeSpan(100L)); + Thread.Sleep(new TimeSpan(0, 0, 1)); c = 0; for (IEnumerator iter = map.Keys.GetEnumerator(); iter.MoveNext(); ) { @@ -163,8 +171,14 @@ public virtual void TestSimpleHashMap() { }*/ - key1 = "foo"; - key2 = "foo"; + // LUCENENET NOTE: As per http://stackoverflow.com/a/543329/181087, + // the following hack is required in order to ensure the string references + // are different. If you assign the same string to 2 different variables + // without doing some kind of manipulation from the original string, the + // references will be the same. + + key1 = "test3 foo".Split(' ')[1]; + key2 = "test4 foo".Split(' ')[1]; map.Put(key1, "bar1"); map.Put(key2, "bar2"); Assert.AreEqual(2, map.Size()); @@ -174,12 +188,15 @@ public virtual void TestSimpleHashMap() Assert.IsTrue(map.Empty); } + /* LUCENENET TODO: Compile issues [Test] public virtual void TestConcurrentHashMap() { // don't make threadCount and keyCount random, otherwise easily OOMs or fails otherwise: const int threadCount = 8, keyCount = 1024; - ExecutorService exec = Executors.newFixedThreadPool(threadCount, new NamedThreadFactory("testConcurrentHashMap")); + + System.Threading.ThreadPool.QueueUserWorkItem(; + //ExecutorService exec = Executors.newFixedThreadPool(threadCount, new NamedThreadFactory("testConcurrentHashMap")); WeakIdentityMap map = WeakIdentityMap.NewConcurrentHashMap(Random().NextBoolean()); // we keep strong references to the keys, // so WeakIdentityMap will not forget about them: @@ -215,14 +232,15 @@ public virtual void TestConcurrentHashMap() { try { - System.runFinalization(); - System.gc(); + //System.runFinalization(); + //System.gc(); + GC.Collect(); int newSize = map.Size(); Assert.IsTrue(size >= newSize, "previousSize(" + size + ")>=newSize(" + newSize + ")"); size = newSize; Thread.Sleep(new TimeSpan(100L)); int c = 0; - for (IEnumerator it = map.Keys.GetEnumerator(); it.MoveNext(); ) + for (IEnumerator it = map.Keys.GetEnumerator(); it.MoveNext();) { Assert.IsNotNull(it.Current); c++; @@ -283,7 +301,7 @@ public void Run() break; case 4: // check iterator still working - for (IEnumerator it = Map.Keys.GetEnumerator(); it.MoveNext(); ) + for (IEnumerator it = Map.Keys.GetEnumerator(); it.MoveNext();) { Assert.IsNotNull(it.Current); } @@ -295,5 +313,6 @@ public void Run() } } } + */ } } \ No newline at end of file From e659c99328f5cfd7089166ec6ed30fd98294905e Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 02:23:03 +0700 Subject: [PATCH 23/47] Fixed String.Format bug in Core.Util.NamedThreadFactory --- src/Lucene.Net.Core/Util/NamedThreadFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Lucene.Net.Core/Util/NamedThreadFactory.cs b/src/Lucene.Net.Core/Util/NamedThreadFactory.cs index 08cc31955e..7e9b27a47f 100644 --- a/src/Lucene.Net.Core/Util/NamedThreadFactory.cs +++ b/src/Lucene.Net.Core/Util/NamedThreadFactory.cs @@ -31,7 +31,7 @@ public class NamedThreadFactory : ThreadFactory { private static int ThreadPoolNumber = 1; private int ThreadNumber = 1; - private const string NAME_PATTERN = "%s-%d-thread"; + private const string NAME_PATTERN = "{0}-{1}-thread"; private readonly string ThreadNamePrefix; /// From e9e4425adbf39d70153c4888929820b4468d8dda Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 02:43:25 +0700 Subject: [PATCH 24/47] Ported Core.TestDemo --- src/Lucene.Net.Tests/Lucene.Net.Tests.csproj | 1 + src/Lucene.Net.Tests/core/TestDemo.cs | 121 ++++++++++--------- 2 files changed, 64 insertions(+), 58 deletions(-) diff --git a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj index 32dea94cf2..31b266685b 100644 --- a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj +++ b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj @@ -428,6 +428,7 @@ Code + Code diff --git a/src/Lucene.Net.Tests/core/TestDemo.cs b/src/Lucene.Net.Tests/core/TestDemo.cs index 0156fd400a..341026eca4 100644 --- a/src/Lucene.Net.Tests/core/TestDemo.cs +++ b/src/Lucene.Net.Tests/core/TestDemo.cs @@ -1,7 +1,8 @@ -namespace org.apache.lucene -{ +using NUnit.Framework; - /* +namespace Lucene.Net +{ + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,67 +19,71 @@ namespace org.apache.lucene * limitations under the License. */ - using Analyzer = Lucene.Net.Analysis.Analyzer; - using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; - using Document = Lucene.Net.Document.Document; - using Field = Lucene.Net.Document.Field; - using DirectoryReader = Lucene.Net.Index.DirectoryReader; - using IndexReader = Lucene.Net.Index.IndexReader; - using Term = Lucene.Net.Index.Term; - using RandomIndexWriter = Lucene.Net.Index.RandomIndexWriter; - using Lucene.Net.Search; - using Directory = Lucene.Net.Store.Directory; - using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + using Analyzer = Lucene.Net.Analysis.Analyzer; + using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; + using Document = Lucene.Net.Documents.Document; + using Field = Lucene.Net.Documents.Field; + using DirectoryReader = Lucene.Net.Index.DirectoryReader; + using IndexReader = Lucene.Net.Index.IndexReader; + using Term = Lucene.Net.Index.Term; + using RandomIndexWriter = Lucene.Net.Index.RandomIndexWriter; + using Lucene.Net.Search; + using Directory = Lucene.Net.Store.Directory; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; - /// - /// A very simple demo used in the API documentation (src/java/overview.html). - /// - /// Please try to keep src/java/overview.html up-to-date when making changes - /// to this class. - /// - public class TestDemo : LuceneTestCase - { + /// + /// A very simple demo used in the API documentation (src/java/overview.html). + /// + /// Please try to keep src/java/overview.html up-to-date when making changes + /// to this class. + /// + public class TestDemo_ : LuceneTestCase + { - public virtual void TestDemo() - { - Analyzer analyzer = new MockAnalyzer(random()); + [Test] + public virtual void TestDemo() + { + Analyzer analyzer = new MockAnalyzer(Random()); - // Store the index in memory: - Directory directory = newDirectory(); - // To store an index on disk, use this instead: - // Directory directory = FSDirectory.open(new File("/tmp/testindex")); - RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, analyzer); - Document doc = new Document(); - string longTerm = "longtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongterm"; - string text = "this is the text to be indexed. " + longTerm; - doc.add(newTextField("fieldname", text, Field.Store.YES)); - iwriter.addDocument(doc); - iwriter.close(); + // Store the index in memory: + using (Directory directory = NewDirectory()) + { + string longTerm = "longtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongterm"; + string text = "this is the text to be indexed. " + longTerm; - // Now search the index: - IndexReader ireader = DirectoryReader.open(directory); // read-only=true - IndexSearcher isearcher = newSearcher(ireader); + // To store an index on disk, use this instead: + // Directory directory = FSDirectory.open(new File("/tmp/testindex")); + using (RandomIndexWriter iwriter = new RandomIndexWriter(Random(), directory, NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, analyzer))) + { + Documents.Document doc = new Documents.Document(); + doc.Add(NewTextField("fieldname", text, Field.Store.YES)); + iwriter.AddDocument(doc); + } - Assert.AreEqual(1, isearcher.search(new TermQuery(new Term("fieldname", longTerm)), 1).totalHits); - Query query = new TermQuery(new Term("fieldname", "text")); - TopDocs hits = isearcher.search(query, null, 1); - Assert.AreEqual(1, hits.totalHits); - // Iterate through the results: - for (int i = 0; i < hits.scoreDocs.length; i++) - { - Document hitDoc = isearcher.doc(hits.scoreDocs[i].doc); - Assert.AreEqual(text, hitDoc.get("fieldname")); - } + // Now search the index: + using (IndexReader ireader = DirectoryReader.Open(directory)) // read-only=true + { + IndexSearcher isearcher = NewSearcher(ireader); - // Test simple phrase query - PhraseQuery phraseQuery = new PhraseQuery(); - phraseQuery.add(new Term("fieldname", "to")); - phraseQuery.add(new Term("fieldname", "be")); - Assert.AreEqual(1, isearcher.search(phraseQuery, null, 1).totalHits); + Assert.AreEqual(1, isearcher.Search(new TermQuery(new Term("fieldname", longTerm)), 1).TotalHits); + Query query = new TermQuery(new Term("fieldname", "text")); + TopDocs hits = isearcher.Search(query, null, 1); + Assert.AreEqual(1, hits.TotalHits); + // Iterate through the results: + for (int i = 0; i < hits.ScoreDocs.Length; i++) + { + Documents.Document hitDoc = isearcher.Doc(hits.ScoreDocs[i].Doc); + Assert.AreEqual(text, hitDoc.Get("fieldname")); + } - ireader.close(); - directory.close(); - } - } + // Test simple phrase query + PhraseQuery phraseQuery = new PhraseQuery(); + phraseQuery.Add(new Term("fieldname", "to")); + phraseQuery.Add(new Term("fieldname", "be")); + Assert.AreEqual(1, isearcher.Search(phraseQuery, null, 1).TotalHits); + } + } + } + } } \ No newline at end of file From 2c3dbc639d30c39807ffdcbf0edc1689bd64b19f Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 03:09:02 +0700 Subject: [PATCH 25/47] Ported Core.TestSearch --- src/Lucene.Net.Tests/Lucene.Net.Tests.csproj | 1 + src/Lucene.Net.Tests/core/TestSearch.cs | 369 +++++++++---------- 2 files changed, 184 insertions(+), 186 deletions(-) diff --git a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj index 31b266685b..289905b79d 100644 --- a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj +++ b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj @@ -432,6 +432,7 @@ Code + Code diff --git a/src/Lucene.Net.Tests/core/TestSearch.cs b/src/Lucene.Net.Tests/core/TestSearch.cs index 236bd7df68..d6ab030cc4 100644 --- a/src/Lucene.Net.Tests/core/TestSearch.cs +++ b/src/Lucene.Net.Tests/core/TestSearch.cs @@ -1,12 +1,16 @@ +using Lucene.Net.Analysis; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Lucene.Net.Search; +using Lucene.Net.Util; +using NUnit.Framework; using System; using System.Collections.Generic; using System.IO; -using NUnit.Framework; -namespace org.apache.lucene +namespace Lucene.Net { - - /* + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,186 +27,179 @@ namespace org.apache.lucene * limitations under the License. */ - - using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; - - using Lucene.Net.Store; - using Lucene.Net.Document; - using Lucene.Net.Analysis; - using Lucene.Net.Index; - using Lucene.Net.Search; - - /// - /// JUnit adaptation of an older test case SearchTest. - public class TestSearch : LuceneTestCase - { - - public virtual void TestNegativeQueryBoost() - { - Query q = new TermQuery(new Term("foo", "bar")); - q.Boost = -42f; - Assert.AreEqual(-42f, q.Boost, 0.0f); - - Directory directory = newDirectory(); - try - { - Analyzer analyzer = new MockAnalyzer(random()); - IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer); - - IndexWriter writer = new IndexWriter(directory, conf); - try - { - Document d = new Document(); - d.add(newTextField("foo", "bar", Field.Store.YES)); - writer.addDocument(d); - } - finally - { - writer.close(); - } - - IndexReader reader = DirectoryReader.open(directory); - try - { - IndexSearcher searcher = newSearcher(reader); - - ScoreDoc[] hits = searcher.search(q, null, 1000).scoreDocs; - Assert.AreEqual(1, hits.Length); - Assert.IsTrue("score is not negative: " + hits[0].score, hits[0].score < 0); - - Explanation explain = searcher.explain(q, hits[0].doc); - Assert.AreEqual("score doesn't match explanation", hits[0].score, explain.Value, 0.001f); - Assert.IsTrue("explain doesn't think doc is a match", explain.Match); - - } - finally - { - reader.close(); - } - } - finally - { - directory.close(); - } - - } - - /// - /// this test performs a number of searches. It also compares output - /// of searches using multi-file index segments with single-file - /// index segments. - /// - /// TODO: someone should check that the results of the searches are - /// still correct by adding assert statements. Right now, the test - /// passes if the results are the same between multi-file and - /// single-file formats, even if the results are wrong. - /// - public virtual void TestSearch() - { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw, true); - DoTestSearch(random(), pw, false); - pw.close(); - sw.close(); - string multiFileOutput = sw.ToString(); - //System.out.println(multiFileOutput); - - sw = new StringWriter(); - pw = new PrintWriter(sw, true); - DoTestSearch(random(), pw, true); - pw.close(); - sw.close(); - string singleFileOutput = sw.ToString(); - - Assert.AreEqual(multiFileOutput, singleFileOutput); - } - - - private void DoTestSearch(Random random, PrintWriter @out, bool useCompoundFile) - { - Directory directory = newDirectory(); - Analyzer analyzer = new MockAnalyzer(random); - IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer); - MergePolicy mp = conf.MergePolicy; - mp.NoCFSRatio = useCompoundFile ? 1.0 : 0.0; - IndexWriter writer = new IndexWriter(directory, conf); - - string[] docs = new string[] {"a b c d e", "a b c d e a b c d e", "a b c d e f g h i j", "a c e", "e c a", "a c e a c e", "a c e a b c"}; - for (int j = 0; j < docs.Length; j++) - { - Document d = new Document(); - d.add(newTextField("contents", docs[j], Field.Store.YES)); - d.add(newStringField("id", "" + j, Field.Store.NO)); - writer.addDocument(d); - } - writer.close(); - - IndexReader reader = DirectoryReader.open(directory); - IndexSearcher searcher = newSearcher(reader); - - ScoreDoc[] hits = null; - - Sort sort = new Sort(SortField.FIELD_SCORE, new SortField("id", SortField.Type.INT)); - - foreach (Query query in BuildQueries()) - { - @out.println("Query: " + query.ToString("contents")); - if (VERBOSE) - { - Console.WriteLine("TEST: query=" + query); - } - - hits = searcher.search(query, null, 1000, sort).scoreDocs; - - @out.println(hits.Length + " total results"); - for (int i = 0 ; i < hits.Length && i < 10; i++) - { - Document d = searcher.doc(hits[i].doc); - @out.println(i + " " + hits[i].score + " " + d.get("contents")); - } - } - reader.close(); - directory.close(); - } - - private IList BuildQueries() - { - IList queries = new List(); - - BooleanQuery booleanAB = new BooleanQuery(); - booleanAB.add(new TermQuery(new Term("contents", "a")), BooleanClause.Occur_e.SHOULD); - booleanAB.add(new TermQuery(new Term("contents", "b")), BooleanClause.Occur_e.SHOULD); - queries.Add(booleanAB); - - PhraseQuery phraseAB = new PhraseQuery(); - phraseAB.add(new Term("contents", "a")); - phraseAB.add(new Term("contents", "b")); - queries.Add(phraseAB); - - PhraseQuery phraseABC = new PhraseQuery(); - phraseABC.add(new Term("contents", "a")); - phraseABC.add(new Term("contents", "b")); - phraseABC.add(new Term("contents", "c")); - queries.Add(phraseABC); - - BooleanQuery booleanAC = new BooleanQuery(); - booleanAC.add(new TermQuery(new Term("contents", "a")), BooleanClause.Occur_e.SHOULD); - booleanAC.add(new TermQuery(new Term("contents", "c")), BooleanClause.Occur_e.SHOULD); - queries.Add(booleanAC); - - PhraseQuery phraseAC = new PhraseQuery(); - phraseAC.add(new Term("contents", "a")); - phraseAC.add(new Term("contents", "c")); - queries.Add(phraseAC); - - PhraseQuery phraseACE = new PhraseQuery(); - phraseACE.add(new Term("contents", "a")); - phraseACE.add(new Term("contents", "c")); - phraseACE.add(new Term("contents", "e")); - queries.Add(phraseACE); - - return queries; - } - } - + /// + /// JUnit adaptation of an older test case SearchTest. + public class TestSearch_ : LuceneTestCase + { + [Test] + public virtual void TestNegativeQueryBoost() + { + Query q = new TermQuery(new Term("foo", "bar")); + q.Boost = -42f; + Assert.AreEqual(-42f, q.Boost, 0.0f); + + Store.Directory directory = NewDirectory(); + try + { + Analyzer analyzer = new MockAnalyzer(Random()); + IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer); + + IndexWriter writer = new IndexWriter(directory, conf); + try + { + Documents.Document d = new Documents.Document(); + d.Add(NewTextField("foo", "bar", Field.Store.YES)); + writer.AddDocument(d); + } + finally + { + writer.Dispose(); + } + + IndexReader reader = DirectoryReader.Open(directory); + try + { + IndexSearcher searcher = NewSearcher(reader); + + ScoreDoc[] hits = searcher.Search(q, null, 1000).ScoreDocs; + Assert.AreEqual(1, hits.Length); + Assert.IsTrue(hits[0].Score < 0, "score is not negative: " + hits[0].Score); + + Explanation explain = searcher.Explain(q, hits[0].Doc); + Assert.AreEqual(hits[0].Score, explain.Value, 0.001f, "score doesn't match explanation"); + Assert.IsTrue(explain.IsMatch, "explain doesn't think doc is a match"); + + } + finally + { + reader.Dispose(); + } + } + finally + { + directory.Dispose(); + } + + } + + /// + /// this test performs a number of searches. It also compares output + /// of searches using multi-file index segments with single-file + /// index segments. + /// + /// TODO: someone should check that the results of the searches are + /// still correct by adding assert statements. Right now, the test + /// passes if the results are the same between multi-file and + /// single-file formats, even if the results are wrong. + /// + [Test] + public virtual void TestSearch() + { + StringWriter sw; + string multiFileOutput; + string singleFileOutput; + using (sw = new StringWriter()) + { + DoTestSearch(Random(), sw, false); + multiFileOutput = sw.ToString(); + } + + //System.out.println(multiFileOutput); + + using (sw = new StringWriter()) + { + DoTestSearch(Random(), sw, true); + singleFileOutput = sw.ToString(); + } + + Assert.AreEqual(multiFileOutput, singleFileOutput); + } + + + private void DoTestSearch(Random random, StringWriter @out, bool useCompoundFile) + { + Store.Directory directory = NewDirectory(); + Analyzer analyzer = new MockAnalyzer(random); + IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer); + MergePolicy mp = conf.MergePolicy; + mp.NoCFSRatio = useCompoundFile ? 1.0 : 0.0; + IndexWriter writer = new IndexWriter(directory, conf); + + string[] docs = new string[] { "a b c d e", "a b c d e a b c d e", "a b c d e f g h i j", "a c e", "e c a", "a c e a c e", "a c e a b c" }; + for (int j = 0; j < docs.Length; j++) + { + Documents.Document d = new Documents.Document(); + d.Add(NewTextField("contents", docs[j], Field.Store.YES)); + d.Add(NewStringField("id", "" + j, Field.Store.NO)); + writer.AddDocument(d); + } + writer.Dispose(); + + IndexReader reader = DirectoryReader.Open(directory); + IndexSearcher searcher = NewSearcher(reader); + + ScoreDoc[] hits = null; + + Sort sort = new Sort(SortField.FIELD_SCORE, new SortField("id", SortField.Type_e.INT)); + + foreach (Query query in BuildQueries()) + { + @out.WriteLine("Query: " + query.ToString("contents")); + if (VERBOSE) + { + Console.WriteLine("TEST: query=" + query); + } + + hits = searcher.Search(query, null, 1000, sort).ScoreDocs; + + @out.WriteLine(hits.Length + " total results"); + for (int i = 0; i < hits.Length && i < 10; i++) + { + Documents.Document d = searcher.Doc(hits[i].Doc); + @out.WriteLine(i + " " + hits[i].Score + " " + d.Get("contents")); + } + } + reader.Dispose(); + directory.Dispose(); + } + + private IList BuildQueries() + { + IList queries = new List(); + + BooleanQuery booleanAB = new BooleanQuery(); + booleanAB.Add(new TermQuery(new Term("contents", "a")), BooleanClause.Occur.SHOULD); + booleanAB.Add(new TermQuery(new Term("contents", "b")), BooleanClause.Occur.SHOULD); + queries.Add(booleanAB); + + PhraseQuery phraseAB = new PhraseQuery(); + phraseAB.Add(new Term("contents", "a")); + phraseAB.Add(new Term("contents", "b")); + queries.Add(phraseAB); + + PhraseQuery phraseABC = new PhraseQuery(); + phraseABC.Add(new Term("contents", "a")); + phraseABC.Add(new Term("contents", "b")); + phraseABC.Add(new Term("contents", "c")); + queries.Add(phraseABC); + + BooleanQuery booleanAC = new BooleanQuery(); + booleanAC.Add(new TermQuery(new Term("contents", "a")), BooleanClause.Occur.SHOULD); + booleanAC.Add(new TermQuery(new Term("contents", "c")), BooleanClause.Occur.SHOULD); + queries.Add(booleanAC); + + PhraseQuery phraseAC = new PhraseQuery(); + phraseAC.Add(new Term("contents", "a")); + phraseAC.Add(new Term("contents", "c")); + queries.Add(phraseAC); + + PhraseQuery phraseACE = new PhraseQuery(); + phraseACE.Add(new Term("contents", "a")); + phraseACE.Add(new Term("contents", "c")); + phraseACE.Add(new Term("contents", "e")); + queries.Add(phraseACE); + + return queries; + } + } } \ No newline at end of file From 0e77936c82232bbcea94dcc1f59e98fc7961fa6a Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 03:19:31 +0700 Subject: [PATCH 26/47] Ported Core.TestSearchForDuplicates --- src/Lucene.Net.Tests/Lucene.Net.Tests.csproj | 1 + .../core/TestSearchForDuplicates.cs | 275 +++++++++--------- 2 files changed, 138 insertions(+), 138 deletions(-) diff --git a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj index 289905b79d..a20ba5c9d6 100644 --- a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj +++ b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj @@ -433,6 +433,7 @@ Code + Code diff --git a/src/Lucene.Net.Tests/core/TestSearchForDuplicates.cs b/src/Lucene.Net.Tests/core/TestSearchForDuplicates.cs index 7b4f114625..ec97868ac9 100644 --- a/src/Lucene.Net.Tests/core/TestSearchForDuplicates.cs +++ b/src/Lucene.Net.Tests/core/TestSearchForDuplicates.cs @@ -1,9 +1,15 @@ +using Lucene.Net.Analysis; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Lucene.Net.Search; +using Lucene.Net.Util; +using NUnit.Framework; using System; +using System.IO; -namespace org.apache.lucene +namespace Lucene.Net { - - /* + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,139 +26,132 @@ namespace org.apache.lucene * limitations under the License. */ - - using Lucene.Net.Store; - using Lucene.Net.Document; - using Lucene.Net.Analysis; - using Lucene.Net.Index; - using Lucene.Net.Search; - using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; - using NUnit.Framework; - - public class TestSearchForDuplicates : LuceneTestCase - { - - internal const string PRIORITY_FIELD = "priority"; - internal const string ID_FIELD = "id"; - internal const string HIGH_PRIORITY = "high"; - internal const string MED_PRIORITY = "medium"; - internal const string LOW_PRIORITY = "low"; - - - /// - /// this test compares search results when using and not using compound - /// files. - /// - /// TODO: There is rudimentary search result validation as well, but it is - /// simply based on asserting the output observed in the old test case, - /// without really knowing if the output is correct. Someone needs to - /// validate this output and make any changes to the checkHits method. - /// - public virtual void TestRun() - { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw, true); - int MAX_DOCS = atLeast(225); - DoTest(random(), pw, false, MAX_DOCS); - pw.close(); - sw.close(); - string multiFileOutput = sw.ToString(); - //System.out.println(multiFileOutput); - - sw = new StringWriter(); - pw = new PrintWriter(sw, true); - DoTest(random(), pw, true, MAX_DOCS); - pw.close(); - sw.close(); - string singleFileOutput = sw.ToString(); - - Assert.AreEqual(multiFileOutput, singleFileOutput); - } - - - private void DoTest(Random random, PrintWriter @out, bool useCompoundFiles, int MAX_DOCS) - { - Directory directory = newDirectory(); - Analyzer analyzer = new MockAnalyzer(random); - IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer); - MergePolicy mp = conf.MergePolicy; - mp.NoCFSRatio = useCompoundFiles ? 1.0 : 0.0; - IndexWriter writer = new IndexWriter(directory, conf); - if (VERBOSE) - { - Console.WriteLine("TEST: now build index MAX_DOCS=" + MAX_DOCS); - } - - for (int j = 0; j < MAX_DOCS; j++) - { - Document d = new Document(); - d.Add(newTextField(PRIORITY_FIELD, HIGH_PRIORITY, Field.Store.YES)); - d.Add(newTextField(ID_FIELD, Convert.ToString(j), Field.Store.YES)); - writer.addDocument(d); - } - writer.Dispose(); - - // try a search without OR - IndexReader reader = DirectoryReader.Open(directory); - IndexSearcher searcher = newSearcher(reader); - - Query query = new TermQuery(new Term(PRIORITY_FIELD, HIGH_PRIORITY)); - @out.println("Query: " + query.ToString(PRIORITY_FIELD)); - if (VERBOSE) - { - Console.WriteLine("TEST: search query=" + query); - } - - Sort sort = new Sort(SortField.FIELD_SCORE, new SortField(ID_FIELD, SortField.Type.INT)); - - ScoreDoc[] hits = searcher.Search(query, null, MAX_DOCS, sort).scoreDocs; - PrintHits(@out, hits, searcher); - CheckHits(hits, MAX_DOCS, searcher); - - // try a new search with OR - searcher = newSearcher(reader); - hits = null; - - BooleanQuery booleanQuery = new BooleanQuery(); - booleanQuery.Add(new TermQuery(new Term(PRIORITY_FIELD, HIGH_PRIORITY)), BooleanClause.Occur_e.SHOULD); - booleanQuery.Add(new TermQuery(new Term(PRIORITY_FIELD, MED_PRIORITY)), BooleanClause.Occur_e.SHOULD); - @out.println("Query: " + booleanQuery.ToString(PRIORITY_FIELD)); - - hits = searcher.search(booleanQuery, null, MAX_DOCS, sort).scoreDocs; - PrintHits(@out, hits, searcher); - CheckHits(hits, MAX_DOCS, searcher); - - reader.Close(); - directory.Close(); - } - - - private void PrintHits(PrintWriter @out, ScoreDoc[] hits, IndexSearcher searcher) - { - @out.println(hits.Length + " total results\n"); - for (int i = 0 ; i < hits.Length; i++) - { - if (i < 10 || (i > 94 && i < 105)) - { - Document d = searcher.Doc(hits[i].Doc); - @out.println(i + " " + d.Get(ID_FIELD)); - } - } - } - - private void CheckHits(ScoreDoc[] hits, int expectedCount, IndexSearcher searcher) - { - Assert.AreEqual(expectedCount, hits.Length, "total results"); - for (int i = 0 ; i < hits.Length; i++) - { - if (i < 10 || (i > 94 && i < 105)) - { - Document d = searcher.Doc(hits[i].Doc); - Assert.AreEqual(Convert.ToString(i), d.Get(ID_FIELD), "check " + i); - } - } - } - - } - + public class TestSearchForDuplicates : LuceneTestCase + { + + internal const string PRIORITY_FIELD = "priority"; + internal const string ID_FIELD = "id"; + internal const string HIGH_PRIORITY = "high"; + internal const string MED_PRIORITY = "medium"; + internal const string LOW_PRIORITY = "low"; + + + /// + /// this test compares search results when using and not using compound + /// files. + /// + /// TODO: There is rudimentary search result validation as well, but it is + /// simply based on asserting the output observed in the old test case, + /// without really knowing if the output is correct. Someone needs to + /// validate this output and make any changes to the checkHits method. + /// + [Test] + public void TestRun() + { + StringWriter sw; + string multiFileOutput; + string singleFileOutput; + int MAX_DOCS = AtLeast(225); + + using (sw = new StringWriter()) + { + DoTest(Random(), sw, false, MAX_DOCS); + multiFileOutput = sw.ToString(); + } + + //System.out.println(multiFileOutput); + + using (sw = new StringWriter()) + { + DoTest(Random(), sw, true, MAX_DOCS); + singleFileOutput = sw.ToString(); + } + + Assert.AreEqual(multiFileOutput, singleFileOutput); + } + + + private void DoTest(Random random, TextWriter @out, bool useCompoundFiles, int MAX_DOCS) + { + Store.Directory directory = NewDirectory(); + Analyzer analyzer = new MockAnalyzer(random); + IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer); + MergePolicy mp = conf.MergePolicy; + mp.NoCFSRatio = useCompoundFiles ? 1.0 : 0.0; + IndexWriter writer = new IndexWriter(directory, conf); + if (VERBOSE) + { + Console.WriteLine("TEST: now build index MAX_DOCS=" + MAX_DOCS); + } + + for (int j = 0; j < MAX_DOCS; j++) + { + Documents.Document d = new Documents.Document(); + d.Add(NewTextField(PRIORITY_FIELD, HIGH_PRIORITY, Field.Store.YES)); + d.Add(NewTextField(ID_FIELD, Convert.ToString(j), Field.Store.YES)); + writer.AddDocument(d); + } + writer.Dispose(); + + // try a search without OR + IndexReader reader = DirectoryReader.Open(directory); + IndexSearcher searcher = NewSearcher(reader); + + Query query = new TermQuery(new Term(PRIORITY_FIELD, HIGH_PRIORITY)); + @out.WriteLine("Query: " + query.ToString(PRIORITY_FIELD)); + if (VERBOSE) + { + Console.WriteLine("TEST: search query=" + query); + } + + Sort sort = new Sort(SortField.FIELD_SCORE, new SortField(ID_FIELD, SortField.Type_e.INT)); + + ScoreDoc[] hits = searcher.Search(query, null, MAX_DOCS, sort).ScoreDocs; + PrintHits(@out, hits, searcher); + CheckHits(hits, MAX_DOCS, searcher); + + // try a new search with OR + searcher = NewSearcher(reader); + hits = null; + + BooleanQuery booleanQuery = new BooleanQuery(); + booleanQuery.Add(new TermQuery(new Term(PRIORITY_FIELD, HIGH_PRIORITY)), BooleanClause.Occur.SHOULD); + booleanQuery.Add(new TermQuery(new Term(PRIORITY_FIELD, MED_PRIORITY)), BooleanClause.Occur.SHOULD); + @out.WriteLine("Query: " + booleanQuery.ToString(PRIORITY_FIELD)); + + hits = searcher.Search(booleanQuery, null, MAX_DOCS, sort).ScoreDocs; + PrintHits(@out, hits, searcher); + CheckHits(hits, MAX_DOCS, searcher); + + reader.Dispose(); + directory.Dispose(); + } + + + private void PrintHits(TextWriter @out, ScoreDoc[] hits, IndexSearcher searcher) + { + @out.WriteLine(hits.Length + " total results\n"); + for (int i = 0; i < hits.Length; i++) + { + if (i < 10 || (i > 94 && i < 105)) + { + Documents.Document d = searcher.Doc(hits[i].Doc); + @out.WriteLine(i + " " + d.Get(ID_FIELD)); + } + } + } + + private void CheckHits(ScoreDoc[] hits, int expectedCount, IndexSearcher searcher) + { + Assert.AreEqual(expectedCount, hits.Length, "total results"); + for (int i = 0; i < hits.Length; i++) + { + if (i < 10 || (i > 94 && i < 105)) + { + Documents.Document d = searcher.Doc(hits[i].Doc); + Assert.AreEqual(Convert.ToString(i), d.Get(ID_FIELD), "check " + i); + } + } + } + } } \ No newline at end of file From f4819a3e4902fd7c257f70068e511145cbc031e4 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 03:59:31 +0700 Subject: [PATCH 27/47] Ported Core.TestExternalCodes + fixed bugs that were preventing it from running. --- .../Pulsing/PulsingPostingsReader.cs | 6 +- src/Lucene.Net.Core/Util/Attribute.cs | 2 +- src/Lucene.Net.Core/Util/AttributeImpl.cs | 276 +++++++++--------- src/Lucene.Net.Tests/Lucene.Net.Tests.csproj | 1 + .../core/TestExternalCodecs.cs | 264 ++++++++--------- 5 files changed, 273 insertions(+), 276 deletions(-) diff --git a/src/Lucene.Net.Codecs/Pulsing/PulsingPostingsReader.cs b/src/Lucene.Net.Codecs/Pulsing/PulsingPostingsReader.cs index 7ab59e970a..46e4eb92b1 100644 --- a/src/Lucene.Net.Codecs/Pulsing/PulsingPostingsReader.cs +++ b/src/Lucene.Net.Codecs/Pulsing/PulsingPostingsReader.cs @@ -699,7 +699,7 @@ public override long Cost() /// /// @lucene.internal /// - internal sealed class PulsingEnumAttributeImpl : AttributeImpl, IPulsingEnumAttribute + internal sealed class PulsingEnumAttribute : AttributeImpl, IPulsingEnumAttribute { // we could store 'other', but what if someone 'chained' multiple postings readers, // this could cause problems? @@ -720,13 +720,11 @@ public override void Clear() // and is calling clearAttributes(), so they don't nuke the reuse information! } - public override void CopyTo(AttributeImpl target) + public override void CopyTo(Util.Attribute target) { // this makes no sense for us, because our state is per-docsenum. // we don't want to copy any stuff over to another docsenum ever! } - } - } } diff --git a/src/Lucene.Net.Core/Util/Attribute.cs b/src/Lucene.Net.Core/Util/Attribute.cs index 6d00033d29..adb42777ec 100644 --- a/src/Lucene.Net.Core/Util/Attribute.cs +++ b/src/Lucene.Net.Core/Util/Attribute.cs @@ -72,7 +72,7 @@ public void Reflect(Type type, string key, object value) } } - public string ReflectAsString(bool prependAttClass) + public virtual string ReflectAsString(bool prependAttClass) { StringBuilder buffer = new StringBuilder(); diff --git a/src/Lucene.Net.Core/Util/AttributeImpl.cs b/src/Lucene.Net.Core/Util/AttributeImpl.cs index f14425b7ae..20fc88ae7d 100644 --- a/src/Lucene.Net.Core/Util/AttributeImpl.cs +++ b/src/Lucene.Net.Core/Util/AttributeImpl.cs @@ -5,8 +5,7 @@ namespace Lucene.Net.Util { - - /* + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -24,150 +23,149 @@ namespace Lucene.Net.Util */ - /// - /// Base class for Attributes that can be added to a - /// . - ///

- /// Attributes are used to add data in a dynamic, yet type-safe way to a source - /// of usually streamed objects, e. g. a . - ///

- public abstract class AttributeImpl : IAttribute, ICloneable - { - /// - /// Clears the values in this AttributeImpl and resets it to its - /// default value. If this implementation implements more than one Attribute interface - /// it clears all. - /// - public abstract void Clear(); - - /// - /// this method returns the current attribute values as a string in the following format - /// by calling the method: - /// - ///
    - ///
  • iff {@code prependAttClass=true}: {@code "AttributeClass#key=value,AttributeClass#key=value"} - ///
  • iff {@code prependAttClass=false}: {@code "key=value,key=value"} - ///
- /// - /// - public string ReflectAsString(bool prependAttClass) - { - StringBuilder buffer = new StringBuilder(); - ReflectWith(new AttributeReflectorAnonymousInnerClassHelper(this, prependAttClass, buffer)); - return buffer.ToString(); - } + /// + /// Base class for Attributes that can be added to a + /// . + ///

+ /// Attributes are used to add data in a dynamic, yet type-safe way to a source + /// of usually streamed objects, e. g. a . + ///

+ public abstract class AttributeImpl : Attribute, IAttribute, ICloneable + { + /// + /// Clears the values in this AttributeImpl and resets it to its + /// default value. If this implementation implements more than one Attribute interface + /// it clears all. + /// + public override abstract void Clear(); - private class AttributeReflectorAnonymousInnerClassHelper : IAttributeReflector - { - private readonly AttributeImpl OuterInstance; + /// + /// this method returns the current attribute values as a string in the following format + /// by calling the method: + /// + ///
    + ///
  • iff {@code prependAttClass=true}: {@code "AttributeClass#key=value,AttributeClass#key=value"} + ///
  • iff {@code prependAttClass=false}: {@code "key=value,key=value"} + ///
+ /// + /// + public override string ReflectAsString(bool prependAttClass) + { + StringBuilder buffer = new StringBuilder(); + ReflectWith(new AttributeReflectorAnonymousInnerClassHelper(this, prependAttClass, buffer)); + return buffer.ToString(); + } - private bool PrependAttClass; - private StringBuilder Buffer; + private class AttributeReflectorAnonymousInnerClassHelper : IAttributeReflector + { + private readonly AttributeImpl OuterInstance; - public AttributeReflectorAnonymousInnerClassHelper(AttributeImpl outerInstance, bool prependAttClass, StringBuilder buffer) - { - this.OuterInstance = outerInstance; - this.PrependAttClass = prependAttClass; - this.Buffer = buffer; - } + private bool PrependAttClass; + private StringBuilder Buffer; - public void Reflect(string key, object value) where T : IAttribute - { - throw new NotImplementedException(); - } + public AttributeReflectorAnonymousInnerClassHelper(AttributeImpl outerInstance, bool prependAttClass, StringBuilder buffer) + { + this.OuterInstance = outerInstance; + this.PrependAttClass = prependAttClass; + this.Buffer = buffer; + } - public virtual void Reflect(Type attClass, string key, object value) - { - if (Buffer.Length > 0) - { - Buffer.Append(','); - } - if (PrependAttClass) - { - Buffer.Append(attClass.Name).Append('#'); - } - Buffer.Append(key).Append('=').Append((value == null) ? "null" : value); - } - } + public void Reflect(string key, object value) where T : IAttribute + { + throw new NotImplementedException(); + } - /// - /// this method is for introspection of attributes, it should simply - /// add the key/values this attribute holds to the given . - /// - ///

The default implementation calls for all - /// non-static fields from the implementing class, using the field name as key - /// and the field value as value. The Attribute class is also determined by reflection. - /// Please note that the default implementation can only handle single-Attribute - /// implementations. - /// - ///

Custom implementations look like this (e.g. for a combined attribute implementation): - /// 

-	  ///   public void reflectWith(AttributeReflector reflector) {
-	  ///     reflector.reflect(CharTermAttribute.class, "term", term());
-	  ///     reflector.reflect(PositionIncrementAttribute.class, "positionIncrement", getPositionIncrement());
-	  ///   }
-	  ///
- /// - ///

If you implement this method, make sure that for each invocation, the same set of - /// interfaces and keys are passed to in the same order, but possibly - /// different values. So don't automatically exclude e.g. {@code null} properties! - ///

- /// - public virtual void ReflectWith(IAttributeReflector reflector) - { - Type clazz = this.GetType(); - LinkedList interfaces = AttributeSource.GetAttributeInterfaces(clazz); - if (interfaces.Count != 1) - { - throw new System.NotSupportedException(clazz.Name + " implements more than one Attribute interface, the default reflectWith() implementation cannot handle this."); - } - //LUCENE-TODO unsure about GetType() - Type interf = (Type)interfaces.First.Value.GetType(); - FieldInfo[] fields = clazz.GetFields(BindingFlags.Instance | BindingFlags.Public); - try - { - for (int i = 0; i < fields.Length; i++) - { - FieldInfo f = fields[i]; - if (f.IsStatic) - { - continue; - } - reflector.Reflect(interf, f.Name, f.GetValue(this)); - } - } - catch (Exception) - { - // this should never happen, because we're just accessing fields - // from 'this' - throw new Exception("Unknown Error"); - } - } + public virtual void Reflect(Type attClass, string key, object value) + { + if (Buffer.Length > 0) + { + Buffer.Append(','); + } + if (PrependAttClass) + { + Buffer.Append(attClass.Name).Append('#'); + } + Buffer.Append(key).Append('=').Append((value == null) ? "null" : value); + } + } - /// - /// Copies the values from this Attribute into the passed-in - /// target attribute. The target implementation must support all the - /// Attributes this implementation supports. - /// - public abstract void CopyTo(AttributeImpl target); + /// + /// this method is for introspection of attributes, it should simply + /// add the key/values this attribute holds to the given . + /// + ///

The default implementation calls for all + /// non-static fields from the implementing class, using the field name as key + /// and the field value as value. The Attribute class is also determined by reflection. + /// Please note that the default implementation can only handle single-Attribute + /// implementations. + /// + ///

Custom implementations look like this (e.g. for a combined attribute implementation): + /// 

+        ///   public void reflectWith(AttributeReflector reflector) {
+        ///     reflector.reflect(CharTermAttribute.class, "term", term());
+        ///     reflector.reflect(PositionIncrementAttribute.class, "positionIncrement", getPositionIncrement());
+        ///   }
+        ///
+ /// + ///

If you implement this method, make sure that for each invocation, the same set of + /// interfaces and keys are passed to in the same order, but possibly + /// different values. So don't automatically exclude e.g. {@code null} properties! + ///

+ /// + public override void ReflectWith(IAttributeReflector reflector) + { + Type clazz = this.GetType(); + LinkedList interfaces = AttributeSource.GetAttributeInterfaces(clazz); + if (interfaces.Count != 1) + { + throw new System.NotSupportedException(clazz.Name + " implements more than one Attribute interface, the default reflectWith() implementation cannot handle this."); + } + //LUCENE-TODO unsure about GetType() + Type interf = (Type)interfaces.First.Value.GetType(); + FieldInfo[] fields = clazz.GetFields(BindingFlags.Instance | BindingFlags.Public); + try + { + for (int i = 0; i < fields.Length; i++) + { + FieldInfo f = fields[i]; + if (f.IsStatic) + { + continue; + } + reflector.Reflect(interf, f.Name, f.GetValue(this)); + } + } + catch (Exception) + { + // this should never happen, because we're just accessing fields + // from 'this' + throw new Exception("Unknown Error"); + } + } - /// - /// Shallow clone. Subclasses must override this if they - /// need to clone any members deeply, - /// - public object Clone() - { - AttributeImpl clone = null; - try - { - clone = (AttributeImpl)base.MemberwiseClone(); - } - catch (Exception) - { - throw new Exception("Clone not supported"); // shouldn't happen - } - return clone; - } - } + /// + /// Copies the values from this Attribute into the passed-in + /// target attribute. The target implementation must support all the + /// Attributes this implementation supports. + /// + public override abstract void CopyTo(Attribute target); + /// + /// Shallow clone. Subclasses must override this if they + /// need to clone any members deeply, + /// + public override object Clone() + { + AttributeImpl clone = null; + try + { + clone = (AttributeImpl)base.MemberwiseClone(); + } + catch (Exception) + { + throw new Exception("Clone not supported"); // shouldn't happen + } + return clone; + } + } } \ No newline at end of file diff --git a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj index a20ba5c9d6..1eab117667 100644 --- a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj +++ b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj @@ -429,6 +429,7 @@ Code + Code diff --git a/src/Lucene.Net.Tests/core/TestExternalCodecs.cs b/src/Lucene.Net.Tests/core/TestExternalCodecs.cs index 6a6a80aa0f..b8a6683772 100644 --- a/src/Lucene.Net.Tests/core/TestExternalCodecs.cs +++ b/src/Lucene.Net.Tests/core/TestExternalCodecs.cs @@ -1,9 +1,9 @@ using System; +using NUnit.Framework; -namespace org.apache.lucene +namespace Lucene.Net { - - /* + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,135 +20,135 @@ namespace org.apache.lucene * limitations under the License. */ - using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; - using PostingsFormat = Lucene.Net.Codecs.PostingsFormat; - using Lucene46Codec = Lucene.Net.Codecs.Lucene46.Lucene46Codec; - using Document = Lucene.Net.Document.Document; - using Field = Lucene.Net.Document.Field; - using DirectoryReader = Lucene.Net.Index.DirectoryReader; - using IndexReader = Lucene.Net.Index.IndexReader; - using IndexWriter = Lucene.Net.Index.IndexWriter; - using Term = Lucene.Net.Index.Term; - using IndexSearcher = Lucene.Net.Search.IndexSearcher; - using TermQuery = Lucene.Net.Search.TermQuery; - using BaseDirectoryWrapper = Lucene.Net.Store.BaseDirectoryWrapper; - using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; - using NUnit.Framework; - - - /* Intentionally outside of oal.index to verify fully + using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; + using PostingsFormat = Lucene.Net.Codecs.PostingsFormat; + using Lucene46Codec = Lucene.Net.Codecs.Lucene46.Lucene46Codec; + using Document = Lucene.Net.Documents.Document; + using Field = Lucene.Net.Documents.Field; + using DirectoryReader = Lucene.Net.Index.DirectoryReader; + using IndexReader = Lucene.Net.Index.IndexReader; + using IndexWriter = Lucene.Net.Index.IndexWriter; + using Term = Lucene.Net.Index.Term; + using IndexSearcher = Lucene.Net.Search.IndexSearcher; + using TermQuery = Lucene.Net.Search.TermQuery; + using BaseDirectoryWrapper = Lucene.Net.Store.BaseDirectoryWrapper; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + + + /* Intentionally outside of oal.index to verify fully external codecs work fine */ - public class TestExternalCodecs : LuceneTestCase - { - - private sealed class CustomPerFieldCodec : Lucene46Codec - { - - internal readonly PostingsFormat RamFormat = PostingsFormat.ForName("RAMOnly"); - internal readonly PostingsFormat DefaultFormat = PostingsFormat.ForName("Lucene41"); - internal readonly PostingsFormat PulsingFormat = PostingsFormat.ForName("Pulsing41"); - - public override PostingsFormat GetPostingsFormatForField(string field) - { - if (field.Equals("field2") || field.Equals("id")) - { - return PulsingFormat; - } - else if (field.Equals("field1")) - { - return DefaultFormat; - } - else - { - return RamFormat; - } - } - } - - // tests storing "id" and "field2" fields as pulsing codec, - // whose term sort is backwards unicode code point, and - // storing "field1" as a custom entirely-in-RAM codec - public virtual void TestPerFieldCodec() - { - - int NUM_DOCS = atLeast(173); - if (VERBOSE) - { - Console.WriteLine("TEST: NUM_DOCS=" + NUM_DOCS); - } - - BaseDirectoryWrapper dir = newDirectory(); - dir.CheckIndexOnClose = false; // we use a custom codec provider - IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setCodec(new CustomPerFieldCodec()).setMergePolicy(newLogMergePolicy(3))); - Document doc = new Document(); - // uses default codec: - doc.Add(newTextField("field1", "this field uses the standard codec as the test", Field.Store.NO)); - // uses pulsing codec: - Field field2 = newTextField("field2", "this field uses the pulsing codec as the test", Field.Store.NO); - doc.Add(field2); - - Field idField = newStringField("id", "", Field.Store.NO); - - doc.Add(idField); - for (int i = 0;i < NUM_DOCS;i++) - { - idField.StringValue = "" + i; - w.addDocument(doc); - if ((i + 1) % 10 == 0) - { - w.commit(); - } - } - if (VERBOSE) - { - Console.WriteLine("TEST: now delete id=77"); - } - w.deleteDocuments(new Term("id", "77")); - - IndexReader r = DirectoryReader.Open(w, true); - - Assert.AreEqual(NUM_DOCS - 1, r.NumDocs()); - IndexSearcher s = newSearcher(r); - Assert.AreEqual(NUM_DOCS - 1, s.Search(new TermQuery(new Term("field1", "standard")), 1).TotalHits); - Assert.AreEqual(NUM_DOCS - 1, s.Search(new TermQuery(new Term("field2", "pulsing")), 1).TotalHits); - r.Close(); - - if (VERBOSE) - { - Console.WriteLine("\nTEST: now delete 2nd doc"); - } - w.deleteDocuments(new Term("id", "44")); - - if (VERBOSE) - { - Console.WriteLine("\nTEST: now force merge"); - } - w.forceMerge(1); - if (VERBOSE) - { - Console.WriteLine("\nTEST: now open reader"); - } - r = DirectoryReader.Open(w, true); - Assert.AreEqual(NUM_DOCS - 2, r.MaxDoc()); - Assert.AreEqual(NUM_DOCS - 2, r.NumDocs()); - s = newSearcher(r); - Assert.AreEqual(NUM_DOCS - 2, s.Search(new TermQuery(new Term("field1", "standard")), 1).TotalHits); - Assert.AreEqual(NUM_DOCS - 2, s.Search(new TermQuery(new Term("field2", "pulsing")), 1).TotalHits); - Assert.AreEqual(1, s.Search(new TermQuery(new Term("id", "76")), 1).TotalHits); - Assert.AreEqual(0, s.Search(new TermQuery(new Term("id", "77")), 1).TotalHits); - Assert.AreEqual(0, s.Search(new TermQuery(new Term("id", "44")), 1).TotalHits); - - if (VERBOSE) - { - Console.WriteLine("\nTEST: now close NRT reader"); - } - r.Close(); - - w.Dispose(); - - dir.Dispose(); - } - } - + public class TestExternalCodecs : LuceneTestCase + { + + private sealed class CustomPerFieldCodec : Lucene46Codec + { + + internal readonly PostingsFormat RamFormat = Codecs.PostingsFormat.ForName("RAMOnly"); + internal readonly PostingsFormat DefaultFormat = Codecs.PostingsFormat.ForName("Lucene41"); + internal readonly PostingsFormat PulsingFormat = Codecs.PostingsFormat.ForName("Pulsing41"); + + public override PostingsFormat GetPostingsFormatForField(string field) + { + if (field.Equals("field2") || field.Equals("id")) + { + return PulsingFormat; + } + else if (field.Equals("field1")) + { + return DefaultFormat; + } + else + { + return RamFormat; + } + } + } + + // tests storing "id" and "field2" fields as pulsing codec, + // whose term sort is backwards unicode code point, and + // storing "field1" as a custom entirely-in-RAM codec + [Test] + public virtual void TestPerFieldCodec() + { + + int NUM_DOCS = AtLeast(173); + if (VERBOSE) + { + Console.WriteLine("TEST: NUM_DOCS=" + NUM_DOCS); + } + + using (BaseDirectoryWrapper dir = NewDirectory()) + { + dir.CheckIndexOnClose = false; // we use a custom codec provider + using (IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetCodec(new CustomPerFieldCodec()).SetMergePolicy(NewLogMergePolicy(3)))) + { + Documents.Document doc = new Documents.Document(); + // uses default codec: + doc.Add(NewTextField("field1", "this field uses the standard codec as the test", Field.Store.NO)); + // uses pulsing codec: + Field field2 = NewTextField("field2", "this field uses the pulsing codec as the test", Field.Store.NO); + doc.Add(field2); + + Field idField = NewStringField("id", "", Field.Store.NO); + + doc.Add(idField); + for (int i = 0; i < NUM_DOCS; i++) + { + idField.StringValue = "" + i; + w.AddDocument(doc); + if ((i + 1) % 10 == 0) + { + w.Commit(); + } + } + if (VERBOSE) + { + Console.WriteLine("TEST: now delete id=77"); + } + w.DeleteDocuments(new Term("id", "77")); + + using (IndexReader r = DirectoryReader.Open(w, true)) + { + Assert.AreEqual(NUM_DOCS - 1, r.NumDocs); + IndexSearcher s = NewSearcher(r); + Assert.AreEqual(NUM_DOCS - 1, s.Search(new TermQuery(new Term("field1", "standard")), 1).TotalHits); + Assert.AreEqual(NUM_DOCS - 1, s.Search(new TermQuery(new Term("field2", "pulsing")), 1).TotalHits); + } + + if (VERBOSE) + { + Console.WriteLine("\nTEST: now delete 2nd doc"); + } + w.DeleteDocuments(new Term("id", "44")); + + if (VERBOSE) + { + Console.WriteLine("\nTEST: now force merge"); + } + w.ForceMerge(1); + if (VERBOSE) + { + Console.WriteLine("\nTEST: now open reader"); + } + using (IndexReader r = DirectoryReader.Open(w, true)) + { + Assert.AreEqual(NUM_DOCS - 2, r.MaxDoc); + Assert.AreEqual(NUM_DOCS - 2, r.NumDocs); + IndexSearcher s = NewSearcher(r); + Assert.AreEqual(NUM_DOCS - 2, s.Search(new TermQuery(new Term("field1", "standard")), 1).TotalHits); + Assert.AreEqual(NUM_DOCS - 2, s.Search(new TermQuery(new Term("field2", "pulsing")), 1).TotalHits); + Assert.AreEqual(1, s.Search(new TermQuery(new Term("id", "76")), 1).TotalHits); + Assert.AreEqual(0, s.Search(new TermQuery(new Term("id", "77")), 1).TotalHits); + Assert.AreEqual(0, s.Search(new TermQuery(new Term("id", "44")), 1).TotalHits); + + if (VERBOSE) + { + Console.WriteLine("\nTEST: now close NRT reader"); + } + } + } + } + } + } } \ No newline at end of file From 4b4485bee1984f66c871418fc7266453c96e1095 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 04:08:30 +0700 Subject: [PATCH 28/47] Moved tests from namespace Lucene.Net.Document to Lucene.Net.Documents so we don't have conflicts with the Document type and cleaned up headers. --- .../core/Document/TestBinaryDocument.cs | 21 +++++---- .../core/Document/TestDateTools.cs | 14 ++---- .../core/Document/TestDocument.cs | 42 +++++++++--------- .../core/Document/TestField.cs | 44 +++++++++---------- 4 files changed, 54 insertions(+), 67 deletions(-) diff --git a/src/Lucene.Net.Tests/core/Document/TestBinaryDocument.cs b/src/Lucene.Net.Tests/core/Document/TestBinaryDocument.cs index 742a0f4f51..7fdb7516ce 100644 --- a/src/Lucene.Net.Tests/core/Document/TestBinaryDocument.cs +++ b/src/Lucene.Net.Tests/core/Document/TestBinaryDocument.cs @@ -1,18 +1,10 @@ -using Lucene.Net.Documents; using Lucene.Net.Support; +using NUnit.Framework; +using System; using System.Text; -namespace Lucene.Net.Document +namespace Lucene.Net.Documents { - using NUnit.Framework; - using System; - using BytesRef = Lucene.Net.Util.BytesRef; - using Directory = Lucene.Net.Store.Directory; - using IndexableField = Lucene.Net.Index.IndexableField; - using IndexReader = Lucene.Net.Index.IndexReader; - using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; - using RandomIndexWriter = Lucene.Net.Index.RandomIndexWriter; - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -30,6 +22,13 @@ namespace Lucene.Net.Document * limitations under the License. */ + using BytesRef = Lucene.Net.Util.BytesRef; + using Directory = Lucene.Net.Store.Directory; + using IndexableField = Lucene.Net.Index.IndexableField; + using IndexReader = Lucene.Net.Index.IndexReader; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + using RandomIndexWriter = Lucene.Net.Index.RandomIndexWriter; + /// /// Tests class. /// diff --git a/src/Lucene.Net.Tests/core/Document/TestDateTools.cs b/src/Lucene.Net.Tests/core/Document/TestDateTools.cs index 26cb90de69..3634e8bfc9 100644 --- a/src/Lucene.Net.Tests/core/Document/TestDateTools.cs +++ b/src/Lucene.Net.Tests/core/Document/TestDateTools.cs @@ -1,18 +1,10 @@ +using Lucene.Net.Util; +using NUnit.Framework; using System; using System.Globalization; -using Lucene.Net.Documents; -namespace Lucene.Net.Document +namespace Lucene.Net.Documents { - /*using Rule = org.junit.Rule; - using RuleChain = org.junit.rules.RuleChain; - using TestRule = org.junit.rules.TestRule; - - using SystemPropertiesRestoreRule = com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;*/ - - using NUnit.Framework; - using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with diff --git a/src/Lucene.Net.Tests/core/Document/TestDocument.cs b/src/Lucene.Net.Tests/core/Document/TestDocument.cs index 458f2a035a..c139d136ea 100644 --- a/src/Lucene.Net.Tests/core/Document/TestDocument.cs +++ b/src/Lucene.Net.Tests/core/Document/TestDocument.cs @@ -1,11 +1,27 @@ -using Lucene.Net.Documents; using Lucene.Net.Support; +using NUnit.Framework; +using System.IO; using System.Text; -namespace Lucene.Net.Document +namespace Lucene.Net.Documents { - using NUnit.Framework; - using System.IO; + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + using BytesRef = Lucene.Net.Util.BytesRef; using Directory = Lucene.Net.Store.Directory; using DirectoryReader = Lucene.Net.Index.DirectoryReader; @@ -15,24 +31,6 @@ namespace Lucene.Net.Document using IndexReader = Lucene.Net.Index.IndexReader; using IndexSearcher = Lucene.Net.Search.IndexSearcher; using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; - - /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - using MockTokenizer = Lucene.Net.Analysis.MockTokenizer; using PhraseQuery = Lucene.Net.Search.PhraseQuery; using Query = Lucene.Net.Search.Query; diff --git a/src/Lucene.Net.Tests/core/Document/TestField.cs b/src/Lucene.Net.Tests/core/Document/TestField.cs index cbe8ecdb0d..37995913f0 100644 --- a/src/Lucene.Net.Tests/core/Document/TestField.cs +++ b/src/Lucene.Net.Tests/core/Document/TestField.cs @@ -1,31 +1,29 @@ +using Lucene.Net.Support; +using NUnit.Framework; +using System; +using System.IO; using System.Text; -using Lucene.Net.Documents; -namespace Lucene.Net.Document +namespace Lucene.Net.Documents { - using Lucene.Net.Support; - using NUnit.Framework; - using System; - using System.IO; - using BytesRef = Lucene.Net.Util.BytesRef; - /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + using BytesRef = Lucene.Net.Util.BytesRef; using CannedTokenStream = Lucene.Net.Analysis.CannedTokenStream; using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; using Token = Lucene.Net.Analysis.Token; From c031b600cf3fe1bb44754a65384d20008dcdfc90 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 04:17:21 +0700 Subject: [PATCH 29/47] Ported Core.TestMergeSchedulerExternal --- src/Lucene.Net.Tests/core/TestMergeSchedulerExternal.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Lucene.Net.Tests/core/TestMergeSchedulerExternal.cs b/src/Lucene.Net.Tests/core/TestMergeSchedulerExternal.cs index 7512d5d74b..d77e380ae0 100644 --- a/src/Lucene.Net.Tests/core/TestMergeSchedulerExternal.cs +++ b/src/Lucene.Net.Tests/core/TestMergeSchedulerExternal.cs @@ -4,9 +4,8 @@ using Lucene.Net.Documents; using NUnit.Framework; -namespace Lucene.Net.Tests +namespace Lucene.Net { - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -23,6 +22,7 @@ namespace Lucene.Net.Tests * See the License for the specific language governing permissions and * limitations under the License. */ + using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; using Document = Documents.Document; using Field = Field; From 202c06875743cdec1ba2d8903ee3337479f65354 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 05:28:54 +0700 Subject: [PATCH 30/47] Finished implementing ignored Classification.SimpleNaiveBayesClassifierTest.TestNGramUsage() --- .../Lucene.Net.Tests.Classification.csproj | 4 ++ .../SimpleNaiveBayesClassifierTest.cs | 57 ++++++++++--------- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/src/Lucene.Net.Tests.Classification/Lucene.Net.Tests.Classification.csproj b/src/Lucene.Net.Tests.Classification/Lucene.Net.Tests.Classification.csproj index 2d618e085c..4ccf2dcfd9 100644 --- a/src/Lucene.Net.Tests.Classification/Lucene.Net.Tests.Classification.csproj +++ b/src/Lucene.Net.Tests.Classification/Lucene.Net.Tests.Classification.csproj @@ -50,6 +50,10 @@ + + {4add0bbc-b900-4715-9526-d871de8eea64} + Lucene.Net.Analysis.Common + {E067B8BB-D8E7-4040-BEB8-EFF8BB4149BD} Lucene.Net.Classification diff --git a/src/Lucene.Net.Tests.Classification/SimpleNaiveBayesClassifierTest.cs b/src/Lucene.Net.Tests.Classification/SimpleNaiveBayesClassifierTest.cs index 7f6d3a0acc..96ff25bb45 100644 --- a/src/Lucene.Net.Tests.Classification/SimpleNaiveBayesClassifierTest.cs +++ b/src/Lucene.Net.Tests.Classification/SimpleNaiveBayesClassifierTest.cs @@ -1,28 +1,32 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - using Lucene.Net.Analysis; +using Lucene.Net.Analysis.Core; +using Lucene.Net.Analysis.Ngram; +using Lucene.Net.Analysis.Reverse; using Lucene.Net.Index; using Lucene.Net.Search; using Lucene.Net.Util; using NUnit.Framework; +using System.IO; namespace Lucene.Net.Classification { + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** * Testcase for {@link SimpleNaiveBayesClassifier} */ @@ -45,20 +49,19 @@ public void TestBasicUsageWithQuery() } [Test] - [Ignore("Need to figure out what to do with NGramAnalyzer, issues with things in Analysis.Common project")] public void TestNGramUsage() { - //CheckCorrectClassification(new SimpleNaiveBayesClassifier(), TECHNOLOGY_INPUT, TECHNOLOGY_RESULT, new NGramAnalyzer(), textFieldName, categoryFieldName); + CheckCorrectClassification(new SimpleNaiveBayesClassifier(), TECHNOLOGY_INPUT, TECHNOLOGY_RESULT, new NGramAnalyzer(), textFieldName, categoryFieldName); } - //private class NGramAnalyzer : Analyzer - //{ - // public override TokenStreamComponents CreateComponents(String fieldName, TextReader reader) - // { - // Tokenizer tokenizer = new KeywordTokenizer(reader); - // return new TokenStreamComponents(tokenizer, new ReverseStringFilter(TEST_VERSION_CURRENT, new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, new ReverseStringFilter(TEST_VERSION_CURRENT, tokenizer), 10, 20))); - // } - //} + private class NGramAnalyzer : Analyzer + { + public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) + { + Tokenizer tokenizer = new KeywordTokenizer(reader); + return new TokenStreamComponents(tokenizer, new ReverseStringFilter(TEST_VERSION_CURRENT, new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, new ReverseStringFilter(TEST_VERSION_CURRENT, tokenizer), 10, 20))); + } + } [Test] public void TestPerformance() From c344d0d210d342ea023a7d5fe972f5ca7d0b6155 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 05:41:50 +0700 Subject: [PATCH 31/47] Finished implementing ignored Core.Index.TestAddIndexes.TestAddIndexMissingCodec() --- .../core/Index/TestAddIndexes.cs | 89 ++++++++++--------- 1 file changed, 45 insertions(+), 44 deletions(-) diff --git a/src/Lucene.Net.Tests/core/Index/TestAddIndexes.cs b/src/Lucene.Net.Tests/core/Index/TestAddIndexes.cs index b3bf19f341..1bb045ba97 100644 --- a/src/Lucene.Net.Tests/core/Index/TestAddIndexes.cs +++ b/src/Lucene.Net.Tests/core/Index/TestAddIndexes.cs @@ -1,21 +1,36 @@ +using Lucene.Net.Documents; +using Lucene.Net.Support; +using NUnit.Framework; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading; -using Lucene.Net.Documents; namespace Lucene.Net.Index { - using Lucene.Net.Support; - using NUnit.Framework; - using System.IO; + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + using AlreadyClosedException = Lucene.Net.Store.AlreadyClosedException; using BaseDirectoryWrapper = Lucene.Net.Store.BaseDirectoryWrapper; using Codec = Lucene.Net.Codecs.Codec; using Directory = Lucene.Net.Store.Directory; using DocIdSetIterator = Lucene.Net.Search.DocIdSetIterator; - - //using Pulsing41PostingsFormat = Lucene.Net.Codecs.pulsing.Pulsing41PostingsFormat; using Document = Documents.Document; using Field = Field; using FieldType = FieldType; @@ -24,29 +39,12 @@ namespace Lucene.Net.Index using LockObtainFailedException = Lucene.Net.Store.LockObtainFailedException; using Lucene46Codec = Lucene.Net.Codecs.Lucene46.Lucene46Codec; using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; - - /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper; using OpenMode_e = Lucene.Net.Index.IndexWriterConfig.OpenMode_e; using PhraseQuery = Lucene.Net.Search.PhraseQuery; using PostingsFormat = Lucene.Net.Codecs.PostingsFormat; + using Pulsing41PostingsFormat = Lucene.Net.Codecs.Pulsing.Pulsing41PostingsFormat; using RAMDirectory = Lucene.Net.Store.RAMDirectory; using StringField = StringField; using TestUtil = Lucene.Net.Util.TestUtil; @@ -1208,7 +1206,6 @@ public UnRegisteredCodec() * simple test that ensures we getting expected exceptions */ [Test] - [Ignore("We don't have all the codecs in place to run this.")] public virtual void TestAddIndexMissingCodec() { BaseDirectoryWrapper toAdd = NewDirectory(); @@ -1228,27 +1225,31 @@ public virtual void TestAddIndexMissingCodec() } } - // LUCENENET TODO: Pulsing41Codec is not in core - /*{ - Directory dir = NewDirectory(); - IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); - conf.SetCodec(TestUtil.AlwaysPostingsFormat(new Pulsing41PostingsFormat(1 + Random().Next(20)))); - IndexWriter w = new IndexWriter(dir, conf); - try - { - w.AddIndexes(toAdd); - Assert.Fail("no such codec"); - } - catch (System.ArgumentException ex) + { + using (Directory dir = NewDirectory()) { - // expected + IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); + conf.SetCodec(TestUtil.AlwaysPostingsFormat(new Pulsing41PostingsFormat(1 + Random().Next(20)))); + IndexWriter w = new IndexWriter(dir, conf); + try + { + w.AddIndexes(toAdd); + Assert.Fail("no such codec"); + } + catch (System.ArgumentException ex) + { + // expected + } + finally + { + w.Dispose(); + } + using (IndexReader open = DirectoryReader.Open(dir)) + { + Assert.AreEqual(0, open.NumDocs); + } } - w.Dispose(); - IndexReader open = DirectoryReader.Open(dir); - Assert.AreEqual(0, open.NumDocs); - open.Dispose(); - dir.Dispose(); - }*/ + } try { From 72ef18564a463342db09a782b6bb9aae430d6be1 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 05:49:39 +0700 Subject: [PATCH 32/47] Enabled ignored Core.Search.TestMultiTermConstantScore.TestBasics() --- src/Lucene.Net.Tests/core/Search/TestMultiTermConstantScore.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Lucene.Net.Tests/core/Search/TestMultiTermConstantScore.cs b/src/Lucene.Net.Tests/core/Search/TestMultiTermConstantScore.cs index 5ae17610f4..f52d360cf3 100644 --- a/src/Lucene.Net.Tests/core/Search/TestMultiTermConstantScore.cs +++ b/src/Lucene.Net.Tests/core/Search/TestMultiTermConstantScore.cs @@ -131,7 +131,6 @@ public static Query Cswcq(Term wild) return query; } - [Ignore] [Test] public virtual void TestBasics() { From 0146b478a6f593ba2d699c68b07979cacbfcf82e Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 05:57:14 +0700 Subject: [PATCH 33/47] Enabled ignored Lucene.Net.Index.TestBinaryTerms.TestBinary() --- .../core/Index/TestBinaryTerms.cs | 38 +++++++++---------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/src/Lucene.Net.Tests/core/Index/TestBinaryTerms.cs b/src/Lucene.Net.Tests/core/Index/TestBinaryTerms.cs index 0c0fabc191..2ae675c622 100644 --- a/src/Lucene.Net.Tests/core/Index/TestBinaryTerms.cs +++ b/src/Lucene.Net.Tests/core/Index/TestBinaryTerms.cs @@ -1,28 +1,27 @@ using Lucene.Net.Documents; +using NUnit.Framework; namespace Lucene.Net.Index { - using NUnit.Framework; - using BytesRef = Lucene.Net.Util.BytesRef; - using Directory = Lucene.Net.Store.Directory; - /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + using BytesRef = Lucene.Net.Util.BytesRef; + using Directory = Lucene.Net.Store.Directory; using Document = Documents.Document; using Field = Field; using FieldType = FieldType; @@ -38,7 +37,6 @@ namespace Lucene.Net.Index [TestFixture] public class TestBinaryTerms : LuceneTestCase { - [Ignore] [Test] public virtual void TestBinary() { From 9530d751bba49a31f25ac7c82a99668949283dac Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 06:13:22 +0700 Subject: [PATCH 34/47] Enabled ignored tests in Core.Index.Test2BDocs + fixed temp dir creation bug in LuceneTestCase --- .../Util/LuceneTestCase.cs | 2 +- src/Lucene.Net.Tests/core/Index/Test2BDocs.cs | 39 +++++++++---------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/Lucene.Net.TestFramework/Util/LuceneTestCase.cs b/src/Lucene.Net.TestFramework/Util/LuceneTestCase.cs index e7eba75045..cb6ee60c85 100644 --- a/src/Lucene.Net.TestFramework/Util/LuceneTestCase.cs +++ b/src/Lucene.Net.TestFramework/Util/LuceneTestCase.cs @@ -2669,7 +2669,7 @@ public static DirectoryInfo CreateTempDir(string prefix) try { - if (!f.Exists) + if (!System.IO.Directory.Exists(f.FullName)) { f.Create(); iterate = false; diff --git a/src/Lucene.Net.Tests/core/Index/Test2BDocs.cs b/src/Lucene.Net.Tests/core/Index/Test2BDocs.cs index 84f69f73ea..8f9ed67dae 100644 --- a/src/Lucene.Net.Tests/core/Index/Test2BDocs.cs +++ b/src/Lucene.Net.Tests/core/Index/Test2BDocs.cs @@ -1,30 +1,29 @@ +using Lucene.Net.Support; +using NUnit.Framework; + namespace Lucene.Net.Index { - using Lucene.Net.Support; - using NUnit.Framework; - using Directory = Lucene.Net.Store.Directory; - /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + using Directory = Lucene.Net.Store.Directory; using Document = Documents.Document; using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; - [Ignore] [TestFixture] public class Test2BDocs : LuceneTestCase { From c7e8c86a7d0aa7ae0eb57980588aa7c47b32079e Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 06:19:04 +0700 Subject: [PATCH 35/47] Updated ignore flag on Index.Test2BBinaryDocValues with the message from the Java version that it takes 45 minutes. --- .../core/Index/Test2BBinaryDocValues.cs | 43 ++++++++----------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/src/Lucene.Net.Tests/core/Index/Test2BBinaryDocValues.cs b/src/Lucene.Net.Tests/core/Index/Test2BBinaryDocValues.cs index 571983329c..4e137b32fc 100644 --- a/src/Lucene.Net.Tests/core/Index/Test2BBinaryDocValues.cs +++ b/src/Lucene.Net.Tests/core/Index/Test2BBinaryDocValues.cs @@ -1,13 +1,26 @@ -using System; using Lucene.Net.Documents; +using NUnit.Framework; +using System; namespace Lucene.Net.Index { - /*using Ignore = org.junit.Ignore; - - using TimeoutSuite = com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;*/ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ - using NUnit.Framework; using BaseDirectoryWrapper = Lucene.Net.Store.BaseDirectoryWrapper; using BinaryDocValuesField = BinaryDocValuesField; using ByteArrayDataInput = Lucene.Net.Store.ByteArrayDataInput; @@ -15,28 +28,10 @@ namespace Lucene.Net.Index using BytesRef = Lucene.Net.Util.BytesRef; using Document = Documents.Document; using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; - - /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper; - [Ignore] + [Ignore("takes ~ 45 minutes")] [TestFixture] public class Test2BBinaryDocValues : LuceneTestCase { From abfe82ce8157b8f813fabe30a672f566daa9027a Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 06:21:28 +0700 Subject: [PATCH 36/47] Updated ignore flag on Index.Test2BSortedDocValues with the message from the Java version that it is very slow. --- .../core/Index/Test2BSortedDocValues.cs | 44 +++++++++---------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/src/Lucene.Net.Tests/core/Index/Test2BSortedDocValues.cs b/src/Lucene.Net.Tests/core/Index/Test2BSortedDocValues.cs index bce3d5b991..55e0218a09 100644 --- a/src/Lucene.Net.Tests/core/Index/Test2BSortedDocValues.cs +++ b/src/Lucene.Net.Tests/core/Index/Test2BSortedDocValues.cs @@ -1,39 +1,35 @@ -using System; using Lucene.Net.Documents; +using NUnit.Framework; +using System; namespace Lucene.Net.Index { - using NUnit.Framework; + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + using BaseDirectoryWrapper = Lucene.Net.Store.BaseDirectoryWrapper; using BytesRef = Lucene.Net.Util.BytesRef; using Document = Documents.Document; using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; - - /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper; using SortedDocValuesField = SortedDocValuesField; - /*using Ignore = org.junit.Ignore; - using TimeoutSuite = com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;*/ - - [Ignore] + [Ignore("very slow")] [TestFixture] public class Test2BSortedDocValues : LuceneTestCase { From 2ebff9483f99160faafb3e1b4a14772434efc254 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 16:04:04 +0700 Subject: [PATCH 37/47] Enabled ignored Core.Util.TestWAH8DocIdSet + fixed bugs in tests --- .../core/Util/TestWAH8DocIdSet.cs | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/src/Lucene.Net.Tests/core/Util/TestWAH8DocIdSet.cs b/src/Lucene.Net.Tests/core/Util/TestWAH8DocIdSet.cs index 6968e198d7..73d338d476 100644 --- a/src/Lucene.Net.Tests/core/Util/TestWAH8DocIdSet.cs +++ b/src/Lucene.Net.Tests/core/Util/TestWAH8DocIdSet.cs @@ -22,7 +22,6 @@ namespace Lucene.Net.Util * limitations under the License. */ - [Ignore] [TestFixture] public class TestWAH8DocIdSet : BaseDocIdSetTestCase { @@ -43,7 +42,6 @@ public override void AssertEquals(int numBits, BitArray ds1, WAH8DocIdSet ds2) Assert.AreEqual(ds1.Cardinality(), ds2.Cardinality()); } - [Ignore] [Test] public virtual void TestUnion() { @@ -72,7 +70,6 @@ public virtual void TestUnion() AssertEquals(numBits, expected, union); } - [Ignore] [Test] public virtual void TestIntersection() { @@ -96,26 +93,14 @@ public virtual void TestIntersection() { for (int previousDoc = -1, doc = set.NextSetBit(0); ; previousDoc = doc, doc = set.NextSetBit(doc + 1)) { - int startIdx = previousDoc + 1; - int endIdx; if (doc == -1) { - endIdx = startIdx + set.Count; - //expected.Clear(previousDoc + 1, set.Count); - for (int i = startIdx; i < endIdx; i++) - { - expected[i] = false; - } + expected.Clear(previousDoc + 1, set.Count); break; } else { - endIdx = startIdx + doc; - for (int i = startIdx; i > endIdx; i++) - { - expected[i] = false; - } - //expected.Clear(previousDoc + 1, doc); + expected.Clear(previousDoc + 1, doc); } } } From e42f400278b53fd6d2c0f1519642b0ec0a152a38 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 16:23:35 +0700 Subject: [PATCH 38/47] Updated ignore attribute on Index.Test2BNumericDocValues with the message from Java that indicates it takes ~30 minutes --- .../core/Index/Test2BNumericDocValues.cs | 48 +++++++++---------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/src/Lucene.Net.Tests/core/Index/Test2BNumericDocValues.cs b/src/Lucene.Net.Tests/core/Index/Test2BNumericDocValues.cs index c62200fa08..57db3d0cca 100644 --- a/src/Lucene.Net.Tests/core/Index/Test2BNumericDocValues.cs +++ b/src/Lucene.Net.Tests/core/Index/Test2BNumericDocValues.cs @@ -1,45 +1,41 @@ -using System; +using Lucene.Net.Attributes; using Lucene.Net.Documents; +using NUnit.Framework; +using System; namespace Lucene.Net.Index { - using NUnit.Framework; + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + using BaseDirectoryWrapper = Lucene.Net.Store.BaseDirectoryWrapper; using Document = Documents.Document; using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; - - /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper; using NumericDocValuesField = NumericDocValuesField; - /*using Ignore = org.junit.Ignore; - - using TimeoutSuite = com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;*/ - //@TimeoutSuite(millis = 80 * TimeUnits.HOUR) @Ignore("takes ~ 30 minutes") @SuppressCodecs("Lucene3x") public class Test2BNumericDocValues extends Lucene.Net.Util.LuceneTestCase - [Ignore] + [Ignore("takes ~ 30 minutes")] [TestFixture] public class Test2BNumericDocValues : LuceneTestCase { // indexes Integer.MAX_VALUE docs with an increasing dv field - [Test] + [Test, LongRunningTest] public virtual void TestNumerics([ValueSource(typeof(ConcurrentMergeSchedulers), "Values")]IConcurrentMergeScheduler scheduler) { BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("2BNumerics")); From 74ee08cd9c6318cb1100ee6e139321158a2398b9 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 16:29:57 +0700 Subject: [PATCH 39/47] Enabled ignored Index.TestStressIndexing2.TestRandom() --- .../core/Index/TestStressIndexing2.cs | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/src/Lucene.Net.Tests/core/Index/TestStressIndexing2.cs b/src/Lucene.Net.Tests/core/Index/TestStressIndexing2.cs index a0fc644814..02d02558f6 100644 --- a/src/Lucene.Net.Tests/core/Index/TestStressIndexing2.cs +++ b/src/Lucene.Net.Tests/core/Index/TestStressIndexing2.cs @@ -1,36 +1,38 @@ +using Lucene.Net.Analysis.Tokenattributes; +using Lucene.Net.Documents; +using Lucene.Net.Randomized.Generators; +using Lucene.Net.Support; +using Lucene.Net.Util; +using NUnit.Framework; using System; using System.Collections.Generic; using System.Diagnostics; using System.Threading; -using Lucene.Net.Analysis.Tokenattributes; -using Lucene.Net.Documents; namespace Lucene.Net.Index { - using Lucene.Net.Randomized.Generators; - using Lucene.Net.Support; - using Lucene.Net.Util; - using NUnit.Framework; + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + using Directory = Lucene.Net.Store.Directory; using DocIdSetIterator = Lucene.Net.Search.DocIdSetIterator; using Document = Documents.Document; using Field = Field; using FieldType = FieldType; - - /* - /// Licensed under the Apache License, Version 2.0 (the "License"); - /// you may not use this file except in compliance with the License. - /// You may obtain a copy of the License at - /// - /// http://www.apache.org/licenses/LICENSE-2.0 - /// - /// Unless required by applicable law or agreed to in writing, software - /// distributed under the License is distributed on an "AS IS" BASIS, - /// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - /// See the License for the specific language governing permissions and - /// limitations under the License. - */ - using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; using OpenMode_e = Lucene.Net.Index.IndexWriterConfig.OpenMode_e; using TermQuery = Lucene.Net.Search.TermQuery; @@ -81,7 +83,6 @@ public virtual void TestRandomIWReader() dir.Dispose(); } - [Ignore] [Test] public virtual void TestRandom() { From 35bf447083ffb7e32ba38e0d8500d217b31597bc Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 16:33:01 +0700 Subject: [PATCH 40/47] Enabled ignored Search.TestPhraseQuery.TestRandomPhrases() --- src/Lucene.Net.Tests/core/Search/TestPhraseQuery.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Lucene.Net.Tests/core/Search/TestPhraseQuery.cs b/src/Lucene.Net.Tests/core/Search/TestPhraseQuery.cs index 9c037e8cbd..d4710e29ac 100644 --- a/src/Lucene.Net.Tests/core/Search/TestPhraseQuery.cs +++ b/src/Lucene.Net.Tests/core/Search/TestPhraseQuery.cs @@ -618,7 +618,6 @@ public virtual void TestRewrite() Assert.IsTrue(rewritten is TermQuery); } - [Ignore] [Test] public virtual void TestRandomPhrases() { From 751db38b6aca8331fa41f2a8a92c80629bce7574 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 16:37:06 +0700 Subject: [PATCH 41/47] Enabled ignored Search.TestBoolean2.TestRandomQueries() --- src/Lucene.Net.Tests/core/Search/TestBoolean2.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Lucene.Net.Tests/core/Search/TestBoolean2.cs b/src/Lucene.Net.Tests/core/Search/TestBoolean2.cs index bf571a5b0e..e8f2e87972 100644 --- a/src/Lucene.Net.Tests/core/Search/TestBoolean2.cs +++ b/src/Lucene.Net.Tests/core/Search/TestBoolean2.cs @@ -284,7 +284,6 @@ public override float Coord(int overlap, int maxOverlap) } } - [Ignore] [Test] public virtual void TestRandomQueries() { From a10dc0ee1080549a7c308b3c6863c7ab1ace76d8 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 11 Sep 2016 16:39:29 +0700 Subject: [PATCH 42/47] Enabled ignored Core.Util.TestBroadWord.TestRank1() --- src/Lucene.Net.Tests/core/Util/TestBroadWord.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Lucene.Net.Tests/core/Util/TestBroadWord.cs b/src/Lucene.Net.Tests/core/Util/TestBroadWord.cs index a5751dc696..b505972d8a 100644 --- a/src/Lucene.Net.Tests/core/Util/TestBroadWord.cs +++ b/src/Lucene.Net.Tests/core/Util/TestBroadWord.cs @@ -28,7 +28,6 @@ private void TstRank(long x) Assert.AreEqual(Number.BitCount(x), BroadWord.BitCount(x), "rank(" + x + ")"); } - [Ignore] [Test] public virtual void TestRank1() { From b9ba93855822ad6cfb5cdf427a4ba1a8008dde7a Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sat, 17 Sep 2016 18:14:56 +0700 Subject: [PATCH 43/47] Removed commented code block from TestFramework.Util.Fst.FSTTester. --- .../Util/fst/FSTTester.cs | 67 ------------------- 1 file changed, 67 deletions(-) diff --git a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs index ccd23797fe..9bcccb9331 100644 --- a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs +++ b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs @@ -945,73 +945,6 @@ private void VerifyPruned(int inputMode, FST fst, int prune1, int prune2) } } - - //// prune 'em - //IEnumerator>> it = prefixes.GetEnumerator(); - //while (it.MoveNext()) - //{ - // KeyValuePair> ent = it.Current; - // IntsRef prefix = ent.Key; - // CountMinOutput cmo = ent.Value; - // if (LuceneTestCase.VERBOSE) - // { - // Console.WriteLine(" term prefix=" + InputToString(inputMode, prefix, false) + " count=" + cmo.Count + " isLeaf=" + cmo.IsLeaf + " output=" + Outputs.OutputToString(cmo.Output) + " isFinal=" + cmo.IsFinal); - // } - // bool keep; - // if (prune1 > 0) - // { - // keep = cmo.Count >= prune1; - // } - // else - // { - // Debug.Assert(prune2 > 0); - // if (prune2 > 1 && cmo.Count >= prune2) - // { - // keep = true; - // } - // else if (prefix.Length > 0) - // { - // // consult our parent - // scratch.Length = prefix.Length - 1; - // Array.Copy(prefix.Ints, prefix.Offset, scratch.Ints, 0, scratch.Length); - // CountMinOutput cmo2 = prefixes.ContainsKey(scratch) ? prefixes[scratch] : null; - // //System.out.println(" parent count = " + (cmo2 == null ? -1 : cmo2.count)); - // keep = cmo2 != null && ((prune2 > 1 && cmo2.Count >= prune2) || (prune2 == 1 && (cmo2.Count >= 2 || prefix.Length <= 1))); - // } - // else if (cmo.Count >= prune2) - // { - // keep = true; - // } - // else - // { - // keep = false; - // } - // } - - // if (!keep) - // { - // it.Reset(); - // //System.out.println(" remove"); - // } - // else - // { - // // clear isLeaf for all ancestors - // //System.out.println(" keep"); - // scratch.CopyInts(prefix); - // scratch.Length--; - // while (scratch.Length >= 0) - // { - // CountMinOutput cmo2 = prefixes.ContainsKey(scratch) ? prefixes[scratch] : null; - // if (cmo2 != null) - // { - // //System.out.println(" clear isLeaf " + inputToString(inputMode, scratch)); - // cmo2.IsLeaf = false; - // } - // scratch.Length--; - // } - // } - //} - if (LuceneTestCase.VERBOSE) { Console.WriteLine("TEST: after prune"); From 979f4e9d43e393ed1e312145f726666d9e333a56 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sat, 17 Sep 2016 21:38:37 +0700 Subject: [PATCH 44/47] Moved common code from Core.Util.Fst.FST and TestFramework.Util.Fst.FSTTester into the support namespace. --- src/Lucene.Net.Core/Lucene.Net.csproj | 1 + src/Lucene.Net.Core/Support/HashCodeMerge.cs | 37 ++++++++++ .../Support/ObjectExtensions.cs | 56 +++++++++++++++ src/Lucene.Net.Core/Util/Fst/FST.cs | 18 ++--- src/Lucene.Net.Core/Util/Fst/NodeHash.cs | 71 +++++++------------ .../Util/fst/FSTTester.cs | 18 +---- 6 files changed, 125 insertions(+), 76 deletions(-) create mode 100644 src/Lucene.Net.Core/Support/ObjectExtensions.cs diff --git a/src/Lucene.Net.Core/Lucene.Net.csproj b/src/Lucene.Net.Core/Lucene.Net.csproj index 9dbcd4c7e3..474060485c 100644 --- a/src/Lucene.Net.Core/Lucene.Net.csproj +++ b/src/Lucene.Net.Core/Lucene.Net.csproj @@ -638,6 +638,7 @@ + diff --git a/src/Lucene.Net.Core/Support/HashCodeMerge.cs b/src/Lucene.Net.Core/Support/HashCodeMerge.cs index ebac0ba665..5268a43dce 100644 --- a/src/Lucene.Net.Core/Support/HashCodeMerge.cs +++ b/src/Lucene.Net.Core/Support/HashCodeMerge.cs @@ -6,6 +6,7 @@ //----------------------------------------------------------------------- using System; +using System.Collections; namespace Lucene.Net.Support { @@ -86,5 +87,41 @@ public static int GetCRC32HashCode(this byte[] buffer, int offset, int count) return (int)crc32Value; } + + /// + /// Gets a hash code for the valueOrEnumerable. If the valueOrEnumerable implements + /// IEnumerable, it enumerates the values and makes a combined hash code representing + /// all of the values in the order they occur in the set. The types of IEnumerable must also be + /// the same, so for example a and a containing + /// the same values will have different hash codes. + /// + /// + /// Any value type, reference type or IEnumerable type. + /// A combined hash code of the value and, if IEnumerable, any values it contains. + public static int GetValueHashCode(this T valueOrEnumerable) + { + if (valueOrEnumerable == null) + return 0; // 0 for null + + if (!(valueOrEnumerable is IEnumerable) || valueOrEnumerable is string) + { + return valueOrEnumerable.GetHashCode(); + } + + int hashCode = valueOrEnumerable.GetType().GetHashCode(); + foreach (object value in valueOrEnumerable as IEnumerable) + { + if (value != null) + { + hashCode = CombineHashCodes(hashCode, value.GetHashCode()); + } + else + { + hashCode = CombineHashCodes(hashCode, 0 /* 0 for null */); + } + } + + return hashCode; + } } } \ No newline at end of file diff --git a/src/Lucene.Net.Core/Support/ObjectExtensions.cs b/src/Lucene.Net.Core/Support/ObjectExtensions.cs new file mode 100644 index 0000000000..37ef70b46b --- /dev/null +++ b/src/Lucene.Net.Core/Support/ObjectExtensions.cs @@ -0,0 +1,56 @@ +using System.Collections; + +namespace Lucene.Net.Support +{ + public static class ObjectExtensions + { + /// + /// Compares the current value against the other value to determine if + /// the values are equal. If the values implement IEnumerable (and are + /// not strings), then it will enumerate over the values and compare them + /// in the same order. + /// + /// This differs from most SetEquals implementations in that they usually + /// don't check the order of the elements in the IEnumerable, but this one does. + /// It also does the check in a safe manner in case the IEnumerable type is nullable, + /// so that null == null. + /// + /// The values that are provided don't necessarily have to implement IEnumerable + /// to check their values for equality. + /// + /// This method is most useful for assertions and testing, but it may + /// also be useful in other scenarios. Do note the IEnumerable values are cast to + /// object before comparing, so it may not be ideal for production scenarios + /// if the values are not reference types. + /// + /// The type of object + /// This object + /// The object that this object will be compared against + /// true if the values are equal; otherwise false + public static bool ValueEquals(this T a, T b) + { + if (a is IEnumerable && b is IEnumerable) + { + var iter = (b as IEnumerable).GetEnumerator(); + foreach (object value in a as IEnumerable) + { + iter.MoveNext(); + if (!object.Equals(value, iter.Current)) + { + return false; + } + } + return true; + } + + return a.Equals(b); + } + + // Special case: strings are IEnumerable, but the default Equals works fine + // for testing value equality + public static bool ValueEquals(this string a, string b) + { + return a.Equals(b); + } + } +} diff --git a/src/Lucene.Net.Core/Util/Fst/FST.cs b/src/Lucene.Net.Core/Util/Fst/FST.cs index 99be83dde2..c38f65872b 100644 --- a/src/Lucene.Net.Core/Util/Fst/FST.cs +++ b/src/Lucene.Net.Core/Util/Fst/FST.cs @@ -1,3 +1,4 @@ +using Lucene.Net.Support; using System; using System.Collections; using System.Collections.Generic; @@ -440,22 +441,11 @@ private bool AssertRootArcs() Debug.Assert(root.Flags == asserting.Flags); Debug.Assert(root.Label == asserting.Label); Debug.Assert(root.NextArc == asserting.NextArc); + // LUCENENET NOTE: In .NET, IEnumerable will not equal another identical IEnumerable // because it checks for reference equality, not that the list contents - // are the same. - if (root.NextFinalOutput is IEnumerable && asserting.NextFinalOutput is IEnumerable) - { - var iter = (asserting.NextFinalOutput as IEnumerable).GetEnumerator(); - foreach (object value in root.NextFinalOutput as IEnumerable) - { - iter.MoveNext(); - Debug.Assert(object.Equals(value, iter.Current)); - } - } - else - { - Debug.Assert(root.NextFinalOutput.Equals(asserting.NextFinalOutput)); - } + // are the same. ValueEquals (a custom extension method) will make that check. + Debug.Assert(root.NextFinalOutput.ValueEquals(asserting.NextFinalOutput)); Debug.Assert(root.Node == asserting.Node); Debug.Assert(root.NumArcs == asserting.NumArcs); Debug.Assert(root.Output.Equals(asserting.Output)); diff --git a/src/Lucene.Net.Core/Util/Fst/NodeHash.cs b/src/Lucene.Net.Core/Util/Fst/NodeHash.cs index 95b6115884..5198fe7f17 100644 --- a/src/Lucene.Net.Core/Util/Fst/NodeHash.cs +++ b/src/Lucene.Net.Core/Util/Fst/NodeHash.cs @@ -2,6 +2,7 @@ namespace Lucene.Net.Util.Fst { + using Support; using System.Collections; /* * Licensed to the Apache Software Foundation (ASF) under one or more @@ -88,29 +89,7 @@ private long Hash(Builder.UnCompiledNode node) long n = ((Builder.CompiledNode)arc.Target).Node; h = PRIME * h + (int)(n ^ (n >> 32)); h = PRIME * h + arc.Output.GetHashCode(); - - // LUCENENET: Since lists do not compare values by default in .NET, - // we need this workaround to get the hashcode of the type + all of the - // values. - if (arc.NextFinalOutput is IEnumerable) - { - h = PRIME * h + arc.NextFinalOutput.GetType().GetHashCode(); - foreach (object value in arc.NextFinalOutput as IEnumerable) - { - if (value != null) - { - h = PRIME * h + value.GetHashCode(); - } - else - { - h = PRIME * h + 0; // 0 for null - } - } - } - else - { - h = PRIME * h + arc.NextFinalOutput.GetHashCode(); - } + h = PRIME * h + arc.NextFinalOutput.GetValueHashCode(); if (arc.IsFinal) { h += 17; @@ -133,29 +112,29 @@ private long Hash(long node) h = PRIME * h + scratchArc.Label; h = PRIME * h + (int)(scratchArc.Target ^ (scratchArc.Target >> 32)); h = PRIME * h + scratchArc.Output.GetHashCode(); - - // LUCENENET: Since lists do not compare values by default in .NET, - // we need this workaround to get the hashcode of the type + all of the - // values. - if (scratchArc.NextFinalOutput is IEnumerable) - { - h = PRIME * h + scratchArc.NextFinalOutput.GetType().GetHashCode(); - foreach (object value in scratchArc.NextFinalOutput as IEnumerable) - { - if (value != null) - { - h = PRIME * h + value.GetHashCode(); - } - else - { - h = PRIME * h + 0; // 0 for null - } - } - } - else - { - h = PRIME * h + scratchArc.NextFinalOutput.GetHashCode(); - } + h = PRIME * h + scratchArc.NextFinalOutput.GetValueHashCode(); + //// LUCENENET: Since lists do not compare values by default in .NET, + //// we need this workaround to get the hashcode of the type + all of the + //// values. + //if (scratchArc.NextFinalOutput is IEnumerable) + //{ + // h = PRIME * h + scratchArc.NextFinalOutput.GetType().GetHashCode(); + // foreach (object value in scratchArc.NextFinalOutput as IEnumerable) + // { + // if (value != null) + // { + // h = PRIME * h + value.GetHashCode(); + // } + // else + // { + // h = PRIME * h + 0; // 0 for null + // } + // } + //} + //else + //{ + // h = PRIME * h + scratchArc.NextFinalOutput.GetHashCode(); + //} if (scratchArc.IsFinal) { diff --git a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs index 9bcccb9331..fc3d75a37b 100644 --- a/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs +++ b/src/Lucene.Net.TestFramework/Util/fst/FSTTester.cs @@ -407,25 +407,11 @@ internal virtual FST DoTest(int prune1, int prune2, bool allowRandomSuffixSha protected internal virtual bool OutputsEqual(T a, T b) { - // LUCENENET: In .NET, lists do not automatically test to ensure + // LUCENENET: In .NET, IEnumerables do not automatically test to ensure // their values are equal, so we need to do that manually. // Note that we are testing the values without regard to whether // the enumerable type is nullable. - if (a is IEnumerable && b is IEnumerable) - { - var iter = (b as IEnumerable).GetEnumerator(); - foreach (object value in a as IEnumerable) - { - iter.MoveNext(); - if (!object.Equals(value, iter.Current)) - { - return false; - } - } - return true; - } - - return a.Equals(b); + return a.ValueEquals(b); } // FST is complete From 44a5cb543394cc9bd2afa42f42d8666c18e66c48 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 18 Sep 2016 05:22:51 +0700 Subject: [PATCH 45/47] Ported part of AtomicReferenceArray from Java for testing. --- src/Lucene.Net.Core/Lucene.Net.csproj | 1 + .../Support/AtomicReferenceArray.cs | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 src/Lucene.Net.Core/Support/AtomicReferenceArray.cs diff --git a/src/Lucene.Net.Core/Lucene.Net.csproj b/src/Lucene.Net.Core/Lucene.Net.csproj index 474060485c..e06e7049e2 100644 --- a/src/Lucene.Net.Core/Lucene.Net.csproj +++ b/src/Lucene.Net.Core/Lucene.Net.csproj @@ -608,6 +608,7 @@ + diff --git a/src/Lucene.Net.Core/Support/AtomicReferenceArray.cs b/src/Lucene.Net.Core/Support/AtomicReferenceArray.cs new file mode 100644 index 0000000000..f6f7316d87 --- /dev/null +++ b/src/Lucene.Net.Core/Support/AtomicReferenceArray.cs @@ -0,0 +1,35 @@ +using System.Threading; + +namespace Lucene.Net.Support +{ + /// + /// Mimics Java's AtomicReferenceArray class (partial implementation) + /// + /// + public class AtomicReferenceArray where T : class + { + private T[] _array; + + public AtomicReferenceArray(int length) + { + _array = new T[length]; + } + + public int Length + { + get { return _array.Length; } + } + + public T this[int index] + { + get + { + return Volatile.Read(ref _array[index]); + } + set + { + Volatile.Write(ref _array[index], value); + } + } + } +} From e8b44e751130bfe1e01489a7879e13eaf1090061 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Sun, 18 Sep 2016 05:25:09 +0700 Subject: [PATCH 46/47] Finished implementation of Core.Util.TestWeakIdenityMap and fixed some bugs with the enumerator logic of WeakIdenityMap (test still failing). --- src/Lucene.Net.Core/Util/WeakIdentityMap.cs | 175 ++++++++++-------- .../core/Util/TestWeakIdentityMap.cs | 146 +++++++++------ 2 files changed, 179 insertions(+), 142 deletions(-) diff --git a/src/Lucene.Net.Core/Util/WeakIdentityMap.cs b/src/Lucene.Net.Core/Util/WeakIdentityMap.cs index aa4e1a811d..7d2ce1cc90 100644 --- a/src/Lucene.Net.Core/Util/WeakIdentityMap.cs +++ b/src/Lucene.Net.Core/Util/WeakIdentityMap.cs @@ -2,7 +2,9 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Linq; using System.Runtime.CompilerServices; +using System.Collections; namespace Lucene.Net.Util { @@ -72,7 +74,7 @@ public sealed class WeakIdentityMap /// Creates a new {@code WeakIdentityMap} based on a non-synchronized . /// The map cleans up the reference queue on every read operation. /// - public static WeakIdentityMap newHashMap() + public static WeakIdentityMap NewHashMap() { return NewHashMap(false); } @@ -162,20 +164,37 @@ public V Put(K key, V value) public IEnumerable Keys { - // .NET port: using this method which mimics IDictionary instead of KeyIterator() get { - foreach (var key in BackingStore.Keys) - { - var target = key.Target; + return new KeyWrapper(this); + } + } - if (target == null) - continue; - else if (target == NULL) - yield return null; - else - yield return (K)target; - } + /// + /// LUCENENET specific class to allow the + /// GetEnumerator() method to be overridden + /// for the keys so we can return an enumerator + /// that is smart enough to clean up the dead keys + /// and also so that MoveNext() returns false in the + /// event there are no more values left (instead of returning + /// a null value in an extra enumeration). + /// + private class KeyWrapper : IEnumerable + { + private readonly WeakIdentityMap outerInstance; + public KeyWrapper(WeakIdentityMap outerInstance) + { + this.outerInstance = outerInstance; + } + public IEnumerator GetEnumerator() + { + outerInstance.Reap(); + return new IteratorAnonymousInnerClassHelper(outerInstance); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); } } @@ -229,93 +248,87 @@ public int Size() return BackingStore.Count; } - /*LUCENE TO-DO I don't think necessary - /// - /// Returns an iterator over all weak keys of this map. - /// Keys already garbage collected will not be returned. - /// this Iterator does not support removals. - /// - public IEnumerator KeyIterator() - { - Reap(); - IEnumerator iterator = BackingStore.Keys.GetEnumerator(); - // IMPORTANT: Don't use oal.util.FilterIterator here: - // We need *strong* reference to current key after setNext()!!! - return new IteratorAnonymousInnerClassHelper(this, iterator); - } - - private class IteratorAnonymousInnerClassHelper : Iterator + private class IteratorAnonymousInnerClassHelper : IEnumerator { - private readonly WeakIdentityMap OuterInstance; - - private IEnumerator Iterator; + private readonly WeakIdentityMap outerInstance; - public IteratorAnonymousInnerClassHelper(WeakIdentityMap outerInstance, IEnumerator iterator) + public IteratorAnonymousInnerClassHelper(WeakIdentityMap outerInstance) { - this.OuterInstance = outerInstance; - this.Iterator = iterator; - next = null; - nextIsSet = false; + this.outerInstance = outerInstance; } - // holds strong reference to next element in backing iterator: - private object next; - // the backing iterator was already consumed: - private bool nextIsSet; - / - public virtual bool HasNext() + // holds strong reference to next element in backing iterator: + private object next = null; + private int position = -1; // start before the beginning of the set + + public K Current { - return nextIsSet || SetNext(); + get + { + return (K)next; + } } - public virtual K Next() + object IEnumerator.Current { - if (!HasNext()) - { - throw new Exception(); - } - Debug.Assert(nextIsSet); - try - { - return (K) next; - } - finally - { - // release strong reference and invalidate current value: - nextIsSet = false; - next = null; - } + get + { + return Current; + } } - public virtual void Remove() + public void Dispose() { - throw new System.NotSupportedException(); + // Nothing to do } - private bool SetNext() + + public bool MoveNext() { - Debug.Assert(!nextIsSet); - while (Iterator.MoveNext()) - { - next = Iterator.Current; - if (next == null) - { - // the key was already GCed, we can remove it from backing map: - Iterator.remove(); - } - else + while (true) { - // unfold "null" special value: - if (next == NULL) - { - next = null; - } - return nextIsSet = true; + IdentityWeakReference key; + + // If the next position doesn't exist, exit + if (++position >= outerInstance.BackingStore.Count) + { + position--; + return false; + } + try + { + key = outerInstance.BackingStore.Keys.ElementAt(position); + } + catch (ArgumentOutOfRangeException) + { + // some other thread beat us to the last element (or removed a prior element) - fail gracefully. + position--; + return false; + } + if (!key.IsAlive) + { + outerInstance.BackingStore.Remove(key); + position--; + continue; + } + // unfold "null" special value: + if (key.Target == NULL) + { + next = null; + } + else + { + next = key.Target; + } + return true; } - } - return false; } - }*/ + + public void Reset() + { + throw new NotSupportedException(); + } + } /// /// Returns an iterator over all values of this map. diff --git a/src/Lucene.Net.Tests/core/Util/TestWeakIdentityMap.cs b/src/Lucene.Net.Tests/core/Util/TestWeakIdentityMap.cs index 82c5c4c66e..3eff0026a1 100644 --- a/src/Lucene.Net.Tests/core/Util/TestWeakIdentityMap.cs +++ b/src/Lucene.Net.Tests/core/Util/TestWeakIdentityMap.cs @@ -2,7 +2,9 @@ using Lucene.Net.Support; using NUnit.Framework; using System; +using System.Collections.Concurrent; using System.Collections.Generic; +using System.Globalization; using System.Threading; namespace Lucene.Net.Util @@ -103,12 +105,12 @@ public virtual void TestSimpleHashMap() Assert.AreEqual(3, map.Size()); int c = 0, keysAssigned = 0; - for (IEnumerator iter = map.Keys.GetEnumerator(); iter.MoveNext(); ) + for (IEnumerator iter = map.Keys.GetEnumerator(); iter.MoveNext();) { //Assert.IsTrue(iter.hasNext()); // try again, should return same result! string k = iter.Current; - Assert.IsTrue(k == key1 || k == key2 | k == key3); // LUCENENET NOTE: Need object.ReferenceEquals here because the == operator does more than check reference equality + Assert.IsTrue(object.ReferenceEquals(k, key1) || object.ReferenceEquals(k, key2) | object.ReferenceEquals(k, key3)); keysAssigned += object.ReferenceEquals(k, key1) ? 1 : (object.ReferenceEquals(k, key2) ? 2 : 4); c++; } @@ -116,7 +118,7 @@ public virtual void TestSimpleHashMap() Assert.AreEqual(1 + 2 + 4, keysAssigned, "all keys must have been seen"); c = 0; - for (IEnumerator iter = map.Values.GetEnumerator(); iter.MoveNext(); ) + for (IEnumerator iter = map.Values.GetEnumerator(); iter.MoveNext();) { string v = iter.Current; Assert.IsTrue(v.StartsWith("bar")); @@ -133,15 +135,13 @@ public virtual void TestSimpleHashMap() { try { - //System.RunFinalization(); - //System.gc(); GC.Collect(); int newSize = map.Size(); Assert.IsTrue(size >= newSize, "previousSize(" + size + ")>=newSize(" + newSize + ")"); size = newSize; - Thread.Sleep(new TimeSpan(0, 0, 1)); + Thread.Sleep(TimeSpan.FromSeconds(1)); c = 0; - for (IEnumerator iter = map.Keys.GetEnumerator(); iter.MoveNext(); ) + for (IEnumerator iter = map.Keys.GetEnumerator(); iter.MoveNext();) { Assert.IsNotNull(iter.Current); c++; @@ -188,22 +188,20 @@ public virtual void TestSimpleHashMap() Assert.IsTrue(map.Empty); } - /* LUCENENET TODO: Compile issues [Test] public virtual void TestConcurrentHashMap() { // don't make threadCount and keyCount random, otherwise easily OOMs or fails otherwise: const int threadCount = 8, keyCount = 1024; - System.Threading.ThreadPool.QueueUserWorkItem(; - //ExecutorService exec = Executors.newFixedThreadPool(threadCount, new NamedThreadFactory("testConcurrentHashMap")); + RunnableAnonymousInnerClassHelper[] workers = new RunnableAnonymousInnerClassHelper[threadCount]; WeakIdentityMap map = WeakIdentityMap.NewConcurrentHashMap(Random().NextBoolean()); // we keep strong references to the keys, // so WeakIdentityMap will not forget about them: AtomicReferenceArray keys = new AtomicReferenceArray(keyCount); for (int j = 0; j < keyCount; j++) { - keys.Set(j, new object()); + keys[j] = new object(); } try @@ -211,19 +209,33 @@ public virtual void TestConcurrentHashMap() for (int t = 0; t < threadCount; t++) { Random rnd = new Random(Random().Next()); - exec.execute(new RunnableAnonymousInnerClassHelper(this, keyCount, map, keys, rnd)); + var worker = new RunnableAnonymousInnerClassHelper(this, keyCount, map, keys, rnd); + workers[t] = worker; + worker.Start(); } } finally { - exec.shutdown(); - while (!exec.awaitTermination(1000L, TimeUnit.MILLISECONDS)) ; + foreach (var w in workers) + { + w.Join(1000L); + } } + // LUCENENET: Since assertions were done on the other threads, we need to check the + // results here. + for (int i = 0; i < workers.Length; i++) + { + assertTrue(string.Format(CultureInfo.InvariantCulture, + "worker thread {0} of {1} failed \n" + workers[i].Error, i, workers.Length), + workers[i].Error == null); + } + + // clear strong refs for (int j = 0; j < keyCount; j++) { - keys.Set(j, null); + keys[j] = null; } // check that GC does not cause problems in reap() method: @@ -232,8 +244,6 @@ public virtual void TestConcurrentHashMap() { try { - //System.runFinalization(); - //System.gc(); GC.Collect(); int newSize = map.Size(); Assert.IsTrue(size >= newSize, "previousSize(" + size + ")>=newSize(" + newSize + ")"); @@ -256,63 +266,77 @@ public virtual void TestConcurrentHashMap() } } - private class RunnableAnonymousInnerClassHelper : IThreadRunnable + private class RunnableAnonymousInnerClassHelper : ThreadClass { - private readonly TestWeakIdentityMap OuterInstance; + private readonly TestWeakIdentityMap outerInstance; - private int KeyCount; - private WeakIdentityMap Map; - private AtomicReferenceArray Keys; - private Random Rnd; + private readonly int keyCount; + private readonly WeakIdentityMap map; + private AtomicReferenceArray keys; + private readonly Random rnd; + private volatile Exception error; public RunnableAnonymousInnerClassHelper(TestWeakIdentityMap outerInstance, int keyCount, WeakIdentityMap map, AtomicReferenceArray keys, Random rnd) { - this.OuterInstance = outerInstance; - this.KeyCount = keyCount; - this.Map = map; - this.Keys = keys; - this.Rnd = rnd; + this.outerInstance = outerInstance; + this.keyCount = keyCount; + this.map = map; + this.keys = keys; + this.rnd = rnd; } - public void Run() + public Exception Error { - int count = AtLeast(Rnd, 10000); - for (int i = 0; i < count; i++) + get { return error; } + } + + + public override void Run() + { + int count = AtLeast(rnd, 10000); + try { - int j = Rnd.Next(KeyCount); - switch (Rnd.Next(5)) + for (int i = 0; i < count; i++) { - case 0: - Map.Put(Keys.Get(j), Convert.ToInt32(j)); - break; - case 1: - int? v = Map.Get(Keys.Get(j)); - if (v != null) - { - Assert.AreEqual(j, (int)v); - } - break; - case 2: - Map.Remove(Keys.Get(j)); - break; - case 3: - // renew key, the old one will be GCed at some time: - Keys.Set(j, new object()); - break; - case 4: - // check iterator still working - for (IEnumerator it = Map.Keys.GetEnumerator(); it.MoveNext();) - { - Assert.IsNotNull(it.Current); - } - break; - default: - Assert.Fail("Should not get here."); - break; + int j = rnd.Next(keyCount); + switch (rnd.Next(5)) + { + case 0: + map.Put(keys[j], Convert.ToInt32(j)); + break; + case 1: + int? v = map.Get(keys[j]); + if (v != null) + { + Assert.AreEqual(j, (int)v); + } + break; + case 2: + map.Remove(keys[j]); + break; + case 3: + // renew key, the old one will be GCed at some time: + keys[j] = new object(); + break; + case 4: + // check iterator still working + for (IEnumerator it = map.Keys.GetEnumerator(); it.MoveNext();) + { + Assert.IsNotNull(it.Current); + } + break; + default: + Assert.Fail("Should not get here."); + break; + } } } + catch (Exception e) + { + e.printStackTrace(); + this.error = e; + } } } - */ } } \ No newline at end of file From c7cf589a8835fdb7b730181fc58e3a9a84c34e75 Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Thu, 22 Sep 2016 10:25:14 +0700 Subject: [PATCH 47/47] Removed commented code from Core.Util.Fst.NodeHash --- src/Lucene.Net.Core/Util/Fst/NodeHash.cs | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/src/Lucene.Net.Core/Util/Fst/NodeHash.cs b/src/Lucene.Net.Core/Util/Fst/NodeHash.cs index 5198fe7f17..3a928f1387 100644 --- a/src/Lucene.Net.Core/Util/Fst/NodeHash.cs +++ b/src/Lucene.Net.Core/Util/Fst/NodeHash.cs @@ -113,29 +113,6 @@ private long Hash(long node) h = PRIME * h + (int)(scratchArc.Target ^ (scratchArc.Target >> 32)); h = PRIME * h + scratchArc.Output.GetHashCode(); h = PRIME * h + scratchArc.NextFinalOutput.GetValueHashCode(); - //// LUCENENET: Since lists do not compare values by default in .NET, - //// we need this workaround to get the hashcode of the type + all of the - //// values. - //if (scratchArc.NextFinalOutput is IEnumerable) - //{ - // h = PRIME * h + scratchArc.NextFinalOutput.GetType().GetHashCode(); - // foreach (object value in scratchArc.NextFinalOutput as IEnumerable) - // { - // if (value != null) - // { - // h = PRIME * h + value.GetHashCode(); - // } - // else - // { - // h = PRIME * h + 0; // 0 for null - // } - // } - //} - //else - //{ - // h = PRIME * h + scratchArc.NextFinalOutput.GetHashCode(); - //} - if (scratchArc.IsFinal) { h += 17;