From 96118654a2a358af8fdfd13455964432820d898d Mon Sep 17 00:00:00 2001 From: Josh Sullivan Date: Mon, 10 Aug 2015 20:06:27 -0400 Subject: [PATCH 1/3] Completed the BoostingQueryTest class --- .../BoostingQueryTest.cs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 src/Lucene.Net.Tests.Queries/BoostingQueryTest.cs diff --git a/src/Lucene.Net.Tests.Queries/BoostingQueryTest.cs b/src/Lucene.Net.Tests.Queries/BoostingQueryTest.cs new file mode 100644 index 0000000000..49c167a463 --- /dev/null +++ b/src/Lucene.Net.Tests.Queries/BoostingQueryTest.cs @@ -0,0 +1,26 @@ +using Lucene.Net.Index; +using Lucene.Net.Queries; +using Lucene.Net.Search; +using Lucene.Net.Util; +using NUnit.Framework; + +namespace Lucene.Net.Tests.Queries +{ + public class BoostingQueryTest : LuceneTestCase + { + // TODO: this suite desperately needs more tests! + // ... like ones that actually run the query + + [Test] + public virtual void TestBoostingQueryEquals() + { + TermQuery q1 = new TermQuery(new Term("subject:", "java")); + TermQuery q2 = new TermQuery(new Term("subject:", "java")); + assertEquals("Two TermQueries with same attributes should be equal", q1, q2); + BoostingQuery bq1 = new BoostingQuery(q1, q2, 0.1f); + QueryUtils.Check(bq1); + BoostingQuery bq2 = new BoostingQuery(q1, q2, 0.1f); + assertEquals("BoostingQuery with same attributes is not equal", bq1, bq2); + } + } +} \ No newline at end of file From 7b4987f77e7395618c2fd62a0565d93aec82e5dc Mon Sep 17 00:00:00 2001 From: Josh Sullivan Date: Mon, 10 Aug 2015 20:28:41 -0400 Subject: [PATCH 2/3] Ported the ChainedFilterTest class. Think I did the right thing with the GregorianCalendar usage... --- .../ChainedFilterTest.cs | 188 ++++++++++++++++++ .../Lucene.Net.Tests.Queries.csproj | 2 + 2 files changed, 190 insertions(+) create mode 100644 src/Lucene.Net.Tests.Queries/ChainedFilterTest.cs diff --git a/src/Lucene.Net.Tests.Queries/ChainedFilterTest.cs b/src/Lucene.Net.Tests.Queries/ChainedFilterTest.cs new file mode 100644 index 0000000000..66be39ae8a --- /dev/null +++ b/src/Lucene.Net.Tests.Queries/ChainedFilterTest.cs @@ -0,0 +1,188 @@ +using System; +using System.Globalization; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Lucene.Net.Queries; +using Lucene.Net.Search; +using Lucene.Net.Store; +using Lucene.Net.Util; +using NUnit.Framework; + +namespace Lucene.Net.Tests.Queries +{ + public class ChainedFilterTest : LuceneTestCase + { + public const int Max = 500; + + private Directory directory; + private IndexSearcher searcher; + private IndexReader reader; + private Query query; + // private DateFilter dateFilter; DateFilter was deprecated and removed + private TermRangeFilter dateFilter; + private QueryWrapperFilter bobFilter; + private QueryWrapperFilter sueFilter; + + [SetUp] + public override void SetUp() + { + base.SetUp(); + directory = NewDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(Random(), directory); + // we use the default Locale/TZ since LuceneTestCase randomizes it + var cal = new DateTime(1970, 1, 1, 0, 0, 0, (int)TestUtil.NextLong(Random(), 0, long.MaxValue), new GregorianCalendar()); + + for (int i = 0; i < Max; i++) + { + Document doc = new Document(); + doc.Add(NewStringField("key", "" + (i + 1), Field.Store.YES)); + doc.Add(NewStringField("owner", (i < Max / 2) ? "bob" : "sue", Field.Store.YES)); + doc.Add(NewStringField("date", cal.ToString(CultureInfo.InvariantCulture), Field.Store.YES)); + writer.AddDocument(doc); + + cal = cal.AddDays(1); + } + reader = writer.Reader; + writer.Dispose(); + + searcher = NewSearcher(reader); + + // query for everything to make life easier + BooleanQuery bq = new BooleanQuery(); + bq.Add(new TermQuery(new Term("owner", "bob")), BooleanClause.Occur.SHOULD); + bq.Add(new TermQuery(new Term("owner", "sue")), BooleanClause.Occur.SHOULD); + query = bq; + + // date filter matches everything too + //Date pastTheEnd = parseDate("2099 Jan 1"); + // dateFilter = DateFilter.Before("date", pastTheEnd); + // just treat dates as strings and select the whole range for now... + dateFilter = TermRangeFilter.NewStringRange("date", "", "ZZZZ", true, true); + + bobFilter = new QueryWrapperFilter(new TermQuery(new Term("owner", "bob"))); + sueFilter = new QueryWrapperFilter(new TermQuery(new Term("owner", "sue"))); + } + + [TearDown] + public override void TearDown() + { + reader.Dispose(); + directory.Dispose(); + base.TearDown(); + } + + private ChainedFilter GetChainedFilter(Filter[] chain, int[] logic) + { + if (logic == null) + { + return new ChainedFilter(chain); + } + return new ChainedFilter(chain, logic); + } + + private ChainedFilter GetChainedFilter(Filter[] chain, int logic) + { + return new ChainedFilter(chain, logic); + } + + + [Test] + public virtual void TestSingleFilter() + { + ChainedFilter chain = GetChainedFilter(new Filter[] { dateFilter }, null); + + int numHits = searcher.Search(query, chain, 1000).TotalHits; + assertEquals(Max, numHits); + + chain = new ChainedFilter(new Filter[] { bobFilter }); + numHits = searcher.Search(query, chain, 1000).TotalHits; + assertEquals(Max / 2, numHits); + + chain = GetChainedFilter(new Filter[] { bobFilter }, new[] { ChainedFilter.AND }); + TopDocs hits = searcher.Search(query, chain, 1000); + numHits = hits.TotalHits; + assertEquals(Max / 2, numHits); + assertEquals("bob", searcher.Doc(hits.ScoreDocs[0].Doc).Get("owner")); + + chain = GetChainedFilter(new Filter[] { bobFilter }, new[] { ChainedFilter.ANDNOT }); + hits = searcher.Search(query, chain, 1000); + numHits = hits.TotalHits; + assertEquals(Max / 2, numHits); + assertEquals("sue", searcher.Doc(hits.ScoreDocs[0].Doc).Get("owner")); + } + + [Test] + public virtual void TestOR() + { + ChainedFilter chain = GetChainedFilter(new Filter[] { sueFilter, bobFilter }, null); + + int numHits = searcher.Search(query, chain, 1000).TotalHits; + assertEquals("OR matches all", Max, numHits); + } + + [Test] + public virtual void TestAND() + { + ChainedFilter chain = GetChainedFilter(new Filter[] { dateFilter, bobFilter }, ChainedFilter.AND); + + TopDocs hits = searcher.Search(query, chain, 1000); + assertEquals("AND matches just bob", Max / 2, hits.TotalHits); + assertEquals("bob", searcher.Doc(hits.ScoreDocs[0].Doc).Get("owner")); + } + + [Test] + public virtual void TestXOR() + { + ChainedFilter chain = GetChainedFilter(new Filter[] { dateFilter, bobFilter }, ChainedFilter.XOR); + + TopDocs hits = searcher.Search(query, chain, 1000); + assertEquals("XOR matches sue", Max / 2, hits.TotalHits); + assertEquals("sue", searcher.Doc(hits.ScoreDocs[0].Doc).Get("owner")); + } + + [Test] + public virtual void TestANDNOT() + { + ChainedFilter chain = GetChainedFilter(new Filter[] { dateFilter, sueFilter }, new int[] { ChainedFilter.AND, ChainedFilter.ANDNOT }); + + TopDocs hits = searcher.Search(query, chain, 1000); + assertEquals("ANDNOT matches just bob", Max / 2, hits.TotalHits); + assertEquals("bob", searcher.Doc(hits.ScoreDocs[0].Doc).Get("owner")); + + chain = GetChainedFilter(new Filter[] { bobFilter, bobFilter }, new int[] { ChainedFilter.ANDNOT, ChainedFilter.ANDNOT }); + + hits = searcher.Search(query, chain, 1000); + assertEquals("ANDNOT bob ANDNOT bob matches all sues", Max / 2, hits.TotalHits); + assertEquals("sue", searcher.Doc(hits.ScoreDocs[0].Doc).Get("owner")); + } + + [Test] + public virtual void TestWithCachingFilter() + { + Directory dir = NewDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(Random(), dir); + IndexReader reader = writer.Reader; + writer.Dispose(); + + IndexSearcher searcher = NewSearcher(reader); + + Query query = new TermQuery(new Term("none", "none")); + + QueryWrapperFilter queryFilter = new QueryWrapperFilter(query); + CachingWrapperFilter cachingFilter = new CachingWrapperFilter(queryFilter); + + searcher.Search(query, cachingFilter, 1); + + CachingWrapperFilter cachingFilter2 = new CachingWrapperFilter(queryFilter); + Filter[] chain = new Filter[2]; + chain[0] = cachingFilter; + chain[1] = cachingFilter2; + ChainedFilter cf = new ChainedFilter(chain); + + // throws java.lang.ClassCastException: org.apache.lucene.util.OpenBitSet cannot be cast to java.util.BitSet + searcher.Search(new MatchAllDocsQuery(), cf, 1); + reader.Dispose(); + dir.Dispose(); + } + } +} \ No newline at end of file diff --git a/src/Lucene.Net.Tests.Queries/Lucene.Net.Tests.Queries.csproj b/src/Lucene.Net.Tests.Queries/Lucene.Net.Tests.Queries.csproj index 74afe8f594..9264faa7ee 100644 --- a/src/Lucene.Net.Tests.Queries/Lucene.Net.Tests.Queries.csproj +++ b/src/Lucene.Net.Tests.Queries/Lucene.Net.Tests.Queries.csproj @@ -43,6 +43,8 @@ + + From 2808ec7408968488aa6fd8091c1be8fe3fbd1eb9 Mon Sep 17 00:00:00 2001 From: Josh Sullivan Date: Tue, 11 Aug 2015 01:31:24 -0400 Subject: [PATCH 3/3] Finished testing Lucene.Net.Queries Added a few implementation changes based around expecting certain arguments to throw if empty/null. --- src/Lucene.Net.Queries/CustomScoreQuery.cs | 8 +- src/Lucene.Net.Queries/TermsFilter.cs | 10 + .../Function/FunctionTestSetup.cs | 172 ++++++++ .../Function/TestBoostedQuery.cs | 79 ++++ .../Function/TestDocValuesFieldSources.cs | 149 +++++++ .../Function/TestFieldScoreQuery.cs | 162 +++++++ .../Function/TestFunctionQuerySort.cs | 80 ++++ .../Function/TestLongNormValueSource.cs | 230 ++++++++++ .../Function/TestOrdValues.cs | 157 +++++++ .../Function/TestValueSources.cs | 339 +++++++++++++++ .../Lucene.Net.Tests.Queries.csproj | 16 +- .../Mlt/TestMoreLikeThis.cs | 134 ++++++ .../TermsFilterTest.cs | 326 ++++++++++++++ .../TestCustomScoreQuery.cs | 397 ++++++++++++++++++ 14 files changed, 2251 insertions(+), 8 deletions(-) create mode 100644 src/Lucene.Net.Tests.Queries/Function/FunctionTestSetup.cs create mode 100644 src/Lucene.Net.Tests.Queries/Function/TestBoostedQuery.cs create mode 100644 src/Lucene.Net.Tests.Queries/Function/TestDocValuesFieldSources.cs create mode 100644 src/Lucene.Net.Tests.Queries/Function/TestFieldScoreQuery.cs create mode 100644 src/Lucene.Net.Tests.Queries/Function/TestFunctionQuerySort.cs create mode 100644 src/Lucene.Net.Tests.Queries/Function/TestLongNormValueSource.cs create mode 100644 src/Lucene.Net.Tests.Queries/Function/TestOrdValues.cs create mode 100644 src/Lucene.Net.Tests.Queries/Function/TestValueSources.cs create mode 100644 src/Lucene.Net.Tests.Queries/Mlt/TestMoreLikeThis.cs create mode 100644 src/Lucene.Net.Tests.Queries/TermsFilterTest.cs create mode 100644 src/Lucene.Net.Tests.Queries/TestCustomScoreQuery.cs diff --git a/src/Lucene.Net.Queries/CustomScoreQuery.cs b/src/Lucene.Net.Queries/CustomScoreQuery.cs index 94e7895edd..0a17ca2689 100644 --- a/src/Lucene.Net.Queries/CustomScoreQuery.cs +++ b/src/Lucene.Net.Queries/CustomScoreQuery.cs @@ -399,10 +399,10 @@ public override long Cost() } } - // public override Weight CreateWeight(IndexSearcher searcher) - // { - // return new CustomWeight(this, searcher); - // } + public override Weight CreateWeight(IndexSearcher searcher) + { + return new CustomWeight(this, searcher); + } /// /// Checks if this is strict custom scoring. diff --git a/src/Lucene.Net.Queries/TermsFilter.cs b/src/Lucene.Net.Queries/TermsFilter.cs index dfe9449ca5..cd28f3f675 100644 --- a/src/Lucene.Net.Queries/TermsFilter.cs +++ b/src/Lucene.Net.Queries/TermsFilter.cs @@ -67,6 +67,11 @@ private class FieldAndTermEnumAnonymousInnerClassHelper : FieldAndTermEnum public FieldAndTermEnumAnonymousInnerClassHelper(List terms) { + if (!terms.Any()) + { + throw new ArgumentException("no terms provided"); + } + this.terms = terms; terms.Sort(); iter = terms.GetEnumerator(); @@ -102,6 +107,11 @@ private class FieldAndTermEnumAnonymousInnerClassHelper2 : FieldAndTermEnum public FieldAndTermEnumAnonymousInnerClassHelper2(string field, List terms) : base(field) { + if (!terms.Any()) + { + throw new ArgumentException("no terms provided"); + } + this.terms = terms; terms.Sort(); iter = terms.GetEnumerator(); diff --git a/src/Lucene.Net.Tests.Queries/Function/FunctionTestSetup.cs b/src/Lucene.Net.Tests.Queries/Function/FunctionTestSetup.cs new file mode 100644 index 0000000000..f06b6712f4 --- /dev/null +++ b/src/Lucene.Net.Tests.Queries/Function/FunctionTestSetup.cs @@ -0,0 +1,172 @@ +using System; +using Lucene.Net.Analysis; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Lucene.Net.Queries.Function; +using Lucene.Net.Queries.Function.ValueSources; +using Lucene.Net.Store; +using Lucene.Net.Util; +using NUnit.Framework; + +namespace Lucene.Net.Tests.Queries.Function +{ + /// + /// Setup for function tests + /// + public abstract class FunctionTestSetup : LuceneTestCase + { + + /// + /// Actual score computation order is slightly different than assumptios + /// this allows for a small amount of variation + /// + protected internal static float TEST_SCORE_TOLERANCE_DELTA = 0.001f; + + protected internal const int N_DOCS = 17; // select a primary number > 2 + + protected internal const string ID_FIELD = "id"; + protected internal const string TEXT_FIELD = "text"; + protected internal const string INT_FIELD = "iii"; + protected internal const string FLOAT_FIELD = "fff"; + + protected internal ValueSource BYTE_VALUESOURCE = new ByteFieldSource(INT_FIELD); + protected internal ValueSource SHORT_VALUESOURCE = new ShortFieldSource(INT_FIELD); + protected internal ValueSource INT_VALUESOURCE = new IntFieldSource(INT_FIELD); + protected internal ValueSource INT_AS_FLOAT_VALUESOURCE = new FloatFieldSource(INT_FIELD); + protected internal ValueSource FLOAT_VALUESOURCE = new FloatFieldSource(FLOAT_FIELD); + + private static readonly string[] DOC_TEXT_LINES = + { + @"Well, this is just some plain text we use for creating the ", + "test documents. It used to be a text from an online collection ", + "devoted to first aid, but if there was there an (online) lawyers ", + "first aid collection with legal advices, \"it\" might have quite ", + "probably advised one not to include \"it\"'s text or the text of ", + "any other online collection in one's code, unless one has money ", + "that one don't need and one is happy to donate for lawyers ", + "charity. Anyhow at some point, rechecking the usage of this text, ", + "it became uncertain that this text is free to use, because ", + "the web site in the disclaimer of he eBook containing that text ", + "was not responding anymore, and at the same time, in projGut, ", + "searching for first aid no longer found that eBook as well. ", + "So here we are, with a perhaps much less interesting ", + "text for the test, but oh much much safer. " + }; + + protected internal static Directory dir; + protected internal static Analyzer anlzr; + + [TearDown] + public override void TearDown() + { + base.TearDown(); + dir.Dispose(); + dir = null; + anlzr = null; + } + + + protected internal static void CreateIndex(bool doMultiSegment) + { + if (VERBOSE) + { + Console.WriteLine("TEST: setUp"); + } + // prepare a small index with just a few documents. + dir = NewDirectory(); + anlzr = new MockAnalyzer(Random()); + IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, anlzr).SetMergePolicy(NewLogMergePolicy()); + if (doMultiSegment) + { + iwc.SetMaxBufferedDocs(TestUtil.NextInt(Random(), 2, 7)); + } + RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc); + // add docs not exactly in natural ID order, to verify we do check the order of docs by scores + int remaining = N_DOCS; + bool[] done = new bool[N_DOCS]; + int i = 0; + while (remaining > 0) + { + if (done[i]) + { + throw new Exception("to set this test correctly N_DOCS=" + N_DOCS + " must be primary and greater than 2!"); + } + AddDoc(iw, i); + done[i] = true; + i = (i + 4) % N_DOCS; + remaining--; + } + if (!doMultiSegment) + { + if (VERBOSE) + { + Console.WriteLine("TEST: setUp full merge"); + } + iw.ForceMerge(1); + } + iw.Dispose(); + if (VERBOSE) + { + Console.WriteLine("TEST: setUp done close"); + } + } + + private static void AddDoc(RandomIndexWriter iw, int i) + { + Document d = new Document(); + Field f; + int scoreAndID = i + 1; + + FieldType customType = new FieldType(TextField.TYPE_STORED); + customType.Tokenized = false; + customType.OmitNorms = true; + + f = NewField(ID_FIELD, Id2String(scoreAndID), customType); // for debug purposes + d.Add(f); + + FieldType customType2 = new FieldType(TextField.TYPE_NOT_STORED); + customType2.OmitNorms = true; + f = NewField(TEXT_FIELD, "text of doc" + scoreAndID + TextLine(i), customType2); // for regular search + d.Add(f); + + f = NewField(INT_FIELD, "" + scoreAndID, customType); // for function scoring + d.Add(f); + + f = NewField(FLOAT_FIELD, scoreAndID + ".000", customType); // for function scoring + d.Add(f); + + iw.AddDocument(d); + Log("added: " + d); + } + + // 17 --> ID00017 + protected internal static string Id2String(int scoreAndID) + { + string s = "000000000" + scoreAndID; + int n = ("" + N_DOCS).Length + 3; + int k = s.Length - n; + return "ID" + s.Substring(k); + } + + // some text line for regular search + private static string TextLine(int docNum) + { + return DOC_TEXT_LINES[docNum % DOC_TEXT_LINES.Length]; + } + + // extract expected doc score from its ID Field: "ID7" --> 7.0 + protected internal static float ExpectedFieldScore(string docIDFieldVal) + { + return Convert.ToSingle(docIDFieldVal.Substring(2)); + } + + // debug messages (change DBG to true for anything to print) + protected internal static void Log(object o) + { + if (VERBOSE) + { + Console.WriteLine(o.ToString()); + } + } + } +} \ No newline at end of file diff --git a/src/Lucene.Net.Tests.Queries/Function/TestBoostedQuery.cs b/src/Lucene.Net.Tests.Queries/Function/TestBoostedQuery.cs new file mode 100644 index 0000000000..8ddd281a05 --- /dev/null +++ b/src/Lucene.Net.Tests.Queries/Function/TestBoostedQuery.cs @@ -0,0 +1,79 @@ +using Lucene.Net.Analysis; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Lucene.Net.Queries.Function; +using Lucene.Net.Queries.Function.ValueSources; +using Lucene.Net.Search; +using Lucene.Net.Store; +using Lucene.Net.Util; +using NUnit.Framework; + +namespace Lucene.Net.Tests.Queries.Function +{ + /// + /// Basic tests for + /// + // TODO: more tests + public class TestBoostedQuery : LuceneTestCase + { + internal static Directory dir; + internal static IndexReader ir; + internal static IndexSearcher @is; + + [SetUp] + public override void SetUp() + { + base.SetUp(); + dir = NewDirectory(); + IndexWriterConfig iwConfig = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); + iwConfig.SetMergePolicy(NewLogMergePolicy()); + RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwConfig); + Document document = new Document(); + Field idField = new StringField("id", "", Field.Store.NO); + document.Add(idField); + iw.AddDocument(document); + ir = iw.Reader; + @is = NewSearcher(ir); + iw.Dispose(); + } + + [TearDown] + public override void TearDown() + { + base.TearDown(); + @is = null; + ir.Dispose(); + ir = null; + dir.Dispose(); + dir = null; + } + + [Test] + public virtual void TestBasic() + { + Query q = new MatchAllDocsQuery(); + TopDocs docs = @is.Search(q, 10); + assertEquals(1, docs.TotalHits); + float score = docs.ScoreDocs[0].Score; + + Query boostedQ = new BoostedQuery(q, new ConstValueSource(2.0f)); + AssertHits(boostedQ, new float[] { score * 2 }); + } + + + private void AssertHits(Query q, float[] scores) + { + ScoreDoc[] expected = new ScoreDoc[scores.Length]; + int[] expectedDocs = new int[scores.Length]; + for (int i = 0; i < expected.Length; i++) + { + expectedDocs[i] = i; + expected[i] = new ScoreDoc(i, scores[i]); + } + TopDocs docs = @is.Search(q, 10, new Sort(new SortField("id", SortField.Type_e.STRING))); + CheckHits.DoCheckHits(Random(), q, "", @is, expectedDocs); + CheckHits.CheckHitsQuery(q, expected, docs.ScoreDocs, expectedDocs); + CheckHits.CheckExplanations(q, "", @is); + } + } +} \ No newline at end of file diff --git a/src/Lucene.Net.Tests.Queries/Function/TestDocValuesFieldSources.cs b/src/Lucene.Net.Tests.Queries/Function/TestDocValuesFieldSources.cs new file mode 100644 index 0000000000..621c6eb502 --- /dev/null +++ b/src/Lucene.Net.Tests.Queries/Function/TestDocValuesFieldSources.cs @@ -0,0 +1,149 @@ +using System; +using Lucene.Net.Analysis; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Lucene.Net.Queries.Function; +using Lucene.Net.Queries.Function.ValueSources; +using Lucene.Net.Randomized.Generators; +using Lucene.Net.Store; +using Lucene.Net.Support; +using Lucene.Net.Util; +using Lucene.Net.Util.Packed; +using NUnit.Framework; + +namespace Lucene.Net.Tests.Queries.Function +{ + // [Util.LuceneTestCase.SuppressCodecs("Lucene3x")] + public class TestDocValuesFieldSources : LuceneTestCase + { + private void DoTest(FieldInfo.DocValuesType_e type) + { + Directory d = NewDirectory(); + IndexWriterConfig iwConfig = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); + int nDocs = AtLeast(50); + Field id = new NumericDocValuesField("id", 0); + Field f; + switch (type) + { + case FieldInfo.DocValuesType_e.BINARY: + f = new BinaryDocValuesField("dv", new BytesRef()); + break; + case FieldInfo.DocValuesType_e.SORTED: + f = new SortedDocValuesField("dv", new BytesRef()); + break; + case FieldInfo.DocValuesType_e.NUMERIC: + f = new NumericDocValuesField("dv", 0); + break; + default: + throw new InvalidOperationException(); + } + Document document = new Document(); + document.Add(id); + document.Add(f); + + object[] vals = new object[nDocs]; + + RandomIndexWriter iw = new RandomIndexWriter(Random(), d, iwConfig); + for (int i = 0; i < nDocs; ++i) + { + id.LongValue = i; + switch (type) + { + case FieldInfo.DocValuesType_e.SORTED: + case FieldInfo.DocValuesType_e.BINARY: + do + { + vals[i] = TestUtil.RandomSimpleString(Random(), 20); + } while (((string)vals[i]).Length == 0); + f.BytesValue = new BytesRef((string)vals[i]); + break; + case FieldInfo.DocValuesType_e.NUMERIC: + int bitsPerValue = Random().NextIntBetween(1, 31); // keep it an int + vals[i] = (long)Random().Next((int)PackedInts.MaxValue(bitsPerValue)); + f.LongValue = (long) vals[i]; + break; + } + iw.AddDocument(document); + if (Random().NextBoolean() && i % 10 == 9) + { + iw.Commit(); + } + } + iw.Dispose(); + + DirectoryReader rd = DirectoryReader.Open(d); + foreach (AtomicReaderContext leave in rd.Leaves) + { + FunctionValues ids = (new LongFieldSource("id")).GetValues(null, leave); + ValueSource vs; + switch (type) + { + case FieldInfo.DocValuesType_e.BINARY: + case FieldInfo.DocValuesType_e.SORTED: + vs = new BytesRefFieldSource("dv"); + break; + case FieldInfo.DocValuesType_e.NUMERIC: + vs = new LongFieldSource("dv"); + break; + default: + throw new InvalidOperationException(); + } + FunctionValues values = vs.GetValues(null, leave); + BytesRef bytes = new BytesRef(); + for (int i = 0; i < leave.AtomicReader.MaxDoc; ++i) + { + assertTrue(values.Exists(i)); + if (vs is BytesRefFieldSource) + { + assertTrue(values.ObjectVal(i) is string); + } + else if (vs is LongFieldSource) + { + assertTrue(values.ObjectVal(i) is long?); + assertTrue(values.BytesVal(i, bytes)); + } + else + { + throw new InvalidOperationException(); + } + + object expected = vals[ids.IntVal(i)]; + switch (type) + { + case FieldInfo.DocValuesType_e.SORTED: + values.OrdVal(i); // no exception + assertTrue(values.NumOrd() >= 1); + goto case FieldInfo.DocValuesType_e.BINARY; + case FieldInfo.DocValuesType_e.BINARY: + assertEquals(expected, values.ObjectVal(i)); + assertEquals(expected, values.StrVal(i)); + assertEquals(expected, values.ObjectVal(i)); + assertEquals(expected, values.StrVal(i)); + assertTrue(values.BytesVal(i, bytes)); + assertEquals(new BytesRef((string)expected), bytes); + break; + case FieldInfo.DocValuesType_e.NUMERIC: + assertEquals(Number.ToInt64(expected.ToString()), values.LongVal(i)); + break; + } + } + } + rd.Dispose(); + d.Dispose(); + } + + [Test] + public void Test() + { + var values = Enum.GetValues(typeof(FieldInfo.DocValuesType_e)); + foreach (FieldInfo.DocValuesType_e type in values) + { + if (type != FieldInfo.DocValuesType_e.SORTED_SET) + { + DoTest(type); + } + } + } + + } +} \ No newline at end of file diff --git a/src/Lucene.Net.Tests.Queries/Function/TestFieldScoreQuery.cs b/src/Lucene.Net.Tests.Queries/Function/TestFieldScoreQuery.cs new file mode 100644 index 0000000000..39938bd2a0 --- /dev/null +++ b/src/Lucene.Net.Tests.Queries/Function/TestFieldScoreQuery.cs @@ -0,0 +1,162 @@ +using Lucene.Net.Index; +using Lucene.Net.Queries.Function; +using Lucene.Net.Search; +using NUnit.Framework; + +namespace Lucene.Net.Tests.Queries.Function +{ + /// + /// Test FieldScoreQuery search. + ///

+ /// Tests here create an index with a few documents, each having + /// an int value indexed field and a float value indexed field. + /// The values of these fields are later used for scoring. + ///

+ /// The rank tests use Hits to verify that docs are ordered (by score) as expected. + ///

+ /// The exact score tests use TopDocs top to verify the exact score. + ///

+ public class TestFieldScoreQuery : FunctionTestSetup + { + [SetUp] + public override void SetUp() + { + base.SetUp(); + CreateIndex(true); + } + + /// + /// Test that FieldScoreQuery of Type.BYTE returns docs in expected order. + /// + [Test] + public void TestRankByte() + { + // INT field values are small enough to be parsed as byte + DoTestRank(BYTE_VALUESOURCE); + } + + /// + /// Test that FieldScoreQuery of Type.SHORT returns docs in expected order. + /// + [Test] + public void TestRankShort() + { + // INT field values are small enough to be parsed as short + DoTestRank(SHORT_VALUESOURCE); + } + + /// + /// Test that FieldScoreQuery of Type.INT returns docs in expected order. + /// + [Test] + public void TestRankInt() + { + DoTestRank(INT_VALUESOURCE); + } + + /// + /// Test that FieldScoreQuery of Type.FLOAT returns docs in expected order. + /// + [Test] + public void TestRankFloat() + { + // INT field can be parsed as float + DoTestRank(INT_AS_FLOAT_VALUESOURCE); + // same values, but in flot format + DoTestRank(FLOAT_VALUESOURCE); + } + + /// + /// Test that FieldScoreQuery returns docs in expected order. + /// + /// + private void DoTestRank(ValueSource valueSource) + { + FunctionQuery functionQuery = new FunctionQuery(valueSource); + IndexReader r = DirectoryReader.Open(dir); + IndexSearcher s = NewSearcher(r); + Log("test: " + functionQuery); + QueryUtils.Check(Random(), functionQuery, s); + ScoreDoc[] h = s.Search(functionQuery, null, 1000).ScoreDocs; + assertEquals("All docs should be matched!", N_DOCS, h.Length); + string prevID = "ID" + (N_DOCS + 1); // greater than all ids of docs in this test + for (int i = 0; i < h.Length; i++) + { + string resID = s.Doc(h[i].Doc).Get(ID_FIELD); + Log(i + ". score=" + h[i].Score + " - " + resID); + Log(s.Explain(functionQuery, h[i].Doc)); + assertTrue("res id " + resID + " should be < prev res id " + prevID, resID.CompareTo(prevID) < 0); + prevID = resID; + } + r.Dispose(); + } + + /// + /// Test that FieldScoreQuery of Type.BYTE returns the expected scores. + //JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: + //ORIGINAL LINE: @Test public void testExactScoreByte() throws Exception + //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: + public virtual void testExactScoreByte() + { + // INT field values are small enough to be parsed as byte + doTestExactScore(BYTE_VALUESOURCE); + } + + /// + /// Test that FieldScoreQuery of Type.SHORT returns the expected scores. + //JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: + //ORIGINAL LINE: @Test public void testExactScoreShort() throws Exception + //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: + public virtual void testExactScoreShort() + { + // INT field values are small enough to be parsed as short + doTestExactScore(SHORT_VALUESOURCE); + } + + /// + /// Test that FieldScoreQuery of Type.INT returns the expected scores. + //JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: + //ORIGINAL LINE: @Test public void testExactScoreInt() throws Exception + //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: + public virtual void testExactScoreInt() + { + doTestExactScore(INT_VALUESOURCE); + } + + /// + /// Test that FieldScoreQuery of Type.FLOAT returns the expected scores. + //JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: + //ORIGINAL LINE: @Test public void testExactScoreFloat() throws Exception + //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: + public virtual void testExactScoreFloat() + { + // INT field can be parsed as float + doTestExactScore(INT_AS_FLOAT_VALUESOURCE); + // same values, but in flot format + doTestExactScore(FLOAT_VALUESOURCE); + } + + // Test that FieldScoreQuery returns docs with expected score. + //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: + //ORIGINAL LINE: private void doTestExactScore(ValueSource valueSource) throws Exception + private void doTestExactScore(ValueSource valueSource) + { + FunctionQuery functionQuery = new FunctionQuery(valueSource); + IndexReader r = DirectoryReader.Open(dir); + IndexSearcher s = NewSearcher(r); + TopDocs td = s.Search(functionQuery, null, 1000); + assertEquals("All docs should be matched!", N_DOCS, td.TotalHits); + ScoreDoc[] sd = td.ScoreDocs; + foreach (ScoreDoc aSd in sd) + { + float score = aSd.Score; + Log(s.Explain(functionQuery, aSd.Doc)); + string id = s.IndexReader.Document(aSd.Doc).Get(ID_FIELD); + float expectedScore = ExpectedFieldScore(id); // "ID7" --> 7.0 + assertEquals("score of " + id + " shuould be " + expectedScore + " != " + score, expectedScore, score, TEST_SCORE_TOLERANCE_DELTA); + } + r.Dispose(); + } + + } +} \ No newline at end of file diff --git a/src/Lucene.Net.Tests.Queries/Function/TestFunctionQuerySort.cs b/src/Lucene.Net.Tests.Queries/Function/TestFunctionQuerySort.cs new file mode 100644 index 0000000000..cbec34cb3f --- /dev/null +++ b/src/Lucene.Net.Tests.Queries/Function/TestFunctionQuerySort.cs @@ -0,0 +1,80 @@ +using System; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Lucene.Net.Queries.Function.ValueSources; +using Lucene.Net.Search; +using Lucene.Net.Store; +using Lucene.Net.Util; +using NUnit.Framework; + +namespace Lucene.Net.Tests.Queries.Function +{ + /// + /// Test that functionquery's getSortField() actually works. + /// + public class TestFunctionQuerySort : LuceneTestCase + { + [Test] + public void TestSearchAfterWhenSortingByFunctionValues() + { + Directory dir = NewDirectory(); + IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, null); + iwc.SetMergePolicy(NewLogMergePolicy()); // depends on docid order + RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, iwc); + + Document doc = new Document(); + Field field = new StringField("value", "", Field.Store.YES); + doc.Add(field); + + // Save docs unsorted (decreasing value n, n-1, ...) + const int NUM_VALS = 5; + for (int val = NUM_VALS; val > 0; val--) + { + field.StringValue = Convert.ToString(val); + writer.AddDocument(doc); + } + + // Open index + IndexReader reader = writer.Reader; + writer.Dispose(); + IndexSearcher searcher = NewSearcher(reader); + + // Get ValueSource from FieldCache + IntFieldSource src = new IntFieldSource("value"); + // ...and make it a sort criterion + SortField sf = src.GetSortField(false).Rewrite(searcher); + Sort orderBy = new Sort(sf); + + // Get hits sorted by our FunctionValues (ascending values) + Query q = new MatchAllDocsQuery(); + TopDocs hits = searcher.Search(q, reader.MaxDoc, orderBy); + assertEquals(NUM_VALS, hits.ScoreDocs.Length); + // Verify that sorting works in general + int i = 0; + foreach (ScoreDoc hit in hits.ScoreDocs) + { + int valueFromDoc = Convert.ToInt32(reader.Document(hit.Doc).Get("value")); + assertEquals(++i, valueFromDoc); + } + + // Now get hits after hit #2 using IS.searchAfter() + int afterIdx = 1; + FieldDoc afterHit = (FieldDoc)hits.ScoreDocs[afterIdx]; + hits = searcher.SearchAfter(afterHit, q, reader.MaxDoc, orderBy); + + // Expected # of hits: NUM_VALS - 2 + assertEquals(NUM_VALS - (afterIdx + 1), hits.ScoreDocs.Length); + + // Verify that hits are actually "after" + int afterValue = (int)((double?)afterHit.Fields[0]); + foreach (ScoreDoc hit in hits.ScoreDocs) + { + int val = Convert.ToInt32(reader.Document(hit.Doc).Get("value")); + assertTrue(afterValue <= val); + assertFalse(hit.Doc == afterHit.Doc); + } + reader.Dispose(); + dir.Dispose(); + } + } +} \ No newline at end of file diff --git a/src/Lucene.Net.Tests.Queries/Function/TestLongNormValueSource.cs b/src/Lucene.Net.Tests.Queries/Function/TestLongNormValueSource.cs new file mode 100644 index 0000000000..38ebe7feb4 --- /dev/null +++ b/src/Lucene.Net.Tests.Queries/Function/TestLongNormValueSource.cs @@ -0,0 +1,230 @@ +using System; +using Lucene.Net.Analysis; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Lucene.Net.Queries.Function; +using Lucene.Net.Queries.Function.ValueSources; +using Lucene.Net.Search; +using Lucene.Net.Search.Similarities; +using Lucene.Net.Store; +using Lucene.Net.Util; +using NUnit.Framework; + +namespace Lucene.Net.Tests.Queries.Function +{ + // [Util.LuceneTestCase.SuppressCodecs("Lucene3x")] + public class TestLongNormValueSource : LuceneTestCase + { + internal static Directory dir; + internal static IndexReader reader; + internal static IndexSearcher searcher; + private static Similarity sim = new PreciseDefaultSimilarity(); + + [SetUp] + public override void SetUp() + { + base.SetUp(); + + dir = NewDirectory(); + IndexWriterConfig iwConfig = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); + iwConfig.SetMergePolicy(NewLogMergePolicy()); + iwConfig.SetSimilarity(sim); + RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwConfig); + + Document doc = new Document(); + doc.Add(new TextField("text", "this is a test test test", Field.Store.NO)); + iw.AddDocument(doc); + + doc = new Document(); + doc.Add(new TextField("text", "second test", Field.Store.NO)); + iw.AddDocument(doc); + + reader = iw.Reader; + searcher = NewSearcher(reader); + iw.Dispose(); + } + + [TearDown] + public override void TearDown() + { + base.TearDown(); + + searcher = null; + reader.Dispose(); + reader = null; + dir.Dispose(); + dir = null; + } + + [Test] + public void TestNorm() + { + Similarity saved = searcher.Similarity; + try + { + // no norm field (so agnostic to indexed similarity) + searcher.Similarity = sim; + AssertHits(new FunctionQuery(new NormValueSource("text")), new float[] { 0f, 0f }); + } + finally + { + searcher.Similarity = saved; + } + } + + protected virtual void AssertHits(Query q, float[] scores) + { + ScoreDoc[] expected = new ScoreDoc[scores.Length]; + int[] expectedDocs = new int[scores.Length]; + for (int i = 0; i < expected.Length; i++) + { + expectedDocs[i] = i; + expected[i] = new ScoreDoc(i, scores[i]); + } + TopDocs docs = searcher.Search(q, 2, new Sort(new SortField("id", SortField.Type_e.STRING))); + + /* + for (int i=0;i + /// Encodes norm as 4-byte float. + internal class PreciseDefaultSimilarity : TFIDFSimilarity + { + /// + /// Sole constructor: parameter-free + public PreciseDefaultSimilarity() + { + } + + /// + /// Implemented as overlap / maxOverlap. + public override float Coord(int overlap, int maxOverlap) + { + return overlap / (float)maxOverlap; + } + + /// + /// Implemented as 1/sqrt(sumOfSquaredWeights). + public override float QueryNorm(float sumOfSquaredWeights) + { + return (float)(1.0 / Math.Sqrt(sumOfSquaredWeights)); + } + + /// + /// Encodes a normalization factor for storage in an index. + ///

+ /// The encoding uses a three-bit mantissa, a five-bit exponent, and the + /// zero-exponent point at 15, thus representing values from around 7x10^9 to + /// 2x10^-9 with about one significant decimal digit of accuracy. Zero is also + /// represented. Negative numbers are rounded up to zero. Values too large to + /// represent are rounded down to the largest representable value. Positive + /// values too small to represent are rounded up to the smallest positive + /// representable value. + ///

+ /// + /// + public override long EncodeNormValue(float f) + { + return BitConverter.DoubleToInt64Bits(f); + } + + /// + /// Decodes the norm value, assuming it is a single byte. + /// + /// + public override float DecodeNormValue(long norm) + { + return (float) BitConverter.Int64BitsToDouble(norm); + } + + /// + /// Implemented as + /// state.getBoost()*lengthNorm(numTerms), where + /// numTerms is if {@link + /// #setDiscountOverlaps} is false, else it's {@link + /// org.apache.lucene.index.FieldInvertState#getLength()} - {@link + /// org.apache.lucene.index.FieldInvertState#getNumOverlap()}. + /// + /// @lucene.experimental + /// + public override float LengthNorm(FieldInvertState state) + { + int numTerms; + if (discountOverlaps) + { + numTerms = state.Length - state.NumOverlap; + } + else + { + numTerms = state.Length; + } + return state.Boost * ((float)(1.0 / Math.Sqrt(numTerms))); + } + + /// + /// Implemented as sqrt(freq). + public override float Tf(float freq) + { + return (float)Math.Sqrt(freq); + } + + /// + /// Implemented as 1 / (distance + 1). + /// + public override float SloppyFreq(int distance) + { + return 1.0f / (distance + 1); + } + + /// + /// The default implementation returns 1 + /// + public override float ScorePayload(int doc, int start, int end, BytesRef payload) + { + return 1; + } + + /// + /// Implemented as log(numDocs/(docFreq+1)) + 1. + /// + public override float Idf(long docFreq, long numDocs) + { + return (float)(Math.Log(numDocs / (double)(docFreq + 1)) + 1.0); + } + + /// + /// True if overlap tokens (tokens with a position of increment of zero) are + /// discounted from the document's length. + /// + protected internal bool discountOverlaps = true; + + /// + /// Determines whether overlap tokens (Tokens with + /// 0 position increment) are ignored when computing + /// norm. By default this is true, meaning overlap + /// tokens do not count when computing norms. + /// + /// @lucene.experimental + /// + /// + public virtual bool DiscountOverlaps + { + set { discountOverlaps = value; } + get { return discountOverlaps; } + } + + public override string ToString() + { + return "DefaultSimilarity"; + } + } +} \ No newline at end of file diff --git a/src/Lucene.Net.Tests.Queries/Function/TestOrdValues.cs b/src/Lucene.Net.Tests.Queries/Function/TestOrdValues.cs new file mode 100644 index 0000000000..ef396ee346 --- /dev/null +++ b/src/Lucene.Net.Tests.Queries/Function/TestOrdValues.cs @@ -0,0 +1,157 @@ +using Lucene.Net.Index; +using Lucene.Net.Queries.Function; +using Lucene.Net.Queries.Function.ValueSources; +using Lucene.Net.Search; +using NUnit.Framework; + +namespace Lucene.Net.Tests.Queries.Function +{ + /// + /// Test search based on OrdFieldSource and ReverseOrdFieldSource. + ///

+ /// Tests here create an index with a few documents, each having + /// an indexed "id" field. + /// The ord values of this field are later used for scoring. + ///

+ /// The order tests use Hits to verify that docs are ordered as expected. + ///

+ /// The exact score tests use TopDocs top to verify the exact score. + ///

+ public class TestOrdValues : FunctionTestSetup + { + [SetUp] + public override void SetUp() + { + base.SetUp(); + CreateIndex(false); + } + + /// + /// Test OrdFieldSource + /// + [Test] + public void TestOrdFieldRank() + { + DoTestRank(ID_FIELD, true); + } + + /// + /// Test ReverseOrdFieldSource + /// + [Test] + public void TestReverseOrdFieldRank() + { + DoTestRank(ID_FIELD, false); + } + + /// + /// Test that queries based on reverse/ordFieldScore scores correctly + /// + /// + /// + private static void DoTestRank(string field, bool inOrder) + { + IndexReader r = DirectoryReader.Open(dir); + IndexSearcher s = NewSearcher(r); + ValueSource vs; + if (inOrder) + { + vs = new OrdFieldSource(field); + } + else + { + vs = new ReverseOrdFieldSource(field); + } + + Query q = new FunctionQuery(vs); + Log("test: " + q); + QueryUtils.Check(Random(), q, s); + ScoreDoc[] h = s.Search(q, null, 1000).ScoreDocs; + assertEquals("All docs should be matched!", N_DOCS, h.Length); + string prevID = inOrder ? "IE" : "IC"; // smaller than all ids of docs in this test ("ID0001", etc.) - greater than all ids of docs in this test ("ID0001", etc.) + + for (int i = 0; i < h.Length; i++) + { + string resID = s.Doc(h[i].Doc).Get(ID_FIELD); + Log(i + ". score=" + h[i].Score + " - " + resID); + Log(s.Explain(q, h[i].Doc)); + if (inOrder) + { + assertTrue("res id " + resID + " should be < prev res id " + prevID, resID.CompareTo(prevID) < 0); + } + else + { + assertTrue("res id " + resID + " should be > prev res id " + prevID, resID.CompareTo(prevID) > 0); + } + prevID = resID; + } + r.Dispose(); + } + + /// + /// Test exact score for OrdFieldSource + /// + [Test] + public void TestOrdFieldExactScore() + { + DoTestExactScore(ID_FIELD, true); + } + + /// + /// Test exact score for ReverseOrdFieldSource + /// + [Test] + public void TestReverseOrdFieldExactScore() + { + DoTestExactScore(ID_FIELD, false); + } + + + /// + /// Test that queries based on reverse/ordFieldScore returns docs with expected score. + /// + /// + /// + private void DoTestExactScore(string field, bool inOrder) + { + IndexReader r = DirectoryReader.Open(dir); + IndexSearcher s = NewSearcher(r); + ValueSource vs; + if (inOrder) + { + vs = new OrdFieldSource(field); + } + else + { + vs = new ReverseOrdFieldSource(field); + } + Query q = new FunctionQuery(vs); + TopDocs td = s.Search(q, null, 1000); + assertEquals("All docs should be matched!", N_DOCS, td.TotalHits); + ScoreDoc[] sd = td.ScoreDocs; + for (int i = 0; i < sd.Length; i++) + { + float score = sd[i].Score; + string id = s.IndexReader.Document(sd[i].Doc).Get(ID_FIELD); + Log("-------- " + i + ". Explain doc " + id); + Log(s.Explain(q, sd[i].Doc)); + float expectedScore = N_DOCS - i - 1; + assertEquals("score of result " + i + " shuould be " + expectedScore + " != " + score, expectedScore, score, TEST_SCORE_TOLERANCE_DELTA); + string expectedId = inOrder ? Id2String(N_DOCS - i) : Id2String(i + 1); // reverse ==> smaller values first - in-order ==> larger values first + assertTrue("id of result " + i + " shuould be " + expectedId + " != " + score, expectedId.Equals(id)); + } + r.Dispose(); + } + + // LUCENE-1250 + [Test] + public void TestEqualsNull() + { + OrdFieldSource ofs = new OrdFieldSource("f"); + assertFalse(ofs.Equals(null)); + + ReverseOrdFieldSource rofs = new ReverseOrdFieldSource("f"); + assertFalse(rofs.Equals(null)); + } + } +} \ No newline at end of file diff --git a/src/Lucene.Net.Tests.Queries/Function/TestValueSources.cs b/src/Lucene.Net.Tests.Queries/Function/TestValueSources.cs new file mode 100644 index 0000000000..d48d41c275 --- /dev/null +++ b/src/Lucene.Net.Tests.Queries/Function/TestValueSources.cs @@ -0,0 +1,339 @@ +using System; +using System.Collections.Generic; +using Lucene.Net.Analysis; +using Lucene.Net.Codecs; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Lucene.Net.Queries.Function; +using Lucene.Net.Queries.Function.ValueSources; +using Lucene.Net.Search; +using Lucene.Net.Search.Similarities; +using Lucene.Net.Store; +using Lucene.Net.Util; +using NUnit.Framework; + +namespace Lucene.Net.Tests.Queries.Function +{ + // TODO: add separate docvalues test + /// + /// barebones tests for function queries. + /// + public class TestValueSources : LuceneTestCase + { + internal static Directory dir; + internal static IndexReader reader; + internal static IndexSearcher searcher; + + internal static readonly IList documents = new[] + { + /* id, byte, double, float, int, long, short, string, text */ + new[] { "0", "5", "3.63", "5.2", "35", "4343", "945", "test", "this is a test test test" }, + new[] { "1", "12", "5.65", "9.3", "54", "1954", "123", "bar", "second test" } + }; + + [SetUp] + public override void SetUp() + { + base.SetUp(); + + dir = NewDirectory(); + IndexWriterConfig iwConfig = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); + iwConfig.SetMergePolicy(NewLogMergePolicy()); + RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwConfig); + Document document = new Document(); + Field idField = new StringField("id", "", Field.Store.NO); + document.Add(idField); + Field byteField = new StringField("byte", "", Field.Store.NO); + document.Add(byteField); + Field doubleField = new StringField("double", "", Field.Store.NO); + document.Add(doubleField); + Field floatField = new StringField("float", "", Field.Store.NO); + document.Add(floatField); + Field intField = new StringField("int", "", Field.Store.NO); + document.Add(intField); + Field longField = new StringField("long", "", Field.Store.NO); + document.Add(longField); + Field shortField = new StringField("short", "", Field.Store.NO); + document.Add(shortField); + Field stringField = new StringField("string", "", Field.Store.NO); + document.Add(stringField); + Field textField = new TextField("text", "", Field.Store.NO); + document.Add(textField); + + foreach (string[] doc in documents) + { + idField.StringValue = doc[0]; + byteField.StringValue = doc[1]; + doubleField.StringValue = doc[2]; + floatField.StringValue = doc[3]; + intField.StringValue = doc[4]; + longField.StringValue = doc[5]; + shortField.StringValue = doc[6]; + stringField.StringValue = doc[7]; + textField.StringValue = doc[8]; + iw.AddDocument(document); + } + + reader = iw.Reader; + searcher = NewSearcher(reader); + iw.Dispose(); + } + + [TearDown] + public override void TearDown() + { + base.TearDown(); + + searcher = null; + reader.Dispose(); + reader = null; + dir.Dispose(); + dir = null; + } + + [Test] + public void TestByte() + { + AssertHits(new FunctionQuery(new ByteFieldSource("byte")), new[] { 5f, 12f }); + } + + [Test] + public void TestConst() + { + AssertHits(new FunctionQuery(new ConstValueSource(0.3f)), new[] { 0.3f, 0.3f }); + } + + [Test] + public void TestDiv() + { + AssertHits(new FunctionQuery(new DivFloatFunction(new ConstValueSource(10f), new ConstValueSource(5f))), new[] { 2f, 2f }); + } + + [Test] + public void TestDocFreq() + { + AssertHits(new FunctionQuery(new DocFreqValueSource("bogus", "bogus", "text", new BytesRef("test"))), new[] { 2f, 2f }); + } + + [Test] + public void TestDoubleConst() + { + AssertHits(new FunctionQuery(new DoubleConstValueSource(0.3d)), new[] { 0.3f, 0.3f }); + } + + [Test] + public void TestDouble() + { + AssertHits(new FunctionQuery(new DoubleFieldSource("double")), new[] { 3.63f, 5.65f }); + } + + [Test] + public void TestFloat() + { + AssertHits(new FunctionQuery(new FloatFieldSource("float")), new[] { 5.2f, 9.3f }); + } + + [Test] + public void TestIDF() + { + Similarity saved = searcher.Similarity; + try + { + searcher.Similarity = new DefaultSimilarity(); + AssertHits(new FunctionQuery(new IDFValueSource("bogus", "bogus", "text", new BytesRef("test"))), new[] { 0.5945349f, 0.5945349f }); + } + finally + { + searcher.Similarity = saved; + } + } + + [Test] + public void TestIf() + { + AssertHits(new FunctionQuery(new IfFunction(new BytesRefFieldSource("id"), new ConstValueSource(1.0f), new ConstValueSource(2.0f) + )), new[] { 1f, 1f }); + // true just if a value exists... + AssertHits(new FunctionQuery(new IfFunction(new LiteralValueSource("false"), new ConstValueSource(1.0f), new ConstValueSource(2.0f) + )), new[] { 1f, 1f }); + } + + [Test] + public void TestInt() + { + AssertHits(new FunctionQuery(new IntFieldSource("int")), new[] { 35f, 54f }); + } + + [Test] + public void TestJoinDocFreq() + { + AssertHits(new FunctionQuery(new JoinDocFreqValueSource("string", "text")), new[] { 2f, 0f }); + } + + [Test] + public void TestLinearFloat() + { + AssertHits(new FunctionQuery(new LinearFloatFunction(new ConstValueSource(2.0f), 3, 1)), new[] { 7f, 7f }); + } + + [Test] + public void TestLong() + { + AssertHits(new FunctionQuery(new LongFieldSource("long")), new[] { 4343f, 1954f }); + } + + [Test] + public void TestMaxDoc() + { + AssertHits(new FunctionQuery(new MaxDocValueSource()), new[] { 2f, 2f }); + } + + [Test] + public void TestMaxFloat() + { + AssertHits(new FunctionQuery(new MaxFloatFunction(new ValueSource[] { new ConstValueSource(1f), new ConstValueSource(2f) })), new[] { 2f, 2f }); + } + + [Test] + public void TestMinFloat() + { + AssertHits(new FunctionQuery(new MinFloatFunction(new ValueSource[] { new ConstValueSource(1f), new ConstValueSource(2f) })), new[] { 1f, 1f }); + } + + [Test] + public void TestNorm() + { + Similarity saved = searcher.Similarity; + try + { + // no norm field (so agnostic to indexed similarity) + searcher.Similarity = new DefaultSimilarity(); + AssertHits(new FunctionQuery(new NormValueSource("byte")), new[] { 0f, 0f }); + } + finally + { + searcher.Similarity = saved; + } + } + + [Test] + public void TestNumDocs() + { + AssertHits(new FunctionQuery(new NumDocsValueSource()), new[] { 2f, 2f }); + } + + [Test] + public void TestPow() + { + AssertHits(new FunctionQuery(new PowFloatFunction(new ConstValueSource(2f), new ConstValueSource(3f))), new[] { 8f, 8f }); + } + + [Test] + public void TestProduct() + { + AssertHits(new FunctionQuery(new ProductFloatFunction(new ValueSource[] { new ConstValueSource(2f), new ConstValueSource(3f) })), new[] { 6f, 6f }); + } + + [Test] + public void TestQuery() + { + AssertHits(new FunctionQuery(new QueryValueSource(new FunctionQuery(new ConstValueSource(2f)), 0f)), new[] { 2f, 2f }); + } + + [Test] + public void TestRangeMap() + { + AssertHits(new FunctionQuery(new RangeMapFloatFunction(new FloatFieldSource("float"), 5, 6, 1, 0f)), new[] { 1f, 0f }); + AssertHits(new FunctionQuery(new RangeMapFloatFunction(new FloatFieldSource("float"), 5, 6, new SumFloatFunction(new ValueSource[] { new ConstValueSource(1f), new ConstValueSource(2f) }), new ConstValueSource(11f))), new[] { 3f, 11f }); + } + + [Test] + public void TestReciprocal() + { + AssertHits(new FunctionQuery(new ReciprocalFloatFunction(new ConstValueSource(2f), 3, 1, 4)), new[] { 0.1f, 0.1f }); + } + + [Test] + public void TestScale() + { + AssertHits(new FunctionQuery(new ScaleFloatFunction(new IntFieldSource("int"), 0, 1)), new[] { 0.0f, 1.0f }); + } + + [Test] + public void TestShort() + { + AssertHits(new FunctionQuery(new ShortFieldSource("short")), new[] { 945f, 123f }); + } + + [Test] + public void TestSumFloat() + { + AssertHits(new FunctionQuery(new SumFloatFunction(new ValueSource[] { new ConstValueSource(1f), new ConstValueSource(2f) })), new[] { 3f, 3f }); + } + + [Test] + public void TestSumTotalTermFreq() + { + if (Codec.Default.Name.Equals("Lucene3x")) + { + AssertHits(new FunctionQuery(new SumTotalTermFreqValueSource("text")), new[] { -1f, -1f }); + } + else + { + AssertHits(new FunctionQuery(new SumTotalTermFreqValueSource("text")), new[] { 8f, 8f }); + } + } + + [Test] + public void TestTermFreq() + { + AssertHits(new FunctionQuery(new TermFreqValueSource("bogus", "bogus", "text", new BytesRef("test"))), new[] { 3f, 1f }); + AssertHits(new FunctionQuery(new TermFreqValueSource("bogus", "bogus", "string", new BytesRef("bar"))), new[] { 0f, 1f }); + } + + [Test] + public void TestTF() + { + Similarity saved = searcher.Similarity; + try + { + // no norm field (so agnostic to indexed similarity) + searcher.Similarity = new DefaultSimilarity(); + AssertHits(new FunctionQuery(new TFValueSource("bogus", "bogus", "text", new BytesRef("test"))), new[] { (float)Math.Sqrt(3d), (float)Math.Sqrt(1d) }); + AssertHits(new FunctionQuery(new TFValueSource("bogus", "bogus", "string", new BytesRef("bar"))), new[] { 0f, 1f }); + } + finally + { + searcher.Similarity = saved; + } + } + + [Test] + public void TestTotalTermFreq() + { + if (Codec.Default.Name.Equals("Lucene3x")) + { + AssertHits(new FunctionQuery(new TotalTermFreqValueSource("bogus", "bogus", "text", new BytesRef("test"))), new[] { -1f, -1f }); + } + else + { + AssertHits(new FunctionQuery(new TotalTermFreqValueSource("bogus", "bogus", "text", new BytesRef("test"))), new[] { 4f, 4f }); + } + } + + private static void AssertHits(Query q, float[] scores) + { + ScoreDoc[] expected = new ScoreDoc[scores.Length]; + int[] expectedDocs = new int[scores.Length]; + for (int i = 0; i < expected.Length; i++) + { + expectedDocs[i] = i; + expected[i] = new ScoreDoc(i, scores[i]); + } + TopDocs docs = searcher.Search(q, null, documents.Count, new Sort(new SortField("id", SortField.Type_e.STRING)), true, false); + CheckHits.DoCheckHits(Random(), q, "", searcher, expectedDocs); + CheckHits.CheckHitsQuery(q, expected, docs.ScoreDocs, expectedDocs); + CheckHits.CheckExplanations(q, "", searcher); + } + } +} \ No newline at end of file diff --git a/src/Lucene.Net.Tests.Queries/Lucene.Net.Tests.Queries.csproj b/src/Lucene.Net.Tests.Queries/Lucene.Net.Tests.Queries.csproj index 9264faa7ee..125123dc24 100644 --- a/src/Lucene.Net.Tests.Queries/Lucene.Net.Tests.Queries.csproj +++ b/src/Lucene.Net.Tests.Queries/Lucene.Net.Tests.Queries.csproj @@ -46,8 +46,19 @@ + + + + + + + + + + + @@ -66,10 +77,7 @@ - - - - +