Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix iterator over global ordinals. #8627

Closed
wants to merge 1 commit into from
Closed

Fix iterator over global ordinals. #8627

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
6 changes: 5 additions & 1 deletion src/main/java/org/elasticsearch/index/fielddata/AbstractRandomAccessOrds.java
View file
Open in desktop
Expand Up @@ -38,7 +38,11 @@ public final void setDocument(int docID) {

@Override
public long nextOrd() {
return ordAt(i++);
if (i < cardinality()) {
return ordAt(i++);
} else {
return NO_MORE_ORDS;
}
}

}
13 changes: 9 additions & 4 deletions src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTests.java
View file
Open in desktop
Expand Up @@ -59,6 +59,11 @@ protected String toString(Object value) {

protected abstract void add2SingleValuedDocumentsAndDeleteOneOfThem() throws Exception;

protected long minRamBytesUsed() {
// minimum number of bytes that this fielddata instance is expected to require
return 1;
}

@Test
public void testDeletedDocs() throws Exception {
add2SingleValuedDocumentsAndDeleteOneOfThem();
Expand All @@ -78,7 +83,7 @@ public void testSingleValueAllSet() throws Exception {
IndexFieldData indexFieldData = getForField("value");
LeafReaderContext readerContext = refreshReader();
AtomicFieldData fieldData = indexFieldData.load(readerContext);
assertThat(fieldData.ramBytesUsed(), greaterThan(0l));
assertThat(fieldData.ramBytesUsed(), greaterThanOrEqualTo(minRamBytesUsed()));

SortedBinaryDocValues bytesValues = fieldData.getBytesValues();

Expand Down Expand Up @@ -141,7 +146,7 @@ public void testSingleValueWithMissing() throws Exception {
fillSingleValueWithMissing();
IndexFieldData indexFieldData = getForField("value");
AtomicFieldData fieldData = indexFieldData.load(refreshReader());
assertThat(fieldData.ramBytesUsed(), greaterThan(0l));
assertThat(fieldData.ramBytesUsed(), greaterThanOrEqualTo(minRamBytesUsed()));

SortedBinaryDocValues bytesValues = fieldData
.getBytesValues();
Expand All @@ -158,7 +163,7 @@ public void testMultiValueAllSet() throws Exception {
fillMultiValueAllSet();
IndexFieldData indexFieldData = getForField("value");
AtomicFieldData fieldData = indexFieldData.load(refreshReader());
assertThat(fieldData.ramBytesUsed(), greaterThan(0l));
assertThat(fieldData.ramBytesUsed(), greaterThanOrEqualTo(minRamBytesUsed()));

SortedBinaryDocValues bytesValues = fieldData.getBytesValues();

Expand Down Expand Up @@ -189,7 +194,7 @@ public void testMultiValueWithMissing() throws Exception {
fillMultiValueWithMissing();
IndexFieldData indexFieldData = getForField("value");
AtomicFieldData fieldData = indexFieldData.load(refreshReader());
assertThat(fieldData.ramBytesUsed(), greaterThan(0l));
assertThat(fieldData.ramBytesUsed(), greaterThanOrEqualTo(minRamBytesUsed()));

SortedBinaryDocValues bytesValues = fieldData.getBytesValues();

Expand Down
154 changes: 88 additions & 66 deletions src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTests.java
View file
Open in desktop
Expand Up @@ -20,9 +20,11 @@
package org.elasticsearch.index.fielddata;

import com.carrotsearch.randomizedtesting.generators.RandomPicks;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.LeafReaderContext;
Expand Down Expand Up @@ -69,32 +71,37 @@
*/
public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImplTests {

private void addField(Document d, String name, String value) {
d.add(new StringField(name, value, Field.Store.YES));
d.add(new SortedSetDocValuesField(name, new BytesRef(value)));
}

protected void fillSingleValueAllSet() throws Exception {
Document d = new Document();
d.add(new StringField("_id", "1", Field.Store.NO));
d.add(new StringField("value", "2", Field.Store.NO));
addField(d, "_id", "1");
addField(d, "value", "2");
writer.addDocument(d);

d = new Document();
d.add(new StringField("_id", "1", Field.Store.NO));
d.add(new StringField("value", "1", Field.Store.NO));
addField(d, "_id", "1");
addField(d, "value", "1");
writer.addDocument(d);

d = new Document();
d.add(new StringField("_id", "3", Field.Store.NO));
d.add(new StringField("value", "3", Field.Store.NO));
addField(d, "_id", "3");
addField(d, "value", "3");
writer.addDocument(d);
}

protected void add2SingleValuedDocumentsAndDeleteOneOfThem() throws Exception {
Document d = new Document();
d.add(new StringField("_id", "1", Field.Store.NO));
d.add(new StringField("value", "2", Field.Store.NO));
addField(d, "_id", "1");
addField(d, "value", "2");
writer.addDocument(d);

d = new Document();
d.add(new StringField("_id", "2", Field.Store.NO));
d.add(new StringField("value", "4", Field.Store.NO));
addField(d, "_id", "2");
addField(d, "value", "4");
writer.addDocument(d);

writer.commit();
Expand All @@ -104,120 +111,120 @@ protected void add2SingleValuedDocumentsAndDeleteOneOfThem() throws Exception {

protected void fillSingleValueWithMissing() throws Exception {
Document d = new Document();
d.add(new StringField("_id", "1", Field.Store.NO));
d.add(new StringField("value", "2", Field.Store.NO));
addField(d, "_id", "1");
addField(d, "value", "2");
writer.addDocument(d);

d = new Document();
d.add(new StringField("_id", "2", Field.Store.NO));
addField(d, "_id", "2");
//d.add(new StringField("value", one(), Field.Store.NO)); // MISSING....
writer.addDocument(d);

d = new Document();
d.add(new StringField("_id", "3", Field.Store.NO));
d.add(new StringField("value", "3", Field.Store.NO));
addField(d, "_id", "3");
addField(d, "value", "3");
writer.addDocument(d);
}

protected void fillMultiValueAllSet() throws Exception {
Document d = new Document();
d.add(new StringField("_id", "1", Field.Store.NO));
d.add(new StringField("value", "2", Field.Store.NO));
d.add(new StringField("value", "4", Field.Store.NO));
addField(d, "_id", "1");
addField(d, "value", "2");
addField(d, "value", "4");
writer.addDocument(d);

d = new Document();
d.add(new StringField("_id", "2", Field.Store.NO));
d.add(new StringField("value", "1", Field.Store.NO));
addField(d, "_id", "2");
addField(d, "value", "1");
writer.addDocument(d);
writer.commit(); // TODO: Have tests with more docs for sorting

d = new Document();
d.add(new StringField("_id", "3", Field.Store.NO));
d.add(new StringField("value", "3", Field.Store.NO));
addField(d, "_id", "3");
addField(d, "value", "3");
writer.addDocument(d);
}

protected void fillMultiValueWithMissing() throws Exception {
Document d = new Document();
d.add(new StringField("_id", "1", Field.Store.NO));
d.add(new StringField("value", "2", Field.Store.NO));
d.add(new StringField("value", "4", Field.Store.NO));
addField(d, "_id", "1");
addField(d, "value", "2");
addField(d, "value", "4");
writer.addDocument(d);

d = new Document();
d.add(new StringField("_id", "2", Field.Store.NO));
addField(d, "_id", "2");
//d.add(new StringField("value", one(), Field.Store.NO)); // MISSING
writer.addDocument(d);

d = new Document();
d.add(new StringField("_id", "3", Field.Store.NO));
d.add(new StringField("value", "3", Field.Store.NO));
addField(d, "_id", "3");
addField(d, "value", "3");
writer.addDocument(d);
}

protected void fillAllMissing() throws Exception {
Document d = new Document();
d.add(new StringField("_id", "1", Field.Store.NO));
addField(d, "_id", "1");
writer.addDocument(d);

d = new Document();
d.add(new StringField("_id", "2", Field.Store.NO));
addField(d, "_id", "2");
writer.addDocument(d);

d = new Document();
d.add(new StringField("_id", "3", Field.Store.NO));
addField(d, "_id", "3");
writer.addDocument(d);
}

protected void fillExtendedMvSet() throws Exception {
Document d = new Document();
d.add(new StringField("_id", "1", Field.Store.NO));
d.add(new StringField("value", "02", Field.Store.NO));
d.add(new StringField("value", "04", Field.Store.NO));
addField(d, "_id", "1");
addField(d, "value", "02");
addField(d, "value", "04");
writer.addDocument(d);

d = new Document();
d.add(new StringField("_id", "2", Field.Store.NO));
addField(d, "_id", "2");
writer.addDocument(d);

d = new Document();
d.add(new StringField("_id", "3", Field.Store.NO));
d.add(new StringField("value", "03", Field.Store.NO));
addField(d, "_id", "3");
addField(d, "value", "03");
writer.addDocument(d);
writer.commit();

d = new Document();
d.add(new StringField("_id", "4", Field.Store.NO));
d.add(new StringField("value", "04", Field.Store.NO));
d.add(new StringField("value", "05", Field.Store.NO));
d.add(new StringField("value", "06", Field.Store.NO));
addField(d, "_id", "4");
addField(d, "value", "04");
addField(d, "value", "05");
addField(d, "value", "06");
writer.addDocument(d);

d = new Document();
d.add(new StringField("_id", "5", Field.Store.NO));
d.add(new StringField("value", "06", Field.Store.NO));
d.add(new StringField("value", "07", Field.Store.NO));
d.add(new StringField("value", "08", Field.Store.NO));
addField(d, "_id", "5");
addField(d, "value", "06");
addField(d, "value", "07");
addField(d, "value", "08");
writer.addDocument(d);

d = new Document();
d.add(new StringField("_id", "6", Field.Store.NO));
writer.addDocument(d);

d = new Document();
d.add(new StringField("_id", "7", Field.Store.NO));
d.add(new StringField("value", "08", Field.Store.NO));
d.add(new StringField("value", "09", Field.Store.NO));
d.add(new StringField("value", "10", Field.Store.NO));
addField(d, "_id", "7");
addField(d, "value", "08");
addField(d, "value", "09");
addField(d, "value", "10");
writer.addDocument(d);
writer.commit();

d = new Document();
d.add(new StringField("_id", "8", Field.Store.NO));
d.add(new StringField("value", "!08", Field.Store.NO));
d.add(new StringField("value", "!09", Field.Store.NO));
d.add(new StringField("value", "!10", Field.Store.NO));
addField(d, "_id", "8");
addField(d, "value", "!08");
addField(d, "value", "!09");
addField(d, "value", "!10");
writer.addDocument(d);
}

Expand All @@ -231,9 +238,6 @@ public void testActualMissingValueReverse() throws IOException {

public void testActualMissingValue(boolean reverse) throws IOException {
// missing value is set to an actual value
Document d = new Document();
final StringField s = new StringField("value", "", Field.Store.YES);
d.add(s);
final String[] values = new String[randomIntBetween(2, 30)];
for (int i = 1; i < values.length; ++i) {
values[i] = TestUtil.randomUnicodeString(getRandom());
Expand All @@ -244,7 +248,8 @@ public void testActualMissingValue(boolean reverse) throws IOException {
if (value == null) {
writer.addDocument(new Document());
} else {
s.setStringValue(value);
Document d = new Document();
addField(d, "value", value);
writer.addDocument(d);
}
if (randomInt(10) == 0) {
Expand Down Expand Up @@ -289,9 +294,6 @@ public void testSortMissingLastReverse() throws IOException {
}

public void testSortMissing(boolean first, boolean reverse) throws IOException {
Document d = new Document();
final StringField s = new StringField("value", "", Field.Store.YES);
d.add(s);
final String[] values = new String[randomIntBetween(2, 10)];
for (int i = 1; i < values.length; ++i) {
values[i] = TestUtil.randomUnicodeString(getRandom());
Expand All @@ -302,7 +304,8 @@ public void testSortMissing(boolean first, boolean reverse) throws IOException {
if (value == null) {
writer.addDocument(new Document());
} else {
s.setStringValue(value);
Document d = new Document();
addField(d, "value", value);
writer.addDocument(d);
}
if (randomInt(10) == 0) {
Expand Down Expand Up @@ -359,15 +362,15 @@ public void testNestedSorting(MultiValueMode sortMode) throws IOException {
final int numValues = randomInt(3);
for (int k = 0; k < numValues; ++k) {
final String value = RandomPicks.randomFrom(getRandom(), values);
child.add(new StringField("text", value, Store.YES));
addField(child, "text", value);
}
docs.add(child);
}
final Document parent = new Document();
parent.add(new StringField("type", "parent", Store.YES));
final String value = RandomPicks.randomFrom(getRandom(), values);
if (value != null) {
parent.add(new StringField("text", value, Store.YES));
addField(parent, "text", value);
}
docs.add(parent);
int bit = parents.prevSetBit(parents.length() - 1) + docs.size();
Expand Down Expand Up @@ -446,6 +449,19 @@ public void testNestedSorting(MultiValueMode sortMode) throws IOException {
searcher.getIndexReader().close();
}

private void assertIteratorConsistentWithRandomAccess(RandomAccessOrds ords, int maxDoc) {
for (int doc = 0; doc < maxDoc; ++doc) {
ords.setDocument(doc);
final int cardinality = ords.cardinality();
for (int i = 0; i < cardinality; ++i) {
assertEquals(ords.nextOrd(), ords.ordAt(i));
}
for (int i = 0; i < 3; ++i) {
assertEquals(ords.nextOrd(), -1);
}
}
}

@Test
public void testGlobalOrdinals() throws Exception {
fillExtendedMvSet();
Expand All @@ -457,8 +473,10 @@ public void testGlobalOrdinals() throws Exception {

// First segment
assertThat(globalOrdinals, instanceOf(GlobalOrdinalsIndexFieldData.class));
AtomicOrdinalsFieldData afd = globalOrdinals.load(topLevelReader.leaves().get(0));
LeafReaderContext leaf = topLevelReader.leaves().get(0);
AtomicOrdinalsFieldData afd = globalOrdinals.load(leaf);
RandomAccessOrds values = afd.getOrdinalsValues();
assertIteratorConsistentWithRandomAccess(values, leaf.reader().maxDoc());
values.setDocument(0);
assertThat(values.cardinality(), equalTo(2));
long ord = values.nextOrd();
Expand All @@ -476,8 +494,10 @@ public void testGlobalOrdinals() throws Exception {
assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("03"));

// Second segment
afd = globalOrdinals.load(topLevelReader.leaves().get(1));
leaf = topLevelReader.leaves().get(1);
afd = globalOrdinals.load(leaf);
values = afd.getOrdinalsValues();
assertIteratorConsistentWithRandomAccess(values, leaf.reader().maxDoc());
values.setDocument(0);
assertThat(values.cardinality(), equalTo(3));
ord = values.nextOrd();
Expand Down Expand Up @@ -515,8 +535,10 @@ public void testGlobalOrdinals() throws Exception {
assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("10"));

// Third segment
afd = globalOrdinals.load(topLevelReader.leaves().get(2));
leaf = topLevelReader.leaves().get(2);
afd = globalOrdinals.load(leaf);
values = afd.getOrdinalsValues();
assertIteratorConsistentWithRandomAccess(values, leaf.reader().maxDoc());
values.setDocument(0);
values.setDocument(0);
assertThat(values.cardinality(), equalTo(3));
Expand Down