Skip to content

Commit

Permalink
Fix point visitor in DiskUsage API (#84909) (#84916)
Browse files Browse the repository at this point in the history 
* Fix point visitor in DiskUsage API (#84909)

We should use numIndexDimensions instead of numDimensions
when comparing packedValues of PointValues.

Relates #84816

* Fix tests
  • Loading branch information
@dnhatn
dnhatn committed Mar 13, 2022
1 parent 83349fd commit 90025dd
Show file tree
Hide file tree
Showing 4 changed files with 150 additions and 6 deletions.
5 changes: 5 additions & 0 deletions docs/changelog/84909.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 84909
summary: Fix point visitor in `DiskUsage` API
area: Search
type: bug
issues: []
54 changes: 54 additions & 0 deletions ...rTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

package org.elasticsearch.action.admin.indices.diskusage;

import org.apache.lucene.geo.GeoTestUtil;
import org.apache.lucene.util.English;
import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.cluster.metadata.IndexMetadata;
Expand Down Expand Up @@ -98,6 +99,59 @@ public void testSimple() throws Exception {
assertMetadataFields(stats);
}

public void testGeoShape() throws Exception {
final XContentBuilder mapping = XContentFactory.jsonBuilder();
mapping.startObject();
{
mapping.startObject("_doc");
{
mapping.startObject("properties");
{
mapping.startObject("location");
mapping.field("type", "geo_shape");
mapping.endObject();
}
mapping.endObject();
}
mapping.endObject();
}
mapping.endObject();

final String index = "test-index";
client().admin()
.indices()
.prepareCreate(index)
.setMapping(mapping)
.setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 5)))
.get();

int numDocs = randomIntBetween(10, 100);
for (int i = 0; i < numDocs; i++) {
final XContentBuilder doc = XContentFactory.jsonBuilder()
.startObject()
.startObject("location")
.field("type", "point")
.field("coordinates", new double[] { GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude() })
.endObject()
.endObject();
client().prepareIndex(index).setId("id-" + i).setSource(doc).get();
}
AnalyzeIndexDiskUsageResponse resp = client().execute(
AnalyzeIndexDiskUsageAction.INSTANCE,
new AnalyzeIndexDiskUsageRequest(new String[] { index }, AnalyzeIndexDiskUsageRequest.DEFAULT_INDICES_OPTIONS, true)
).actionGet();

final IndexDiskUsageStats stats = resp.getStats().get(index);
logger.info("--> stats {}", stats);
assertNotNull(stats);
assertThat(stats.getIndexSizeInBytes(), greaterThan(100L));

final IndexDiskUsageStats.PerFieldDiskUsage locationField = stats.getFields().get("location");
assertThat(locationField.totalBytes(), greaterThan(0L));
assertThat(locationField.getPointsBytes(), greaterThan(0L));
assertMetadataFields(stats);
}

void assertMetadataFields(IndexDiskUsageStats stats) {
final IndexDiskUsageStats.PerFieldDiskUsage sourceField = stats.getFields().get("_source");
assertThat(sourceField.getInvertedIndexBytes(), equalTo(0L));
Expand Down
8 changes: 6 additions & 2 deletions ...rc/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -386,8 +386,12 @@ void analyzePoints(SegmentReader reader, IndexDiskUsageStats stats) throws IOExc
directory.resetBytesRead();
if (field.getPointDimensionCount() > 0) {
final PointValues values = pointsReader.getValues(field.name);
values.intersect(new PointsVisitor(values.getMinPackedValue(), values.getNumDimensions(), values.getBytesPerDimension()));
values.intersect(new PointsVisitor(values.getMaxPackedValue(), values.getNumDimensions(), values.getBytesPerDimension()));
values.intersect(
new PointsVisitor(values.getMinPackedValue(), values.getNumIndexDimensions(), values.getBytesPerDimension())
);
values.intersect(
new PointsVisitor(values.getMaxPackedValue(), values.getNumIndexDimensions(), values.getBytesPerDimension())
);
stats.addPoints(field.name, directory.getBytesRead());
}
}
Expand Down
89 changes: 85 additions & 4 deletions ...st/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,15 @@
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.LatLonShape;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.geo.GeoTestUtil;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
Expand All @@ -35,6 +38,7 @@
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReader;
Expand Down Expand Up @@ -170,7 +174,7 @@ public void testTermVectors() throws Exception {
}
}

public void testPoints() throws Exception {
public void testBinaryPoints() throws Exception {
try (Directory dir = newDirectory()) {
final CodecMode codec = randomFrom(CodecMode.values());
indexRandomly(dir, codec, between(100, 1000), doc -> {
Expand Down Expand Up @@ -207,6 +211,59 @@ public void testPoints() throws Exception {
}
}

public void testTriangle() throws Exception {
try (Directory dir = newDirectory()) {
final CodecMode codec = randomFrom(CodecMode.values());
indexRandomly(dir, codec, between(100, 1000), doc -> {
final double ratio = randomDouble();
if (ratio <= 0.25) {
addFieldsToDoc(
doc,
LatLonShape.createIndexableFields("triangle_1", GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude())
);
}
if (ratio <= 0.50) {
addFieldsToDoc(
doc,
LatLonShape.createIndexableFields("triangle_2", GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude())
);
}
addFieldsToDoc(
doc,
LatLonShape.createIndexableFields("triangle_3", GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude())
);
});
final IndexDiskUsageStats stats = IndexDiskUsageAnalyzer.analyze(testShardId(), lastCommit(dir), () -> {});
final IndexDiskUsageStats perField = collectPerFieldStats(dir);
logger.info("--> stats {} per field {}", stats, perField);
assertFieldStats("total", "points", stats.total().getPointsBytes(), perField.total().getPointsBytes(), 0.01, 2048);
assertFieldStats(
"triangle_1",
"points",
stats.getFields().get("triangle_1").getPointsBytes(),
stats.total().getPointsBytes() / 7,
0.01,
2048
);
assertFieldStats(
"triangle_2",
"triangle",
stats.getFields().get("triangle_2").getPointsBytes(),
stats.total().getPointsBytes() * 2 / 7,
0.01,
2048
);
assertFieldStats(
"triangle_3",
"triangle",
stats.getFields().get("triangle_3").getPointsBytes(),
stats.total().getPointsBytes() * 4 / 7,
0.01,
2048
);
}
}

public void testCompletionField() throws Exception {
IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
.setUseCompoundFile(false)
Expand Down Expand Up @@ -288,6 +345,12 @@ public void testMixedFields() throws Exception {
}
}

private static void addFieldsToDoc(Document doc, IndexableField[] fields) {
for (IndexableField field : fields) {
doc.add(field);
}
}

enum CodecMode {
BEST_SPEED {
@Override
Expand Down Expand Up @@ -366,10 +429,15 @@ static void addRandomStoredFields(Document doc, int numFields) {
}
}

static void addRandomPoints(Document doc) {
static void addRandomIntLongPoints(Document doc) {
final int numValues = random().nextInt(5);
for (int i = 0; i < numValues; i++) {
doc.add(new IntPoint("pt-" + randomIntBetween(1, 2), random().nextInt()));
if (randomBoolean()) {
doc.add(new IntPoint("int_point_" + randomIntBetween(1, 2), random().nextInt()));
}
if (randomBoolean()) {
doc.add(new LongPoint("long_point_" + randomIntBetween(1, 2), random().nextLong()));
}
}
}

Expand Down Expand Up @@ -398,7 +466,20 @@ static void addRandomFields(Document doc) {
addRandomPostings(doc);
}
if (randomBoolean()) {
addRandomPoints(doc);
addRandomIntLongPoints(doc);
}
if (randomBoolean()) {
final int numValues = random().nextInt(5);
for (int i = 0; i < numValues; i++) {
addFieldsToDoc(
doc,
LatLonShape.createIndexableFields(
"triangle_" + randomIntBetween(1, 2),
GeoTestUtil.nextLatitude(),
GeoTestUtil.nextLongitude()
)
);
}
}
if (randomBoolean()) {
addRandomStoredFields(doc, between(1, 3));
Expand Down

0 comments on commit 90025dd

Please sign in to comment.