Skip to content

Commit

Permalink
Reduce number of ES requests for IndexSetsResource#globalStats()
Browse files Browse the repository at this point in the history 
Instead of polling index statistics for each index in an index set,
we now only use a single request to get the required information.
  • Loading branch information
Jochen Schalanda committed Oct 10, 2017
1 parent bfa248a commit 8a341f9
Show file tree
Hide file tree
Showing 3 changed files with 110 additions and 15 deletions.
25 changes: 13 additions & 12 deletions ...g2-server/src/main/java/org/graylog2/rest/resources/system/indexer/IndexSetsResource.java
View file
Expand Up @@ -34,7 +34,9 @@
import org.graylog2.indexer.indexset.DefaultIndexSetConfig;
import org.graylog2.indexer.indexset.IndexSetConfig;
import org.graylog2.indexer.indexset.IndexSetService;
import org.graylog2.indexer.indices.Indices;
import org.graylog2.indexer.indices.jobs.IndexSetCleanupJob;
import org.graylog2.indexer.indices.stats.IndexStatistics;
import org.graylog2.plugin.cluster.ClusterConfigService;
import org.graylog2.rest.resources.system.indexer.requests.IndexSetUpdateRequest;
import org.graylog2.rest.resources.system.indexer.responses.IndexSetResponse;
Expand Down Expand Up @@ -65,11 +67,13 @@
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.StringJoiner;
import java.util.stream.Collectors;

import static java.util.Objects.requireNonNull;
Expand All @@ -81,6 +85,7 @@
public class IndexSetsResource extends RestResource {
private static final Logger LOG = LoggerFactory.getLogger(IndexSetsResource.class);

private final Indices indices;
private final IndexSetService indexSetService;
private final IndexSetRegistry indexSetRegistry;
private final IndexSetValidator indexSetValidator;
Expand All @@ -90,13 +95,15 @@ public class IndexSetsResource extends RestResource {
private final SystemJobManager systemJobManager;

@Inject
public IndexSetsResource(final IndexSetService indexSetService,
public IndexSetsResource(final Indices indices,
final IndexSetService indexSetService,
final IndexSetRegistry indexSetRegistry,
final IndexSetValidator indexSetValidator,
final IndexSetCleanupJob.Factory indexSetCleanupJobFactory,
final IndexSetStatsCreator indexSetStatsCreator,
final ClusterConfigService clusterConfigService,
final SystemJobManager systemJobManager) {
this.indices = requireNonNull(indices);
this.indexSetService = requireNonNull(indexSetService);
this.indexSetRegistry = indexSetRegistry;
this.indexSetValidator = indexSetValidator;
Expand Down Expand Up @@ -163,17 +170,11 @@ public IndexSetResponse list(@ApiParam(name = "skip", value = "The number of ele
public IndexSetStats globalStats() {
checkPermission(RestPermissions.INDEXSETS_READ);

long indices = 0;
long documents = 0;
long size = 0;
for (IndexSetStats stats : indexSetRegistry.getAll().stream()
.collect(Collectors.toMap(indexSet -> indexSet.getConfig().id(), indexSetStatsCreator::getForIndexSet)).values()) {
indices += stats.indices();
documents += stats.documents();
size += stats.size();
}

return IndexSetStats.create(indices, documents, size);
final Set<String> indexWildcards = indexSetRegistry.getAll().stream()
.map(IndexSet::getIndexWildcard)
.collect(Collectors.toSet());
final Set<IndexStatistics> indicesStats = indices.getIndicesStats(indexWildcards);
return IndexSetStats.fromIndexStatistics(indicesStats);
}

@GET
Expand Down
18 changes: 18 additions & 0 deletions ...ver/src/main/java/org/graylog2/rest/resources/system/indexer/responses/IndexSetStats.java
View file
Expand Up @@ -20,6 +20,10 @@
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.auto.value.AutoValue;
import org.graylog2.indexer.indices.stats.IndexStatistics;
import org.graylog2.rest.models.system.indexer.responses.IndexStats;

import java.util.Collection;

@JsonAutoDetect
@AutoValue
Expand All @@ -43,4 +47,18 @@ public static IndexSetStats create(@JsonProperty(FIELD_INDICES) long indices,
@JsonProperty(FIELD_SIZE) long size) {
return new AutoValue_IndexSetStats(indices, documents, size);
}

public static IndexSetStats fromIndexStatistics(Collection<IndexStatistics> indexStatistics) {
final long totalIndicesCount = indexStatistics.size();
final long totalDocumentsCount = indexStatistics.stream()
.map(IndexStatistics::allShards)
.map(IndexStats::documents)
.mapToLong(IndexStats.DocsStats::count)
.sum();
final long totalSizeInBytes = indexStatistics.stream()
.map(IndexStatistics::allShards)
.mapToLong(IndexStats::storeSizeBytes)
.sum();
return create(totalIndicesCount, totalDocumentsCount, totalSizeInBytes);
}
}
82 changes: 79 additions & 3 deletions ...erver/src/test/java/org/graylog2/rest/resources/system/indexer/IndexSetsResourceTest.java
View file
Expand Up @@ -21,15 +21,19 @@
import org.graylog2.indexer.IndexSetRegistry;
import org.graylog2.indexer.IndexSetStatsCreator;
import org.graylog2.indexer.IndexSetValidator;
import org.graylog2.indexer.TestIndexSet;
import org.graylog2.indexer.indexset.DefaultIndexSetConfig;
import org.graylog2.indexer.indexset.IndexSetConfig;
import org.graylog2.indexer.indexset.IndexSetService;
import org.graylog2.indexer.indices.Indices;
import org.graylog2.indexer.indices.jobs.IndexSetCleanupJob;
import org.graylog2.indexer.indices.stats.IndexStatistics;
import org.graylog2.indexer.retention.strategies.NoopRetentionStrategy;
import org.graylog2.indexer.retention.strategies.NoopRetentionStrategyConfig;
import org.graylog2.indexer.rotation.strategies.MessageCountRotationStrategy;
import org.graylog2.indexer.rotation.strategies.MessageCountRotationStrategyConfig;
import org.graylog2.plugin.cluster.ClusterConfigService;
import org.graylog2.rest.models.system.indexer.responses.IndexStats;
import org.graylog2.rest.resources.system.indexer.requests.IndexSetUpdateRequest;
import org.graylog2.rest.resources.system.indexer.responses.IndexSetResponse;
import org.graylog2.rest.resources.system.indexer.responses.IndexSetStats;
Expand Down Expand Up @@ -58,6 +62,7 @@

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyCollection;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
Expand All @@ -73,6 +78,8 @@ public class IndexSetsResourceTest {
@Rule
public final ExpectedException expectedException = ExpectedException.none();

@Mock
private Indices indices;
@Mock
private IndexSetService indexSetService;
@Mock
Expand All @@ -99,7 +106,7 @@ public IndexSetsResourceTest() {
@Before
public void setUp() throws Exception {
this.permitted = true;
this.indexSetsResource = new TestResource(indexSetService, indexSetRegistry, indexSetValidator, indexSetCleanupJobFactory, indexSetStatsCreator, clusterConfigService, systemJobManager, () -> permitted);
this.indexSetsResource = new TestResource(indices, indexSetService, indexSetRegistry, indexSetValidator, indexSetCleanupJobFactory, indexSetStatsCreator, clusterConfigService, systemJobManager, () -> permitted);
}

private void notPermitted() {
Expand Down Expand Up @@ -539,6 +546,75 @@ public void deleteDenied() {
}
}

@Test
public void globalStats() throws Exception {
final IndexStatistics indexStatistics = IndexStatistics.create(
"prefix_0",
IndexStats.create(
IndexStats.TimeAndTotalStats.create(0L, 0L),
IndexStats.TimeAndTotalStats.create(0L, 0L),
IndexStats.TimeAndTotalStats.create(0L, 0L),
IndexStats.TimeAndTotalStats.create(0L, 0L),
IndexStats.TimeAndTotalStats.create(0L, 0L),
IndexStats.TimeAndTotalStats.create(0L, 0L),
IndexStats.TimeAndTotalStats.create(0L, 0L),
0L,
23L,
2L,
IndexStats.DocsStats.create(42L, 0L)
),
IndexStats.create(
IndexStats.TimeAndTotalStats.create(0L, 0L),
IndexStats.TimeAndTotalStats.create(0L, 0L),
IndexStats.TimeAndTotalStats.create(0L, 0L),
IndexStats.TimeAndTotalStats.create(0L, 0L),
IndexStats.TimeAndTotalStats.create(0L, 0L),
IndexStats.TimeAndTotalStats.create(0L, 0L),
IndexStats.TimeAndTotalStats.create(0L, 0L),
0L,
23L,
2L,
IndexStats.DocsStats.create(42L, 0L)
),
Collections.emptyList()
);
when(indices.getIndicesStats(anyCollection())).thenReturn(Collections.singleton(indexStatistics));

final IndexSetStats indexSetStats = indexSetsResource.globalStats();

assertThat(indexSetStats).isNotNull();
assertThat(indexSetStats.indices()).isEqualTo(1L);
assertThat(indexSetStats.documents()).isEqualTo(42L);
assertThat(indexSetStats.size()).isEqualTo(23L);
}

@Test
public void globalStats0() throws Exception {
when(indexSetRegistry.getAll()).thenReturn(Collections.emptySet());
when(indices.getIndicesStats(anyCollection())).thenReturn(Collections.emptySet());

final IndexSetStats indexSetStats = indexSetsResource.globalStats();

assertThat(indexSetStats).isNotNull();
assertThat(indexSetStats.indices()).isEqualTo(0L);
assertThat(indexSetStats.documents()).isEqualTo(0L);
assertThat(indexSetStats.size()).isEqualTo(0L);
}

@Test
public void globalStatsDenied() {
notPermitted();

expectedException.expect(ForbiddenException.class);
expectedException.expectMessage("Not authorized");

try {
indexSetsResource.globalStats();
} finally {
verifyZeroInteractions(indexSetService);
}
}

@Test
public void setDefaultMakesIndexDefaultIfWritable() throws Exception {
final String indexSetId = "newDefaultIndexSetId";
Expand Down Expand Up @@ -645,8 +721,8 @@ public void setDefaultDoesNotDoAnythingIfIndexSetIsNotWritable() throws Exceptio
private static class TestResource extends IndexSetsResource {
private final Provider<Boolean> permitted;

TestResource(IndexSetService indexSetService, IndexSetRegistry indexSetRegistry, IndexSetValidator indexSetValidator, IndexSetCleanupJob.Factory indexSetCleanupJobFactory, IndexSetStatsCreator indexSetStatsCreator, ClusterConfigService clusterConfigService, SystemJobManager systemJobManager, Provider<Boolean> permitted) {
super(indexSetService, indexSetRegistry, indexSetValidator, indexSetCleanupJobFactory, indexSetStatsCreator, clusterConfigService, systemJobManager);
TestResource(Indices indices, IndexSetService indexSetService, IndexSetRegistry indexSetRegistry, IndexSetValidator indexSetValidator, IndexSetCleanupJob.Factory indexSetCleanupJobFactory, IndexSetStatsCreator indexSetStatsCreator, ClusterConfigService clusterConfigService, SystemJobManager systemJobManager, Provider<Boolean> permitted) {
super(indices, indexSetService, indexSetRegistry, indexSetValidator, indexSetCleanupJobFactory, indexSetStatsCreator, clusterConfigService, systemJobManager);
this.permitted = permitted;
}

Expand Down

0 comments on commit 8a341f9

Please sign in to comment.