Bundling/streamlining requests for indices page. (#2017)

* Add endpoints for indices overview and multiple index details.

This helps us to keep the required payload for updating the indices page
minimal, because we can get the required information to render the
initial page and the details for selected indices separately.

* Add new store/actions which make use of new index/indices overview.

* Allows components to (un-)/register for index details.

* Use new index subscriptions and index/indices overview in components.

* Add license headers.

* Fixing some linter hints.

* Fixing property names for IndexSizeSummary.

* Fixing typo, correcting action name.

* Fixed property name.

* Declaring closed indices as not being reopened.

Right now, closing an index does not clear its reopened flag (#2019).
Therefore we are filtering that out explicitly for closed indices, so no
double badges (which would be misleading) are displayed.

* Scoping indexer overview response before triggering it.

* IndexRange props are optional, as they do not exist for closed indices.

* Removing unused function.

* Waiting for index details to be fetched before rendering.

* Fetching indices metadata once for bulk requests.

This is prefetching the reopened status for existing indices in the
IndexerOverviewResource and the IndicesResource for bulk listings of
reopened indices.

* Removing unnecessary imports, reformatting code.

graylog2-server/src/main/java/org/graylog2/indexer/indices/Indices.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
@@ -59,6 +60,7 @@
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -84,11 +86,13 @@
 import javax.annotation.Nullable;
 import javax.inject.Inject;
 import javax.inject.Singleton;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 
@@ -179,6 +183,13 @@ public Map<String, IndexStats> getAll() {
         return response.getIndices();
     }
 
+    public Map<String, IndexStats> getAllDocCounts() {
+        final IndicesStatsRequest request = c.admin().indices().prepareStats(allIndicesAlias()).setDocs(true).request();
+        final IndicesStatsResponse response = c.admin().indices().stats(request).actionGet();
+
+        return response.getIndices();
+    }
+
     @Nullable
     public IndexStats indexStats(final String indexName) {
         final IndicesStatsRequest request = c.admin().indices().prepareStats(indexName).request();
@@ -340,6 +351,14 @@ public boolean isReopened(String indexName) {
         return checkForReopened(metaData);
     }
 
+    public Map<String, Boolean> areReopened(Collection<String> indices) {
+        final ClusterStateResponse clusterState = c.admin().cluster().prepareState().all().get();
+        final ImmutableOpenMap<String, IndexMetaData> indicesMetaData = clusterState.getState().getMetaData().getIndices();
+        return indices.stream().collect(
+            Collectors.toMap((index) -> index, (index) -> checkForReopened(indicesMetaData.get(index)))
+        );
+    }
+
     protected Boolean checkForReopened(IndexMetaData metaData) {
         return metaData.getSettings().getAsBoolean("index." + REOPENED_INDEX_SETTING, false);
     }

graylog2-server/src/main/java/org/graylog2/rest/models/system/indexer/requests/IndicesReadRequest.java

@@ -0,0 +1,36 @@
+/**
+ * This file is part of Graylog.
+ *
+ * Graylog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.graylog2.rest.models.system.indexer.requests;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.auto.value.AutoValue;
+
+import java.util.List;
+
+@AutoValue
+@JsonAutoDetect
+public abstract class IndicesReadRequest {
+    @JsonProperty("indices")
+    public abstract List<String> indices();
+
+    @JsonCreator
+    public static IndicesReadRequest create(@JsonProperty("indices") List<String> indices) {
+        return new AutoValue_IndicesReadRequest(indices);
+    }
+}

graylog2-server/src/main/java/org/graylog2/rest/models/system/indexer/responses/IndexSizeSummary.java

@@ -0,0 +1,42 @@
+/**
+ * This file is part of Graylog.
+ *
+ * Graylog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.graylog2.rest.models.system.indexer.responses;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.auto.value.AutoValue;
+
+@AutoValue
+@JsonAutoDetect
+public abstract class IndexSizeSummary {
+    @JsonProperty("events")
+    public abstract long events();
+
+    @JsonProperty("deleted")
+    public abstract long deleted();
+
+    @JsonProperty("bytes")
+    public abstract long bytes();
+
+    @JsonCreator
+    public static IndexSizeSummary create(@JsonProperty("events") long events,
+                                          @JsonProperty("deleted") long deleted,
+                                          @JsonProperty("bytes") long bytes) {
+        return new AutoValue_IndexSizeSummary(events, deleted, bytes);
+    }
+}

graylog2-server/src/main/java/org/graylog2/rest/models/system/indexer/responses/IndexSummary.java

@@ -0,0 +1,54 @@
+/**
+ * This file is part of Graylog.
+ *
+ * Graylog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.graylog2.rest.models.system.indexer.responses;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.auto.value.AutoValue;
+
+import javax.annotation.Nullable;
+
+@AutoValue
+@JsonAutoDetect
+public abstract class IndexSummary {
+    @JsonProperty("size")
+    @Nullable
+    public abstract IndexSizeSummary size();
+
+    @JsonProperty("range")
+    @Nullable
+    public abstract IndexRangeSummary range();
+
+    @JsonProperty("is_deflector")
+    public abstract boolean isDeflector();
+
+    @JsonProperty("is_closed")
+    public abstract boolean isClosed();
+
+    @JsonProperty("is_reopened")
+    public abstract boolean isReopened();
+
+    @JsonCreator
+    public static IndexSummary create(@JsonProperty("size") @Nullable IndexSizeSummary size,
+                                      @JsonProperty("range") @Nullable IndexRangeSummary range,
+                                      @JsonProperty("is_deflector") boolean isDeflector,
+                                      @JsonProperty("is_closed") boolean isClosed,
+                                      @JsonProperty("is_reopened") boolean isReopened) {
+        return new AutoValue_IndexSummary(size, range, isDeflector, isClosed, isReopened);
+    }
+}

graylog2-server/src/main/java/org/graylog2/rest/models/system/indexer/responses/IndexerClusterOverview.java

@@ -0,0 +1,38 @@
+/**
+ * This file is part of Graylog.
+ *
+ * Graylog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.graylog2.rest.models.system.indexer.responses;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.auto.value.AutoValue;
+
+@AutoValue
+@JsonAutoDetect
+public abstract class IndexerClusterOverview {
+    @JsonProperty("health")
+    public abstract ClusterHealth health();
+
+    @JsonProperty("name")
+    public abstract String name();
+
+    @JsonCreator
+    public static IndexerClusterOverview create(@JsonProperty("health") ClusterHealth health,
+                                                @JsonProperty("name") String name) {
+        return new AutoValue_IndexerClusterOverview(health, name);
+    }
+}

graylog2-server/src/main/java/org/graylog2/rest/models/system/indexer/responses/IndexerOverview.java

@@ -0,0 +1,51 @@
+/**
+ * This file is part of Graylog.
+ *
+ * Graylog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.graylog2.rest.models.system.indexer.responses;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.auto.value.AutoValue;
+import org.graylog2.rest.models.count.responses.MessageCountResponse;
+import org.graylog2.rest.models.system.deflector.responses.DeflectorSummary;
+
+import java.util.List;
+import java.util.Map;
+
+@AutoValue
+@JsonAutoDetect
+public abstract class IndexerOverview {
+    @JsonProperty("deflector")
+    public abstract DeflectorSummary deflectorSummary();
+
+    @JsonProperty("indexer_cluster")
+    public abstract IndexerClusterOverview indexerCluster();
+
+    @JsonProperty("counts")
+    public abstract MessageCountResponse messageCountResponse();
+
+    @JsonProperty("indices")
+    public abstract Map<String, IndexSummary> indices();
+
+    @JsonCreator
+    public static IndexerOverview create(@JsonProperty("deflector_summary") DeflectorSummary deflectorSummary,
+                                         @JsonProperty("indexer_cluster") IndexerClusterOverview indexerCluster,
+                                         @JsonProperty("counts") MessageCountResponse messageCountResponse,
+                                         @JsonProperty("indices") Map<String, IndexSummary> indices) {
+        return new AutoValue_IndexerOverview(deflectorSummary, indexerCluster, messageCountResponse, indices);
+    }
+}

graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/IndexerOverviewResource.java

@@ -0,0 +1,105 @@
+/**
+ * This file is part of Graylog.
+ *
+ * Graylog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.graylog2.rest.resources.system.indexer;
+
+import com.codahale.metrics.annotation.Timed;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.apache.shiro.authz.annotation.RequiresAuthentication;
+import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.graylog2.indexer.indices.Indices;
+import org.graylog2.rest.models.system.deflector.responses.DeflectorSummary;
+import org.graylog2.rest.models.system.indexer.responses.IndexRangeSummary;
+import org.graylog2.rest.models.system.indexer.responses.IndexSizeSummary;
+import org.graylog2.rest.models.system.indexer.responses.IndexSummary;
+import org.graylog2.rest.models.system.indexer.responses.IndexerClusterOverview;
+import org.graylog2.rest.models.system.indexer.responses.IndexerOverview;
+import org.graylog2.rest.resources.count.CountResource;
+import org.graylog2.rest.resources.system.DeflectorResource;
+import org.graylog2.rest.resources.system.IndexRangesResource;
+import org.graylog2.shared.rest.resources.RestResource;
+
+import javax.inject.Inject;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@RequiresAuthentication
+@Api(value = "Indexer/Overview", description = "Indexing overview")
+@Path("/system/indexer/overview")
+public class IndexerOverviewResource extends RestResource {
+    private final DeflectorResource deflectorResource;
+    private final IndexerClusterResource indexerClusterResource;
+    private final IndexRangesResource indexRangesResource;
+    private final CountResource countResource;
+    private final Indices indices;
+
+    @Inject
+    public IndexerOverviewResource(DeflectorResource deflectorResource,
+                                   IndexerClusterResource indexerClusterResource,
+                                   IndexRangesResource indexRangesResource,
+                                   CountResource countResource,
+                                   Indices indices) {
+        this.deflectorResource = deflectorResource;
+        this.indexerClusterResource = indexerClusterResource;
+        this.indexRangesResource = indexRangesResource;
+        this.countResource = countResource;
+        this.indices = indices;
+    }
+
+    @GET
+    @Timed
+    @ApiOperation(value = "Get overview of current indexing state, including deflector config, cluster state, index ranges & message counts.")
+    @Produces(MediaType.APPLICATION_JSON)
+    public IndexerOverview index() throws ClassNotFoundException {
+        final DeflectorSummary deflectorSummary = deflectorResource.deflector();
+        final List<IndexRangeSummary> indexRanges = indexRangesResource.list().ranges();
+        final Map<String, IndexStats> allDocCounts = indices.getAllDocCounts();
+        final Map<String, Boolean> areReopened = indices.areReopened(allDocCounts.keySet());
+        final Map<String, IndexSummary> indicesSummaries = allDocCounts.values()
+            .stream()
+            .parallel()
+            .collect(Collectors.toMap(IndexStats::getIndex,
+                (indexStats) -> IndexSummary.create(
+                    IndexSizeSummary.create(indexStats.getPrimaries().getDocs().getCount(),
+                        indexStats.getPrimaries().getDocs().getDeleted(),
+                        indexStats.getPrimaries().getStore().sizeInBytes()),
+                    indexRanges.stream().filter((indexRangeSummary) -> indexRangeSummary.indexName().equals(indexStats.getIndex())).findFirst().orElse(null),
+                    deflectorSummary.currentTarget().equals(indexStats.getIndex()),
+                    false,
+                    areReopened.get(indexStats.getIndex()))
+            ));
+
+        indices.getClosedIndices().stream().forEach((indexName) -> {
+            indicesSummaries.put(indexName, IndexSummary.create(
+                null,
+                indexRanges.stream().filter((indexRangeSummary) -> indexRangeSummary.indexName().equals(indexName)).findFirst().orElse(null),
+                deflectorSummary.currentTarget().equals(indexName),
+                true,
+                false
+            ));
+        });
+
+        return IndexerOverview.create(deflectorSummary,
+            IndexerClusterOverview.create(indexerClusterResource.clusterHealth(), indexerClusterResource.clusterName().name()),
+            countResource.total(),indicesSummaries);
+    }
+}