Only reacalculate index set ranges in index set maintenance menu (#3252)

* Only reacalculate index set ranges in index set maintenance menu Instead of triggering a recalc job for all indices in all index sets, only trigger a index range recalc for the current index set. * Adapt UI texts to last changes
Graylog2 · Dec 27, 2016 · fee01fc · fee01fc
1 parent cbdec69
commit fee01fc
Show file tree

Hide file tree

Showing 8 changed files with 60 additions and 30 deletions.
diff --git a/...src/main/java/org/graylog2/indexer/indices/jobs/SetIndexReadOnlyAndCalculateRangeJob.java b/...src/main/java/org/graylog2/indexer/indices/jobs/SetIndexReadOnlyAndCalculateRangeJob.java
@@ -49,7 +49,7 @@ public SetIndexReadOnlyAndCalculateRangeJob(SetIndexReadOnlyJob.Factory setIndex
     public void execute() {
         final SystemJob setIndexReadOnlyJob = setIndexReadOnlyJobFactory.create(indexName);
         setIndexReadOnlyJob.execute();
-        final SystemJob createNewSingleIndexRangeJob = createNewSingleIndexRangeJobFactory.create(indexSetRegistry, indexName);
+        final SystemJob createNewSingleIndexRangeJob = createNewSingleIndexRangeJobFactory.create(indexSetRegistry.getAllIndexSets(), indexName);
         createNewSingleIndexRangeJob.execute();
 
     }

diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/ranges/CreateNewSingleIndexRangeJob.java b/graylog2-server/src/main/java/org/graylog2/indexer/ranges/CreateNewSingleIndexRangeJob.java
@@ -18,27 +18,29 @@
 
 import com.google.inject.assistedinject.Assisted;
 import com.google.inject.assistedinject.AssistedInject;
-import org.graylog2.indexer.IndexSetRegistry;
+import org.graylog2.indexer.IndexSet;
 import org.graylog2.shared.system.activities.ActivityWriter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Set;
+
 import static com.google.common.base.Preconditions.checkNotNull;
 
 public class CreateNewSingleIndexRangeJob extends RebuildIndexRangesJob {
     private static final Logger LOG = LoggerFactory.getLogger(CreateNewSingleIndexRangeJob.class);
     private final String indexName;
 
     public interface Factory {
-        CreateNewSingleIndexRangeJob create(IndexSetRegistry indexSetRegistry, String indexName);
+        CreateNewSingleIndexRangeJob create(Set<IndexSet> indexSets, String indexName);
     }
 
     @AssistedInject
-    public CreateNewSingleIndexRangeJob(@Assisted IndexSetRegistry indexSetRegistry,
+    public CreateNewSingleIndexRangeJob(@Assisted Set<IndexSet> indexSets,
                                         @Assisted String indexName,
                                         ActivityWriter activityWriter,
                                         IndexRangeService indexRangeService) {
-        super(indexSetRegistry, activityWriter, indexRangeService);
+        super(indexSets, activityWriter, indexRangeService);
         this.indexName = checkNotNull(indexName);
     }
 

diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/ranges/RebuildIndexRangesJob.java b/graylog2-server/src/main/java/org/graylog2/indexer/ranges/RebuildIndexRangesJob.java
@@ -24,7 +24,6 @@
 
 import org.graylog2.database.NotFoundException;
 import org.graylog2.indexer.IndexSet;
-import org.graylog2.indexer.IndexSetRegistry;
 import org.graylog2.indexer.indices.TooManyAliasesException;
 import org.graylog2.shared.system.activities.Activity;
 import org.graylog2.shared.system.activities.ActivityWriter;
@@ -37,7 +36,7 @@
 
 public class RebuildIndexRangesJob extends SystemJob {
     public interface Factory {
-        RebuildIndexRangesJob create(IndexSetRegistry indexSetRegistry);
+        RebuildIndexRangesJob create(Set<IndexSet> indexSets);
     }
 
     private static final Logger LOG = LoggerFactory.getLogger(RebuildIndexRangesJob.class);
@@ -47,15 +46,15 @@ public interface Factory {
     private volatile int indicesToCalculate = 0;
     private volatile int indicesCalculated = 0;
 
-    protected final IndexSetRegistry indexSetRegistry;
+    protected final Set<IndexSet> indexSets;
     private final ActivityWriter activityWriter;
     protected final IndexRangeService indexRangeService;
 
     @AssistedInject
-    public RebuildIndexRangesJob(@Assisted IndexSetRegistry indexSetRegistry,
+    public RebuildIndexRangesJob(@Assisted Set<IndexSet> indexSets,
                                  ActivityWriter activityWriter,
                                  IndexRangeService indexRangeService) {
-        this.indexSetRegistry = indexSetRegistry;
+        this.indexSets = indexSets;
         this.activityWriter = activityWriter;
         this.indexRangeService = indexRangeService;
     }
@@ -86,8 +85,7 @@ public void execute() {
 
         // for each index set we know about
         final ListMultimap<IndexSet, String> indexSets = MultimapBuilder.hashKeys().arrayListValues().build();
-        final Set<IndexSet> allIndexSets = indexSetRegistry.getAllIndexSets();
+        for (IndexSet indexSet : this.indexSets) {
-        for (IndexSet indexSet : allIndexSets) {
             final String[] managedIndicesNames = indexSet.getManagedIndicesNames();
             for (String name : managedIndicesNames) {
                 indexSets.put(indexSet, name);

diff --git a/...src/main/java/org/graylog2/migrations/V20161130141500_DefaultStreamRecalcIndexRanges.java b/...src/main/java/org/graylog2/migrations/V20161130141500_DefaultStreamRecalcIndexRanges.java
@@ -120,7 +120,7 @@ public void upgrade() {
                 continue;
             }
             LOG.info("Recalculating streams in index {}", indexName);
-            final CreateNewSingleIndexRangeJob createNewSingleIndexRangeJob = rebuildIndexRangeJobFactory.create(indexSetRegistry, indexName);
+            final CreateNewSingleIndexRangeJob createNewSingleIndexRangeJob = rebuildIndexRangeJobFactory.create(indexSetRegistry.getAllIndexSets(), indexName);
             createNewSingleIndexRangeJob.execute();
         }
 

diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/IndexRangesResource.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/IndexRangesResource.java
@@ -28,6 +28,7 @@
 import org.graylog2.audit.AuditEventTypes;
 import org.graylog2.audit.jersey.AuditEvent;
 import org.graylog2.database.NotFoundException;
+import org.graylog2.indexer.IndexSet;
 import org.graylog2.indexer.IndexSetRegistry;
 import org.graylog2.indexer.ranges.CreateNewSingleIndexRangeJob;
 import org.graylog2.indexer.ranges.IndexRange;
@@ -40,6 +41,7 @@
 import org.graylog2.system.jobs.SystemJob;
 import org.graylog2.system.jobs.SystemJobConcurrencyException;
 import org.graylog2.system.jobs.SystemJobManager;
+import org.hibernate.validator.constraints.NotBlank;
 import org.hibernate.validator.constraints.NotEmpty;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -54,7 +56,9 @@
 import javax.ws.rs.Produces;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
+import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 import java.util.SortedSet;
 
 @RequiresAuthentication
@@ -143,14 +147,29 @@ public IndexRangeSummary show(
     @Produces(MediaType.APPLICATION_JSON)
     @AuditEvent(type = AuditEventTypes.ES_INDEX_RANGE_UPDATE_JOB)
     public Response rebuild() {
-        final SystemJob rebuildJob = rebuildIndexRangesJobFactory.create(indexSetRegistry);
+        submitIndexRangesJob(indexSetRegistry.getAllIndexSets());
-        try {
+
-            this.systemJobManager.submit(rebuildJob);
+        return Response.accepted().build();
-        } catch (SystemJobConcurrencyException e) {
+    }
-            final String errorMsg = "Concurrency level of this job reached: " + e.getMessage();
+
-            LOG.error(errorMsg, e);
+    @POST
-            throw new ForbiddenException(errorMsg);
+    @Timed
-        }
+    @Path("/index_set/{indexSetId}/rebuild")
+    @RequiresPermissions(RestPermissions.INDEXRANGES_REBUILD)
+    @ApiOperation(value = "Rebuild/sync index range information for the given index set.",
+            notes = "This triggers a systemjob that scans every index in the given index set and stores meta information " +
+                    "about what indices contain messages in what timeranges. It atomically overwrites " +
+                    "already existing meta information.")
+    @ApiResponses(value = {
+            @ApiResponse(code = 202, message = "Rebuild/sync systemjob triggered.")
+    })
+    @Produces(MediaType.APPLICATION_JSON)
+    @AuditEvent(type = AuditEventTypes.ES_INDEX_RANGE_UPDATE_JOB)
+    public Response rebuildIndexSet(@ApiParam(name = "indexSetId") @PathParam("indexSetId") @NotBlank final String indexSetId) {
+        final IndexSet indexSet = indexSetRegistry.get(indexSetId)
+                .orElseThrow(() -> new javax.ws.rs.NotFoundException("Index set <" + indexSetId + "> not found!"));
+
+        submitIndexRangesJob(Collections.singleton(indexSet));
 
         return Response.accepted().build();
     }
@@ -175,7 +194,7 @@ public Response rebuildIndex(
         }
         checkPermission(RestPermissions.INDEXRANGES_REBUILD, index);
 
-        final SystemJob rebuildJob = singleIndexRangeJobFactory.create(indexSetRegistry, index);
+        final SystemJob rebuildJob = singleIndexRangeJobFactory.create(indexSetRegistry.getAllIndexSets(), index);
         try {
             this.systemJobManager.submit(rebuildJob);
         } catch (SystemJobConcurrencyException e) {
@@ -186,4 +205,16 @@ public Response rebuildIndex(
 
         return Response.accepted().build();
     }
+
+    private void submitIndexRangesJob(final Set<IndexSet> indexSets) {
+        final SystemJob rebuildJob = rebuildIndexRangesJobFactory.create(indexSets);
+        try {
+            this.systemJobManager.submit(rebuildJob);
+        } catch (SystemJobConcurrencyException e) {
+            final String errorMsg = "Concurrency level of this job reached: " + e.getMessage();
+            LOG.error(errorMsg, e);
+            throw new ForbiddenException(errorMsg);
+        }
+    }
+
 }
diff --git a/graylog2-web-interface/src/components/indices/IndicesMaintenanceDropdown.jsx b/graylog2-web-interface/src/components/indices/IndicesMaintenanceDropdown.jsx
@@ -15,9 +15,8 @@ const IndicesMaintenanceDropdown = React.createClass({
   },
 
   _onRecalculateIndexRange() {
-    if (window.confirm('This will recalculate index ranges for ALL index sets using a background system job. Do you want to proceed?')) {
+    if (window.confirm('This will recalculate index ranges for this index set using a background system job. Do you want to proceed?')) {
-      // TODO 2.2: Only rebuild index set here? It currently rebuilds all index ranges!
+      IndexRangesActions.recalculate(this.props.indexSetId);
-      IndexRangesActions.recalculate();
     }
   },
   _onCycleDeflector() {
@@ -35,7 +34,7 @@ const IndicesMaintenanceDropdown = React.createClass({
     return (
       <ButtonGroup>
         <DropdownButton bsStyle="info" title="Maintenance" id="indices-maintenance-actions" pullRight>
-          <MenuItem eventKey="1" onClick={this._onRecalculateIndexRange}>Recalculate all index ranges</MenuItem>
+          <MenuItem eventKey="1" onClick={this._onRecalculateIndexRange}>Recalculate index ranges</MenuItem>
           {cycleButton}
         </DropdownButton>
       </ButtonGroup>

diff --git a/graylog2-web-interface/src/routing/ApiRoutes.js b/graylog2-web-interface/src/routing/ApiRoutes.js
@@ -79,7 +79,7 @@ const ApiRoutes = {
   },
   IndexRangesApiController: {
     list: () => { return { url: '/system/indices/ranges' }; },
-    rebuild: () => { return { url: '/system/indices/ranges/rebuild' }; },
+    rebuild: (indexSetId) => { return { url: `/system/indices/ranges/index_set/${indexSetId}/rebuild` }; },
     rebuildSingle: (index) => { return { url: `/system/indices/ranges/${index}/rebuild` }; },
   },
   IndexSetsApiController: {

diff --git a/graylog2-web-interface/src/stores/indices/IndexRangesStore.js b/graylog2-web-interface/src/stores/indices/IndexRangesStore.js
@@ -28,9 +28,9 @@ const IndexRangesStore = Reflux.createStore({
 
     IndexRangesActions.list.promise(promise);
   },
-  recalculate() {
+  recalculate(indexSetId) {
-    const url = URLUtils.qualifyUrl(ApiRoutes.IndexRangesApiController.rebuild().url);
+    const url = URLUtils.qualifyUrl(ApiRoutes.IndexRangesApiController.rebuild(indexSetId).url);
-    const promise = fetch ('POST', url);
+    const promise = fetch('POST', url);
     promise
       .then(UserNotification.success('Index ranges will be recalculated shortly'))
       .catch((error) => {