Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Audit event cleanup #2746

Merged
merged 3 commits into from
Aug 31, 2016
Merged

Audit event cleanup #2746

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
30cfca9
Replace es_index_retention:initiate/complete events
bernd Aug 25, 2016
c85f6c1
Only emit es_index_rotation:complete audit event
bernd Aug 25, 2016
7431be2
Rename formatString to messageTemplate in FormattedAuditEvent
bernd Aug 31, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions graylog2-server/src/main/java/org/graylog2/audit/AuditEventTypes.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,10 @@ public class AuditEventTypes implements PluginAuditEventTypes {
public static final String ES_INDEX_RANGE_CREATE = PREFIX + "es_index_range:create";
public static final String ES_INDEX_RANGE_DELETE = PREFIX + "es_index_range:delete";
public static final String ES_INDEX_RANGE_UPDATE_JOB = PREFIX + "es_index_range_update_job:start";
public static final String ES_INDEX_RETENTION_COMPLETE = PREFIX + "es_index_retention:complete";
public static final String ES_INDEX_RETENTION_INITIATE = PREFIX + "es_index_retention:initiate";
public static final String ES_INDEX_RETENTION_CLOSE = PREFIX + "es_index_retention:close";
public static final String ES_INDEX_RETENTION_DELETE = PREFIX + "es_index_retention:delete";
public static final String ES_INDEX_RETENTION_STRATEGY_UPDATE = PREFIX + "es_index_retention_strategy:update";
public static final String ES_INDEX_ROTATION_COMPLETE = PREFIX + "es_index_rotation:complete";
public static final String ES_INDEX_ROTATION_INITIATE = PREFIX + "es_index_rotation:initiate";
public static final String ES_INDEX_ROTATION_STRATEGY_UPDATE = PREFIX + "es_index_rotation_strategy:update";
public static final String ES_WRITE_INDEX_UPDATE = PREFIX + "es_write_index:update";
public static final String ES_WRITE_INDEX_UPDATE_JOB_START = PREFIX + "es_write_index_update_job:start";
Expand Down Expand Up @@ -175,11 +174,10 @@ public class AuditEventTypes implements PluginAuditEventTypes {
.add(ES_INDEX_RANGE_CREATE)
.add(ES_INDEX_RANGE_DELETE)
.add(ES_INDEX_RANGE_UPDATE_JOB)
.add(ES_INDEX_RETENTION_COMPLETE)
.add(ES_INDEX_RETENTION_INITIATE)
.add(ES_INDEX_RETENTION_CLOSE)
.add(ES_INDEX_RETENTION_DELETE)
.add(ES_INDEX_RETENTION_STRATEGY_UPDATE)
.add(ES_INDEX_ROTATION_COMPLETE)
.add(ES_INDEX_ROTATION_INITIATE)
.add(ES_INDEX_ROTATION_STRATEGY_UPDATE)
.add(ES_WRITE_INDEX_UPDATE)
.add(ES_WRITE_INDEX_UPDATE_JOB_START)
Expand Down
6 changes: 3 additions & 3 deletions graylog2-server/src/main/java/org/graylog2/audit/formatter/FormattedAuditEvent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ public interface FormattedAuditEvent {
String action();

/**
* The format string that will be used to present the audit event to humans.
* The message template string that will be used to present the audit event to humans.
*
* All data in {@link #attributes()} as well as the following fields can be used as variables.
*
Expand All @@ -92,12 +92,12 @@ public interface FormattedAuditEvent {
*
* @return
*/
String formatString();
String messageTemplate();

/**
* The audit event attributes that will be stored in the database.
*
* All information that is needed by the {@link #formatString()} should be in here.
* All information that is needed by the {@link #messageTemplate()} should be in here.
*
* Make sure you do not store any sensitive information like passwords and API tokens!
*
Expand Down
22 changes: 2 additions & 20 deletions ...a/org/graylog2/indexer/retention/strategies/AbstractIndexCountBasedRetentionStrategy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,11 @@

package org.graylog2.indexer.retention.strategies;

import com.google.common.collect.ImmutableMap;
import org.graylog2.audit.AuditActor;
import org.graylog2.audit.AuditEventSender;
import org.graylog2.indexer.Deflector;
import org.graylog2.indexer.IndexHelper;
import org.graylog2.indexer.indices.Indices;
import org.graylog2.periodical.IndexRetentionThread;
import org.graylog2.plugin.indexer.retention.RetentionStrategy;
import org.graylog2.plugin.system.NodeId;
import org.graylog2.shared.system.activities.Activity;
import org.graylog2.shared.system.activities.ActivityWriter;
import org.slf4j.Logger;
Expand All @@ -36,25 +32,19 @@
import java.util.Set;

import static java.util.Objects.requireNonNull;
import static org.graylog2.audit.AuditEventTypes.ES_INDEX_RETENTION_COMPLETE;
import static org.graylog2.audit.AuditEventTypes.ES_INDEX_RETENTION_INITIATE;

public abstract class AbstractIndexCountBasedRetentionStrategy implements RetentionStrategy {
private static final Logger LOG = LoggerFactory.getLogger(AbstractIndexCountBasedRetentionStrategy.class);

private final Deflector deflector;
private final Indices indices;
private final NodeId nodeId;
private final ActivityWriter activityWriter;
private final AuditEventSender auditEventSender;

public AbstractIndexCountBasedRetentionStrategy(Deflector deflector, Indices indices, NodeId nodeId,
ActivityWriter activityWriter, AuditEventSender auditEventSender) {
public AbstractIndexCountBasedRetentionStrategy(Deflector deflector, Indices indices,
ActivityWriter activityWriter) {
this.deflector = requireNonNull(deflector);
this.indices = requireNonNull(indices);
this.nodeId = nodeId;
this.activityWriter = requireNonNull(activityWriter);
this.auditEventSender = requireNonNull(auditEventSender);
}

protected abstract Optional<Integer> getMaxNumberOfIndices();
Expand Down Expand Up @@ -85,9 +75,6 @@ public void retain() {
LOG.info(msg);
activityWriter.write(new Activity(msg, IndexRetentionThread.class));

final ImmutableMap<String, Object> auditEventContext = ImmutableMap.of("retention_strategy", this.getClass().getCanonicalName());
auditEventSender.success(AuditActor.system(nodeId), ES_INDEX_RETENTION_INITIATE, auditEventContext);

runRetention(deflectorIndices, removeCount);
}

Expand Down Expand Up @@ -115,11 +102,6 @@ private void runRetention(Map<String, Set<String>> deflectorIndices, int removeC

// Sorry if this should ever go mad. Run retention strategy!
retain(indexName);

final ImmutableMap<String, Object> auditEventContext = ImmutableMap.of(
"index_name", indexName,
"retention_strategy", strategyName);
auditEventSender.success(AuditActor.system(nodeId), ES_INDEX_RETENTION_COMPLETE, auditEventContext);
}
}
}
14 changes: 13 additions & 1 deletion ...ver/src/main/java/org/graylog2/indexer/retention/strategies/ClosingRetentionStrategy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
package org.graylog2.indexer.retention.strategies;

import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableMap;
import org.graylog2.audit.AuditActor;
import org.graylog2.audit.AuditEventSender;
import org.graylog2.indexer.Deflector;
import org.graylog2.indexer.indices.Indices;
Expand All @@ -31,11 +33,15 @@
import java.util.Optional;
import java.util.concurrent.TimeUnit;

import static org.graylog2.audit.AuditEventTypes.ES_INDEX_RETENTION_CLOSE;

public class ClosingRetentionStrategy extends AbstractIndexCountBasedRetentionStrategy {
private static final Logger LOG = LoggerFactory.getLogger(ClosingRetentionStrategy.class);

private final Indices indices;
private final NodeId nodeId;
private final ClusterConfigService clusterConfigService;
private final AuditEventSender auditEventSender;

@Inject
public ClosingRetentionStrategy(Deflector deflector,
Expand All @@ -44,9 +50,11 @@ public ClosingRetentionStrategy(Deflector deflector,
NodeId nodeId,
ClusterConfigService clusterConfigService,
AuditEventSender auditEventSender) {
super(deflector, indices, nodeId, activityWriter, auditEventSender);
super(deflector, indices, activityWriter);
this.indices = indices;
this.nodeId = nodeId;
this.clusterConfigService = clusterConfigService;
this.auditEventSender = auditEventSender;
}

@Override
Expand All @@ -65,6 +73,10 @@ public void retain(String indexName) {
final Stopwatch sw = Stopwatch.createStarted();

indices.close(indexName);
auditEventSender.success(AuditActor.system(nodeId), ES_INDEX_RETENTION_CLOSE, ImmutableMap.of(
"index_name", indexName,
"retention_strategy", this.getClass().getCanonicalName()
));

LOG.info("Finished index retention strategy [close] for index <{}> in {}ms.", indexName,
sw.stop().elapsed(TimeUnit.MILLISECONDS));
Expand Down
14 changes: 13 additions & 1 deletion ...er/src/main/java/org/graylog2/indexer/retention/strategies/DeletionRetentionStrategy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
package org.graylog2.indexer.retention.strategies;

import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableMap;
import org.graylog2.audit.AuditActor;
import org.graylog2.audit.AuditEventSender;
import org.graylog2.indexer.Deflector;
import org.graylog2.indexer.indices.Indices;
Expand All @@ -31,11 +33,15 @@
import java.util.Optional;
import java.util.concurrent.TimeUnit;

import static org.graylog2.audit.AuditEventTypes.ES_INDEX_RETENTION_DELETE;

public class DeletionRetentionStrategy extends AbstractIndexCountBasedRetentionStrategy {
private static final Logger LOG = LoggerFactory.getLogger(DeletionRetentionStrategy.class);

private final Indices indices;
private final ClusterConfigService clusterConfigService;
private final NodeId nodeId;
private final AuditEventSender auditEventSender;

@Inject
public DeletionRetentionStrategy(Deflector deflector,
Expand All @@ -44,9 +50,11 @@ public DeletionRetentionStrategy(Deflector deflector,
ClusterConfigService clusterConfigService,
NodeId nodeId,
AuditEventSender auditEventSender) {
super(deflector, indices, nodeId, activityWriter, auditEventSender);
super(deflector, indices, activityWriter);
this.indices = indices;
this.clusterConfigService = clusterConfigService;
this.nodeId = nodeId;
this.auditEventSender = auditEventSender;
}

@Override
Expand All @@ -65,6 +73,10 @@ public void retain(String indexName) {
final Stopwatch sw = Stopwatch.createStarted();

indices.delete(indexName);
auditEventSender.success(AuditActor.system(nodeId), ES_INDEX_RETENTION_DELETE, ImmutableMap.of(
"index_name", indexName,
"retention_strategy", this.getClass().getCanonicalName()
));

LOG.info("Finished index retention strategy [delete] for index <{}> in {}ms.", indexName,
sw.stop().elapsed(TimeUnit.MILLISECONDS));
Expand Down
7 changes: 2 additions & 5 deletions ...server/src/main/java/org/graylog2/indexer/retention/strategies/NoopRetentionStrategy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,9 @@
*/
package org.graylog2.indexer.retention.strategies;

import org.graylog2.audit.AuditEventSender;
import org.graylog2.indexer.Deflector;
import org.graylog2.indexer.indices.Indices;
import org.graylog2.plugin.indexer.retention.RetentionStrategyConfig;
import org.graylog2.plugin.system.NodeId;
import org.graylog2.shared.system.activities.ActivityWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -32,9 +30,8 @@ public class NoopRetentionStrategy extends AbstractIndexCountBasedRetentionStrat
private static final Logger LOG = LoggerFactory.getLogger(NoopRetentionStrategy.class);

@Inject
public NoopRetentionStrategy(Deflector deflector, Indices indices, ActivityWriter activityWriter,
AuditEventSender auditEventSender, NodeId nodeId) {
super(deflector, indices, nodeId, activityWriter, auditEventSender);
public NoopRetentionStrategy(Deflector deflector, Indices indices, ActivityWriter activityWriter) {
super(deflector, indices, activityWriter);
}

@Override
Expand Down
15 changes: 4 additions & 11 deletions ...rver/src/main/java/org/graylog2/indexer/rotation/strategies/AbstractRotationStrategy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,9 @@
import org.slf4j.LoggerFactory;

import javax.annotation.Nullable;
import java.util.Map;

import static java.util.Objects.requireNonNull;
import static org.graylog2.audit.AuditEventTypes.ES_INDEX_ROTATION_COMPLETE;
import static org.graylog2.audit.AuditEventTypes.ES_INDEX_ROTATION_INITIATE;

public abstract class AbstractRotationStrategy implements RotationStrategy {
private static final Logger LOG = LoggerFactory.getLogger(AbstractRotationStrategy.class);
Expand Down Expand Up @@ -62,28 +60,23 @@ public void rotate() {
try {
indexName = deflector.getNewestTargetName();
} catch (NoTargetIndexException e) {
final ImmutableMap<String, Object> auditEventContext = ImmutableMap.of("rotation_strategy", strategyName);
auditEventSender.failure(AuditActor.system(nodeId), ES_INDEX_ROTATION_INITIATE, auditEventContext);

LOG.error("Could not find current deflector target. Aborting.", e);
return;
}

final Map<String, Object> auditEventContext = ImmutableMap.of(
"index_name", indexName,
"rotation_strategy", strategyName);
final Result rotate = shouldRotate(indexName);
if (rotate == null) {
LOG.error("Cannot perform rotation at this moment.");

auditEventSender.failure(AuditActor.system(nodeId), ES_INDEX_ROTATION_INITIATE, auditEventContext);
return;
}
LOG.debug("Rotation strategy result: {}", rotate.getDescription());
if (rotate.shouldRotate()) {
LOG.info("Deflector index <{}> should be rotated, Pointing deflector to new index now!", indexName);
deflector.cycle();
auditEventSender.success(AuditActor.system(nodeId), ES_INDEX_ROTATION_COMPLETE, auditEventContext);
auditEventSender.success(AuditActor.system(nodeId), ES_INDEX_ROTATION_COMPLETE, ImmutableMap.of(
"index_name", indexName,
"rotation_strategy", strategyName
));
} else {
LOG.debug("Deflector index <{}> should not be rotated. Not doing anything.", indexName);
}
Expand Down