Audit event changes (#2718)

* Rename urn:graylog:server to urn:graylog:node

* Add `node_id` to audit event context and use version string

* Update AuditEventTypes and record some new events

* Remove unused extractor:import and extractor:export audit events

* Remove unused grok pattern export audit event

* Make grok pattern import a separate event

* Remove more unused events and update cluster resource

* Make audit event type prefix public

* Swap PREFIX and NAMESPACE constants in AuditEventTypes

* Update AuditCoverageTest

* Add NoAuditEvent for more cluster proxy resources

* Return saved search ID

* Remove unused session:update audit event

* Remove unused static_field:update

* Return systemJobId in system jobs resource

* Fix access token deletion in api browser

* Remove unused user_access_token:update audit event

graylog2-server/src/main/java/org/graylog2/audit/AuditActor.java

@@ -26,7 +26,7 @@
 
 @AutoValue
 public abstract class AuditActor {
-    private static final String URN_GRAYLOG_SERVER = "urn:graylog:server:";
+    private static final String URN_GRAYLOG_NODE = "urn:graylog:node:";
     private static final String URN_GRAYLOG_USER = "urn:graylog:user:";
 
     public abstract String urn();
@@ -39,6 +39,6 @@ public static AuditActor user(@Nonnull String username) {
     }
 
     public static AuditActor system(@Nonnull NodeId nodeId) {
-        return new AutoValue_AuditActor(URN_GRAYLOG_SERVER + requireNonNull(nodeId, "nodeId must not be null").toString());
+        return new AutoValue_AuditActor(URN_GRAYLOG_NODE + requireNonNull(nodeId, "nodeId must not be null").toString());
     }
 }

graylog2-server/src/main/java/org/graylog2/audit/formatter/FormattedAuditEvent.java

@@ -29,7 +29,7 @@ public interface FormattedAuditEvent {
      * Examples:
      *
      *    {@code urn:graylog:user:jane}
-     *    {@code urn:graylog:server:28164cbe-4ad9-4c9c-a76e-088655aa7889}
+     *    {@code urn:graylog:node:28164cbe-4ad9-4c9c-a76e-088655aa7889}
      *
      * @return the actor URN
      */

graylog2-server/src/main/java/org/graylog2/bootstrap/ServerBootstrap.java

@@ -98,8 +98,9 @@ protected void startCommand() {
         final AuditEventSender auditEventSender = injector.getInstance(AuditEventSender.class);
         final NodeId nodeId = injector.getInstance(NodeId.class);
         final Map<String, Object> auditEventContext = ImmutableMap.of(
-            "version", version,
-            "java", Tools.getSystemInformation()
+            "version", version.toString(),
+            "java", Tools.getSystemInformation(),
+            "node_id", nodeId.toString()
         );
         auditEventSender.success(AuditActor.system(nodeId), NODE_STARTUP_INITIATE, auditEventContext);
 

graylog2-server/src/main/java/org/graylog2/indexer/Deflector.java

@@ -17,14 +17,18 @@
 package org.graylog2.indexer;
 
 import com.google.common.collect.ComparisonChain;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSortedSet;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.indices.InvalidAliasNameException;
+import org.graylog2.audit.AuditActor;
+import org.graylog2.audit.AuditEventSender;
 import org.graylog2.indexer.indices.Indices;
 import org.graylog2.indexer.indices.TooManyAliasesException;
 import org.graylog2.indexer.indices.jobs.SetIndexReadOnlyAndCalculateRangeJob;
 import org.graylog2.indexer.ranges.IndexRange;
 import org.graylog2.indexer.ranges.IndexRangeService;
+import org.graylog2.plugin.system.NodeId;
 import org.graylog2.shared.system.activities.Activity;
 import org.graylog2.shared.system.activities.ActivityWriter;
 import org.graylog2.system.jobs.SystemJob;
@@ -46,6 +50,7 @@
 import java.util.stream.Collectors;
 
 import static com.google.common.base.Strings.isNullOrEmpty;
+import static org.graylog2.audit.AuditEventTypes.ES_WRITE_INDEX_UPDATE;
 
 /**
  * Format of actual indexes behind the Deflector:
@@ -70,6 +75,8 @@ public class Deflector { // extends Ablenkblech
     private final Indices indices;
     private final Pattern deflectorIndexPattern;
     private final Pattern indexPattern;
+    private final AuditEventSender auditEventSender;
+    private final NodeId nodeId;
     private final SetIndexReadOnlyAndCalculateRangeJob.Factory setIndexReadOnlyAndCalculateRangeJobFactory;
 
     @Inject
@@ -78,12 +85,16 @@ public Deflector(final SystemJobManager systemJobManager,
                      final ActivityWriter activityWriter,
                      final Indices indices,
                      final IndexRangeService indexRangeService,
+                     final AuditEventSender auditEventSender,
+                     final NodeId nodeId,
                      final SetIndexReadOnlyAndCalculateRangeJob.Factory setIndexReadOnlyAndCalculateRangeJobFactory) {
         this.indexPrefix = indexPrefix;
 
         this.systemJobManager = systemJobManager;
         this.activityWriter = activityWriter;
         this.indexRangeService = indexRangeService;
+        this.auditEventSender = auditEventSender;
+        this.nodeId = nodeId;
         this.setIndexReadOnlyAndCalculateRangeJobFactory = setIndexReadOnlyAndCalculateRangeJobFactory;
 
         this.deflectorName = buildName(indexPrefix);
@@ -183,6 +194,7 @@ public void cycle() {
         LOG.info("Done!");
 
         activityWriter.write(activity);
+        auditEventSender.success(AuditActor.system(nodeId), ES_WRITE_INDEX_UPDATE, ImmutableMap.of("indexName", newTarget));
     }
 
     private void addDeflectorIndexRange(String newTarget) {

graylog2-server/src/main/java/org/graylog2/indexer/indices/Indices.java

@@ -18,6 +18,7 @@
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
@@ -308,9 +309,9 @@ public boolean create(String indexName, int numShards, int numReplicas, Settings
 
         final boolean acknowledged = c.admin().indices().create(cir).actionGet().isAcknowledged();
         if (acknowledged) {
-            auditEventSender.success(AuditActor.system(nodeId), ES_INDEX_CREATE);
+            auditEventSender.success(AuditActor.system(nodeId), ES_INDEX_CREATE, ImmutableMap.of("indexName", indexName));
         } else {
-            auditEventSender.failure(AuditActor.system(nodeId), ES_INDEX_CREATE);
+            auditEventSender.failure(AuditActor.system(nodeId), ES_INDEX_CREATE, ImmutableMap.of("indexName", indexName));
         }
         return acknowledged;
     }

graylog2-server/src/main/java/org/graylog2/rest/resources/cluster/ClusterDeflectorResource.java

@@ -20,8 +20,7 @@
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
 import org.apache.shiro.authz.annotation.RequiresAuthentication;
-import org.graylog2.audit.AuditEventTypes;
-import org.graylog2.audit.jersey.AuditEvent;
+import org.graylog2.audit.jersey.NoAuditEvent;
 import org.graylog2.cluster.Node;
 import org.graylog2.cluster.NodeService;
 import org.graylog2.rest.RemoteInterfaceProvider;
@@ -57,7 +56,7 @@ public ClusterDeflectorResource(@Context HttpHeaders httpHeaders,
     @Timed
     @ApiOperation(value = "Finds master node and triggers deflector cycle")
     @Path("/cycle")
-    @AuditEvent(type = AuditEventTypes.ES_WRITE_INDEX_UPDATE)
+    @NoAuditEvent("this is a proxy resource, the event will be triggered on the individual nodes")
     public void cycle() throws IOException {
         final Optional<Node> master = nodeService.allActive().values().stream().filter(Node::isMaster).findFirst();
         if (!master.isPresent()) {

graylog2-server/src/main/java/org/graylog2/rest/resources/cluster/ClusterLoadBalancerStatusResource.java

@@ -23,8 +23,7 @@
 import io.swagger.annotations.ApiParam;
 import org.apache.shiro.authz.annotation.RequiresAuthentication;
 import org.apache.shiro.authz.annotation.RequiresPermissions;
-import org.graylog2.audit.AuditEventTypes;
-import org.graylog2.audit.jersey.AuditEvent;
+import org.graylog2.audit.jersey.NoAuditEvent;
 import org.graylog2.cluster.Node;
 import org.graylog2.cluster.NodeNotFoundException;
 import org.graylog2.cluster.NodeService;
@@ -70,7 +69,7 @@ public ClusterLoadBalancerStatusResource(NodeService nodeService,
     @ApiOperation(value = "Override load balancer status of this graylog-server node. Next lifecycle " +
             "change will override it again to its default. Set to ALIVE, DEAD, or THROTTLED.")
     @Path("/override/{status}")
-    @AuditEvent(type = AuditEventTypes.LOAD_BALANCER_STATUS_UPDATE)
+    @NoAuditEvent("this is a proxy resource, the audit event will be emitted on the target node")
     public void override(@ApiParam(name = "nodeId", value = "The id of the node whose LB status will be changed", required = true)
                          @PathParam("nodeId") String nodeId,
                          @ApiParam(name = "status") @PathParam("status") String status) throws IOException, NodeNotFoundException {

graylog2-server/src/main/java/org/graylog2/rest/resources/cluster/ClusterLoggersResource.java

@@ -23,8 +23,7 @@
 import io.swagger.annotations.ApiResponse;
 import io.swagger.annotations.ApiResponses;
 import org.apache.shiro.authz.annotation.RequiresAuthentication;
-import org.graylog2.audit.AuditEventTypes;
-import org.graylog2.audit.jersey.AuditEvent;
+import org.graylog2.audit.jersey.NoAuditEvent;
 import org.graylog2.cluster.Node;
 import org.graylog2.cluster.NodeNotFoundException;
 import org.graylog2.cluster.NodeService;
@@ -84,7 +83,7 @@ public Map<String, Optional<SubsystemSummary>> subsystems() {
     @ApiResponses(value = {
         @ApiResponse(code = 404, message = "No such subsystem.")
     })
-    @AuditEvent(type = AuditEventTypes.LOG_LEVEL_UPDATE)
+    @NoAuditEvent("proxy resource, audit event will be emitted on target nodes")
     public void setSubsystemLoggerLevel(
         @ApiParam(name = "nodeId", required = true) @PathParam("nodeId") @NotEmpty String nodeId,
         @ApiParam(name = "subsystem", required = true) @PathParam("subsystem") @NotEmpty String subsystemTitle,

graylog2-server/src/main/java/org/graylog2/rest/resources/cluster/ClusterSystemProcessingResource.java

@@ -22,8 +22,7 @@
 import io.swagger.annotations.ApiOperation;
 import io.swagger.annotations.ApiParam;
 import org.apache.shiro.authz.annotation.RequiresAuthentication;
-import org.graylog2.audit.AuditEventTypes;
-import org.graylog2.audit.jersey.AuditEvent;
+import org.graylog2.audit.jersey.NoAuditEvent;
 import org.graylog2.cluster.Node;
 import org.graylog2.cluster.NodeNotFoundException;
 import org.graylog2.cluster.NodeService;
@@ -75,7 +74,7 @@ private RemoteSystemProcessingResource getRemoteSystemProcessingResource(String
             notes = "If the message journal is enabled, incoming messages will be spooled on disk, if it is disabled, " +
                     "you might lose messages from inputs which cannot buffer themselves, like AMQP or Kafka-based inputs.")
     @Path("pause")
-    @AuditEvent(type = AuditEventTypes.MESSAGE_PROCESSING_STOP)
+    @NoAuditEvent("proxy resource, audit event will be emitted on target node")
     public void pause(@ApiParam(name = "nodeId", value = "The id of the node where processing will be paused.", required = true)
                       @PathParam("nodeId") String nodeId) throws IOException, NodeNotFoundException {
         final Response response = this.getRemoteSystemProcessingResource(nodeId).pause().execute();
@@ -89,7 +88,7 @@ public void pause(@ApiParam(name = "nodeId", value = "The id of the node where p
     @Timed
     @ApiOperation(value = "Resume message processing on node")
     @Path("resume")
-    @AuditEvent(type = AuditEventTypes.MESSAGE_PROCESSING_START)
+    @NoAuditEvent("proxy resource, audit event will be emitted on target node")
     public void resume(@ApiParam(name = "nodeId", value = "The id of the node where processing will be resumed.", required = true)
                        @PathParam("nodeId") String nodeId) throws IOException, NodeNotFoundException {
         final Response response = this.getRemoteSystemProcessingResource(nodeId).resume().execute();

graylog2-server/src/main/java/org/graylog2/rest/resources/search/SavedSearchesResource.java

@@ -93,7 +93,7 @@ public Response create(@ApiParam(name = "JSON body", required = true)
                 .path("{searchId}")
                 .build(id);
 
-        return Response.created(searchUri).build();
+        return Response.created(searchUri).entity(ImmutableMap.of("search_id", id)).build();
     }
 
     @GET

graylog2-server/src/main/java/org/graylog2/rest/resources/system/DeflectorResource.java

@@ -87,7 +87,7 @@ public DeflectorSummary deflector() throws TooManyAliasesException {
     @RequiresPermissions(RestPermissions.DEFLECTOR_CYCLE)
     @Path("/cycle")
     @RestrictToMaster
-    @AuditEvent(type = AuditEventTypes.ES_WRITE_INDEX_UPDATE)
+    @AuditEvent(type = AuditEventTypes.ES_WRITE_INDEX_UPDATE_JOB_START)
     public void cycle() {
         final String msg = "Cycling deflector. Reason: REST request.";
         LOG.info(msg);

graylog2-server/src/main/java/org/graylog2/rest/resources/system/GrokResource.java

@@ -111,7 +111,7 @@ public Response createPattern(@ApiParam(name = "pattern", required = true)
     @PUT
     @Timed
     @ApiOperation("Add a list of new patterns")
-    @AuditEvent(type = AuditEventTypes.GROK_PATTERN_IMPORT)
+    @AuditEvent(type = AuditEventTypes.GROK_PATTERN_IMPORT_CREATE)
     public Response bulkUpdatePatterns(@ApiParam(name = "patterns", required = true) @NotNull GrokPatternList patternList,
                                        @ApiParam(name = "replace", value = "Replace all patterns with the new ones.")
                                        @QueryParam("replace") @DefaultValue("false") boolean replace) throws ValidationException {

graylog2-server/src/main/java/org/graylog2/rest/resources/system/IndexRangesResource.java

@@ -141,7 +141,7 @@ public IndexRangeSummary show(
             @ApiResponse(code = 202, message = "Rebuild/sync systemjob triggered.")
     })
     @Produces(MediaType.APPLICATION_JSON)
-    @AuditEvent(type = AuditEventTypes.ES_INDEX_RANGE_UPDATE)
+    @AuditEvent(type = AuditEventTypes.ES_INDEX_RANGE_UPDATE_JOB)
     public Response rebuild() {
         final SystemJob rebuildJob = rebuildIndexRangesJobFactory.create(this.deflector);
         try {
@@ -166,7 +166,7 @@ public Response rebuild() {
             @ApiResponse(code = 202, message = "Rebuild/sync system job triggered.")
     })
     @Produces(MediaType.APPLICATION_JSON)
-    @AuditEvent(type = AuditEventTypes.ES_INDEX_RANGE_UPDATE)
+    @AuditEvent(type = AuditEventTypes.ES_INDEX_RANGE_UPDATE_JOB)
     public Response rebuildIndex(
             @ApiParam(name = "index", value = "The name of the Graylog-managed Elasticsearch index", required = true)
             @PathParam("index") @NotEmpty String index) {

graylog2-server/src/main/java/org/graylog2/rest/resources/system/jobs/SystemJobResource.java

@@ -170,7 +170,7 @@ public Response trigger(@ApiParam(name = "JSON body", required = true)
             throw new ForbiddenException("Maximum concurrency level of this job reached", e);
         }
 
-        return Response.accepted().build();
+        return Response.accepted().entity(ImmutableMap.of("system_job_id", job.getId())).build();
     }
 
     @DELETE

graylog2-server/src/main/java/org/graylog2/rest/resources/users/UsersResource.java

@@ -458,7 +458,7 @@ public Token generateNewToken(
     @AuditEvent(type = AuditEventTypes.USER_ACCESS_TOKEN_DELETE)
     public void revokeToken(
             @ApiParam(name = "username", required = true) @PathParam("username") String username,
-            @ApiParam(name = "access token", required = true) @PathParam("token") String token) {
+            @ApiParam(name = "token", required = true) @PathParam("token") String token) {
         final User user = _tokensCheckAndLoadUser(username);
         final AccessToken accessToken = accessTokenService.load(token);
 

graylog2-server/src/test/java/org/graylog2/audit/AuditActorTest.java

@@ -47,7 +47,7 @@ public void testSystem() throws Exception {
         when(nodeId.toString()).thenReturn("28164cbe-4ad9-4c9c-a76e-088655aa78892");
         final AuditActor actor = AuditActor.system(nodeId);
 
-        assertThat(actor.urn()).isEqualTo("urn:graylog:server:28164cbe-4ad9-4c9c-a76e-088655aa78892");
+        assertThat(actor.urn()).isEqualTo("urn:graylog:node:28164cbe-4ad9-4c9c-a76e-088655aa78892");
     }
 
     @Test(expected = NullPointerException.class)

graylog2-server/src/test/java/org/graylog2/audit/AuditCoverageTest.java

@@ -85,6 +85,10 @@ public void testAuditEventTypeFormat() throws Exception {
         final Set<String> auditEventTypes = new AuditEventTypes().auditEventTypes();
 
         for (Field field : fields) {
+            // Skip public NAMESPACE field, which is meant to identify server audit events
+            if (field.getName().equals("NAMESPACE")) {
+                continue;
+            }
             String type = "";
             try {
                 type = (String) field.get(field.getType().newInstance());

graylog2-server/src/test/java/org/graylog2/indexer/DeflectorTest.java

@@ -22,10 +22,12 @@
 
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
+import org.graylog2.audit.AuditEventSender;
 import org.graylog2.indexer.indices.Indices;
 import org.graylog2.indexer.indices.jobs.SetIndexReadOnlyAndCalculateRangeJob;
 import org.graylog2.indexer.ranges.CreateNewSingleIndexRangeJob;
 import org.graylog2.indexer.ranges.IndexRangeService;
+import org.graylog2.plugin.system.NodeId;
 import org.graylog2.system.activities.SystemMessageActivityWriter;
 import org.graylog2.system.jobs.SystemJobManager;
 import org.junit.Before;
@@ -62,6 +64,10 @@ public class DeflectorTest {
     private SetIndexReadOnlyAndCalculateRangeJob.Factory setIndexReadOnlyAndCalculateRangeJobFactory;
     @Mock
     private Indices indices;
+    @Mock
+    private AuditEventSender auditEventSender;
+    @Mock
+    private NodeId nodeId;
     private Deflector deflector;
 
     @Mock
@@ -74,6 +80,8 @@ public void setUp() {
             activityWriter,
             indices,
             indexRangeService,
+            auditEventSender,
+            nodeId,
             setIndexReadOnlyAndCalculateRangeJobFactory);
     }
 
@@ -185,6 +193,8 @@ public void getNewestTargetNumber() throws NoTargetIndexException {
             activityWriter,
             indices,
             indexRangeService,
+            auditEventSender,
+            nodeId,
             setIndexReadOnlyAndCalculateRangeJobFactory);
 
         final int number = deflector.getNewestTargetNumber();
@@ -207,6 +217,8 @@ public void getAllGraylogIndexNames() {
             activityWriter,
             indices,
             indexRangeService,
+            auditEventSender,
+            nodeId,
             setIndexReadOnlyAndCalculateRangeJobFactory);
 
         final String[] allGraylogIndexNames = deflector.getAllGraylogIndexNames();
@@ -230,6 +242,8 @@ public void getAllGraylogDeflectorIndices() {
             activityWriter,
             indices,
             indexRangeService,
+            auditEventSender,
+            nodeId,
             setIndexReadOnlyAndCalculateRangeJobFactory);
 
         final Map<String, Set<String>> deflectorIndices = deflector.getAllGraylogDeflectorIndices();
@@ -254,6 +268,8 @@ public void testCleanupAliases() throws Exception {
                 activityWriter,
                 indices,
                 indexRangeService,
+                auditEventSender,
+                nodeId,
                 setIndexReadOnlyAndCalculateRangeJobFactory);
 
         deflector.cleanupAliases(Sets.newHashSet("graylog_2", "graylog_3", "foobar"));

graylog2-server/src/test/java/org/graylog2/indexer/ranges/EsIndexRangeServiceTest.java

@@ -25,6 +25,7 @@
 import com.lordofthejars.nosqlunit.elasticsearch2.EmbeddedElasticsearch;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.settings.Settings;
+import org.graylog2.audit.AuditEventSender;
 import org.graylog2.audit.NullAuditEventSender;
 import org.graylog2.configuration.ElasticsearchConfiguration;
 import org.graylog2.database.NotFoundException;
@@ -76,6 +77,10 @@ public String getIndexPrefix() {
     private Indices indices;
     @Mock
     private EventBus localEventBus;
+    @Mock
+    private AuditEventSender auditEventSender;
+    @Mock
+    private NodeId nodeId;
     private EsIndexRangeService indexRangeService;
 
     public EsIndexRangeServiceTest() {
@@ -88,7 +93,7 @@ public void setUp() throws Exception {
         final Messages messages = new Messages(client, ELASTICSEARCH_CONFIGURATION, new MetricRegistry());
         indices = new Indices(client, ELASTICSEARCH_CONFIGURATION, new IndexMapping(), messages, mock(NodeId.class), new NullAuditEventSender());
         final Deflector deflector = new Deflector(null, ELASTICSEARCH_CONFIGURATION.getIndexPrefix(), new NullActivityWriter(),
-            indices, null, null);
+            indices, null, auditEventSender, nodeId, null);
         indexRangeService = new EsIndexRangeService(client, deflector, localEventBus, new MetricRegistry());
     }