Exporting through frontend without buffering response in browser.

graylog2-server/src/main/java/org/graylog/plugins/views/ViewsBindings.java

@@ -49,6 +49,7 @@
 import org.graylog.plugins.views.search.filter.QueryStringFilter;
 import org.graylog.plugins.views.search.filter.StreamFilter;
 import org.graylog.plugins.views.search.rest.DashboardsResource;
+import org.graylog.plugins.views.search.rest.ExportJobsResource;
 import org.graylog.plugins.views.search.rest.FieldTypesResource;
 import org.graylog.plugins.views.search.rest.MessageExportFormatFilter;
 import org.graylog.plugins.views.search.rest.MessagesResource;
@@ -113,6 +114,7 @@ protected void configure() {
         addSystemRestResource(DashboardsResource.class);
         addSystemRestResource(FieldTypesResource.class);
         addSystemRestResource(MessagesResource.class);
+        addSystemRestResource(ExportJobsResource.class);
         addSystemRestResource(PivotSeriesFunctionsResource.class);
         addSystemRestResource(QualifyingViewsResource.class);
         addSystemRestResource(SavedSearchesResource.class);

graylog2-server/src/main/java/org/graylog/plugins/views/audit/ViewsAuditEventTypes.java

@@ -49,6 +49,8 @@ public class ViewsAuditEventTypes implements PluginAuditEventTypes {
     public static final String MESSAGES_EXPORT_REQUESTED = MESSAGES_EXPORT + "_requested";
     public static final String MESSAGES_EXPORT_SUCCEEDED = MESSAGES_EXPORT + "_succeeded";
 
+    public static final String EXPORT_JOB = "export_job";
+    public static final String EXPORT_JOB_CREATED = PREFIX + EXPORT_JOB + ":created";
 
     private static final ImmutableSet<String> EVENT_TYPES = ImmutableSet.<String>builder()
             .add(VIEW_CREATE)
@@ -68,6 +70,8 @@ public class ViewsAuditEventTypes implements PluginAuditEventTypes {
             .add(MESSAGES_EXPORT_REQUESTED)
             .add(MESSAGES_EXPORT_SUCCEEDED)
 
+            .add(EXPORT_JOB_CREATED)
+
             .build();
 
     @Override

graylog2-server/src/main/java/org/graylog/plugins/views/search/export/CommandFactory.java

@@ -108,7 +108,7 @@ private ExportMessagesCommand.Builder builderFrom(ResultFormat resultFormat) {
         requestBuilder.fieldsInOrder(resultFormat.fieldsInOrder());
 
         if (resultFormat.limit().isPresent()) {
-            requestBuilder.limit(resultFormat.limit().getAsInt());
+            requestBuilder.limit(resultFormat.limit().orElseThrow(() -> new IllegalStateException("No value present!")));
         }
 
         return requestBuilder;

graylog2-server/src/main/java/org/graylog/plugins/views/search/export/ExportJob.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.views.search.export;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.joda.time.DateTime;
+
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = ExportJob.FIELD_TYPE)
+@JsonSubTypes({
+        @JsonSubTypes.Type(value = MessagesRequestExportJob.class, name = MessagesRequestExportJob.TYPE),
+        @JsonSubTypes.Type(value = SearchExportJob.class, name = SearchExportJob.TYPE),
+        @JsonSubTypes.Type(value = SearchTypeExportJob.class, name = SearchTypeExportJob.TYPE)
+})
+public interface ExportJob {
+    String FIELD_TYPE = "type";
+    String FIELD_ID = "_id";
+    String FIELD_CREATED_AT = "created_at";
+    String id();
+
+    @JsonProperty
+    DateTime createdAt();
+}

graylog2-server/src/main/java/org/graylog/plugins/views/search/export/ExportJobFactory.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.views.search.export;
+
+import org.bson.types.ObjectId;
+
+import java.util.function.Supplier;
+
+public class ExportJobFactory {
+    private final Supplier<String> idGenerator = () -> new ObjectId().toHexString();
+
+    public ExportJob fromMessagesRequest(MessagesRequest messagesRequest) {
+        return MessagesRequestExportJob.fromMessagesRequest(idGenerator.get(), messagesRequest);
+    }
+
+    public ExportJob forSearch(String searchId, ResultFormat resultFormat) {
+        return SearchExportJob.forSearch(idGenerator.get(), searchId, resultFormat);
+    }
+
+    public ExportJob forSearchType(String searchId, String searchTypeId, ResultFormat resultFormat) {
+        return SearchTypeExportJob.forSearchType(idGenerator.get(), searchId, searchTypeId, resultFormat);
+    }
+}

graylog2-server/src/main/java/org/graylog/plugins/views/search/export/ExportJobService.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.views.search.export;
+
+import com.mongodb.BasicDBObject;
+import org.bson.types.ObjectId;
+import org.graylog2.bindings.providers.MongoJackObjectMapperProvider;
+import org.graylog2.database.MongoConnection;
+import org.mongojack.JacksonDBCollection;
+import org.mongojack.WriteResult;
+
+import javax.inject.Inject;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+
+public class ExportJobService {
+    protected final JacksonDBCollection<ExportJob, ObjectId> db;
+
+    @Inject
+    public ExportJobService(MongoConnection mongoConnection,
+                            MongoJackObjectMapperProvider mapper) {
+        db = JacksonDBCollection.wrap(mongoConnection.getDatabase().getCollection("export_jobs"),
+                ExportJob.class,
+                ObjectId.class,
+                mapper.get());
+
+        db.createIndex(new BasicDBObject(ExportJob.FIELD_CREATED_AT, 1), new BasicDBObject("expireAfterSeconds", TimeUnit.HOURS.toSeconds(1L)));
+    }
+
+    public Optional<ExportJob> get(String id) {
+        if (!ObjectId.isValid(id)) {
+            return Optional.empty();
+        }
+        return Optional.ofNullable(db.findOneById(new ObjectId(id)));
+    }
+
+    public String save(ExportJob exportJob) {
+        final WriteResult<ExportJob, ObjectId> save = db.insert(exportJob);
+
+        return save.getSavedId().toHexString();
+    }
+}

graylog2-server/src/main/java/org/graylog/plugins/views/search/export/MessagesRequestExportJob.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.views.search.export;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.auto.value.AutoValue;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+@AutoValue
+@JsonAutoDetect
+public abstract class MessagesRequestExportJob implements ExportJob {
+    static final String TYPE = "messages_export";
+    private static final String FIELD_MESSAGES_REQUEST = "messages_request";
+
+    @JsonProperty("type")
+    public String type() {
+        return TYPE;
+    }
+
+    @JsonProperty(FIELD_MESSAGES_REQUEST)
+    public abstract MessagesRequest messagesRequest();
+
+    static MessagesRequestExportJob fromMessagesRequest(String id, MessagesRequest messagesRequest) {
+        return new AutoValue_MessagesRequestExportJob(id, DateTime.now(DateTimeZone.UTC), messagesRequest);
+    }
+
+    @JsonCreator
+    static MessagesRequestExportJob create(
+            @JsonProperty(FIELD_ID) String id,
+            @JsonProperty(FIELD_MESSAGES_REQUEST) MessagesRequest messagesRequest
+    ) {
+        return new AutoValue_MessagesRequestExportJob(id, DateTime.now(DateTimeZone.UTC), messagesRequest);
+    }
+}

graylog2-server/src/main/java/org/graylog/plugins/views/search/export/ResultFormat.java

@@ -30,7 +30,6 @@
 import java.util.LinkedHashSet;
 import java.util.Map;
 import java.util.Optional;
-import java.util.OptionalInt;
 
 import static org.graylog.plugins.views.search.export.ExportMessagesCommand.DEFAULT_FIELDS;
 import static org.graylog.plugins.views.search.export.LinkedHashSetUtil.linkedHashSetOf;
@@ -49,12 +48,14 @@ public abstract class ResultFormat {
     public abstract Optional<AbsoluteRange> timerange();
 
     @JsonProperty
-    @Positive
-    public abstract OptionalInt limit();
+    public abstract Optional<Integer> limit();
 
     @JsonProperty
     public abstract Map<String, Object> executionState();
 
+    @JsonProperty
+    public abstract Optional<String> filename();
+
     public static ResultFormat.Builder builder() {
         return ResultFormat.Builder.create();
     }
@@ -73,14 +74,17 @@ public Builder fieldsInOrder(String... fields) {
         }
 
         @JsonProperty
-        public abstract Builder limit(Integer limit);
+        public abstract Builder limit(@Positive @Nullable Integer limit);
 
         @JsonProperty
         public abstract Builder executionState(Map<String, Object> executionState);
 
         @JsonProperty
         public abstract Builder timerange(@Nullable AbsoluteRange timeRange);
 
+        @JsonProperty
+        public abstract Builder filename(@Nullable String filename);
+
         public abstract ResultFormat build();
 
         @JsonCreator

graylog2-server/src/main/java/org/graylog/plugins/views/search/export/SearchExportJob.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.views.search.export;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.auto.value.AutoValue;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+import javax.validation.constraints.NotEmpty;
+
+@AutoValue
+@JsonAutoDetect
+public abstract class SearchExportJob implements ExportJob {
+    static final String TYPE = "search_export";
+    private static final String FIELD_SEARCH_ID = "search_id";
+    private static final String FIELD_RESULT_FORMAT = "result_format";
+
+    @JsonProperty("type")
+    public String type() {
+        return TYPE;
+    }
+
+    @JsonProperty(FIELD_SEARCH_ID)
+    public abstract String searchId();
+
+    @JsonProperty(FIELD_RESULT_FORMAT)
+    @NotEmpty
+    public abstract ResultFormat resultFormat();
+
+    static SearchExportJob forSearch(String id, String searchId, ResultFormat resultFormat) {
+        return new AutoValue_SearchExportJob(id, DateTime.now(DateTimeZone.UTC), searchId, resultFormat);
+    }
+
+    @JsonCreator
+    static SearchExportJob create(
+            @JsonProperty(FIELD_ID) String id,
+            @JsonProperty(FIELD_SEARCH_ID) String searchId,
+            @JsonProperty(FIELD_RESULT_FORMAT) ResultFormat resultFormat
+    ) {
+        return new AutoValue_SearchExportJob(id,
+                DateTime.now(DateTimeZone.UTC),
+                searchId,
+                resultFormat);
+    }
+}