Assign messages to the default stream (#3098)

* add default stream to each message when entering the processing engine

Unify legacy ProcessBufferProcessor class with its sole subclass
turn its provider into a factory to save mutable state
create singleton provider for default stream

* prevent updates (create, edit, delete) to default stream or its stream rules

the default stream it doesn't have rules and the UI will not show the actions necessary to modify it, this prevents using the API to mutate it

* the default stream doesn't have stream rules, don't check the service in tests

* Remove obsolete DefaultStreamCreated marker

graylog2-server/src/main/java/org/graylog2/bindings/ServerBindings.java

@@ -20,6 +20,7 @@
 import com.google.inject.assistedinject.FactoryModuleBuilder;
 import com.google.inject.multibindings.MapBinder;
 import com.google.inject.multibindings.Multibinder;
+
 import org.apache.shiro.mgt.DefaultSecurityManager;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.node.Node;
@@ -30,13 +31,13 @@
 import org.graylog2.bindings.providers.BundleImporterProvider;
 import org.graylog2.bindings.providers.ClusterEventBusProvider;
 import org.graylog2.bindings.providers.DefaultSecurityManagerProvider;
+import org.graylog2.bindings.providers.DefaultStreamProvider;
 import org.graylog2.bindings.providers.EsClientProvider;
 import org.graylog2.bindings.providers.EsNodeProvider;
 import org.graylog2.bindings.providers.MongoConnectionProvider;
 import org.graylog2.bindings.providers.RulesEngineProvider;
 import org.graylog2.bindings.providers.SystemJobFactoryProvider;
 import org.graylog2.bindings.providers.SystemJobManagerProvider;
-import org.graylog2.buffers.processors.ServerProcessBufferProcessor;
 import org.graylog2.bundles.BundleService;
 import org.graylog2.cluster.ClusterConfigServiceImpl;
 import org.graylog2.dashboards.widgets.WidgetCacheTime;
@@ -61,6 +62,8 @@
 import org.graylog2.plugin.cluster.ClusterConfigService;
 import org.graylog2.plugin.decorators.SearchResponseDecorator;
 import org.graylog2.plugin.inject.Graylog2Module;
+import org.graylog2.plugin.streams.DefaultStream;
+import org.graylog2.plugin.streams.Stream;
 import org.graylog2.rest.NotFoundExceptionMapper;
 import org.graylog2.rest.ScrollChunkWriter;
 import org.graylog2.rest.ValidationExceptionMapper;
@@ -134,6 +137,9 @@ private void bindFactoryModules() {
         install(new FactoryModuleBuilder().build(LdapSettingsImpl.Factory.class));
         install(new FactoryModuleBuilder().build(WidgetCacheTime.Factory.class));
         install(new FactoryModuleBuilder().build(UserImpl.Factory.class));
+
+        install(new FactoryModuleBuilder().build(ProcessBufferProcessor.Factory.class));
+        bind(Stream.class).annotatedWith(DefaultStream.class).toProvider(DefaultStreamProvider.class);
     }
 
     private void bindSingletons() {
@@ -176,7 +182,6 @@ private void bindInterfaces() {
         bind(ActivityWriter.class).to(SystemMessageActivityWriter.class);
         bind(PersistedInputs.class).to(PersistedInputsImpl.class);
 
-        bind(ProcessBufferProcessor.class).to(ServerProcessBufferProcessor.class);
         bind(RoleService.class).to(RoleServiceImpl.class).in(Scopes.SINGLETON);
     }
 

graylog2-server/src/main/java/org/graylog2/bindings/providers/DefaultStreamProvider.java

@@ -0,0 +1,71 @@
+/**
+ * This file is part of Graylog.
+ *
+ * Graylog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.graylog2.bindings.providers;
+
+import org.graylog2.database.NotFoundException;
+import org.graylog2.gelfclient.util.Uninterruptibles;
+import org.graylog2.plugin.streams.Stream;
+import org.graylog2.streams.StreamService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.inject.Inject;
+import javax.inject.Provider;
+import javax.inject.Singleton;
+
+@Singleton
+public class DefaultStreamProvider implements Provider<Stream> {
+    private static final Logger LOG = LoggerFactory.getLogger(DefaultStreamProvider.class);
+
+    private final StreamService service;
+
+    private AtomicReference<Stream> sharedInstance = new AtomicReference<>();
+
+    @Inject
+    private DefaultStreamProvider(StreamService service) {
+        this.service = service;
+    }
+
+    @Override
+    public Stream get() {
+        Stream defaultStream = sharedInstance.get();
+        if (defaultStream != null) {
+            return defaultStream;
+        }
+
+        synchronized (this) {
+            defaultStream = sharedInstance.get();
+            if (defaultStream != null) {
+                return defaultStream;
+            }
+            do {
+                try {
+                    LOG.debug("Loading shared default stream instance");
+                    defaultStream = service.load(Stream.DEFAULT_STREAM_ID);
+                } catch (NotFoundException ignored) {
+                    LOG.warn("Unable to load default stream, retrying. Processing is blocked until this succeeds.");
+                    Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
+                }
+            } while (defaultStream == null);
+            sharedInstance.set(defaultStream);
+        }
+        return defaultStream;
+    }
+}

graylog2-server/src/main/java/org/graylog2/periodical/DefaultStreamMigrationPeriodical.java

@@ -18,18 +18,14 @@
 
 import com.google.common.collect.ImmutableMap;
 import org.bson.types.ObjectId;
+import org.graylog2.database.NotFoundException;
 import org.graylog2.events.ClusterEventBus;
 import org.graylog2.plugin.cluster.ClusterConfigService;
 import org.graylog2.plugin.database.ValidationException;
 import org.graylog2.plugin.periodical.Periodical;
 import org.graylog2.plugin.streams.Stream;
-import org.graylog2.plugin.streams.StreamRule;
-import org.graylog2.plugin.streams.StreamRuleType;
 import org.graylog2.streams.StreamImpl;
-import org.graylog2.streams.StreamRuleImpl;
-import org.graylog2.streams.StreamRuleService;
 import org.graylog2.streams.StreamService;
-import org.graylog2.streams.config.DefaultStreamCreated;
 import org.graylog2.streams.events.StreamsChangedEvent;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
@@ -47,17 +43,14 @@ public class DefaultStreamMigrationPeriodical extends Periodical {
     private static final Logger LOG = LoggerFactory.getLogger(DefaultStreamMigrationPeriodical.class);
 
     private final StreamService streamService;
-    private final StreamRuleService streamRuleService;
     private final ClusterEventBus clusterEventBus;
     private final ClusterConfigService clusterConfigService;
 
     @Inject
     public DefaultStreamMigrationPeriodical(final StreamService streamService,
-                                            final StreamRuleService streamRuleService,
                                             final ClusterEventBus clusterEventBus,
                                             final ClusterConfigService clusterConfigService) {
         this.streamService = streamService;
-        this.streamRuleService = streamRuleService;
         this.clusterEventBus = clusterEventBus;
         this.clusterConfigService = clusterConfigService;
     }
@@ -75,21 +68,11 @@ public void doRun() {
                 .put(StreamImpl.FIELD_DEFAULT_STREAM, true)
                 .build();
         final Stream stream = new StreamImpl(id, fields, Collections.emptyList(), Collections.emptySet(), Collections.emptySet());
-        final StreamRule streamRule = new StreamRuleImpl(
-                ImmutableMap.<String, Object>builder()
-                        .put(StreamRuleImpl.FIELD_TYPE, StreamRuleType.ALWAYS_MATCH.getValue())
-                        .put(StreamRuleImpl.FIELD_FIELD, "timestamp")
-                        .put(StreamRuleImpl.FIELD_INVERTED, false)
-                        .put(StreamRuleImpl.FIELD_STREAM_ID, id)
-                        .put(StreamRuleImpl.FIELD_DESCRIPTION, "Match all messages")
-                        .build());
+
         try {
             streamService.save(stream);
-            streamRuleService.save(streamRule);
-
             LOG.info("Successfully created default stream: {}", stream.getTitle());
 
-            clusterConfigService.write(DefaultStreamCreated.create());
             clusterEventBus.post(StreamsChangedEvent.create(stream.getId()));
         } catch (ValidationException e) {
             LOG.error("Couldn't create default stream", e);
@@ -113,7 +96,12 @@ public boolean masterOnly() {
 
     @Override
     public boolean startOnThisNode() {
-        return clusterConfigService.get(DefaultStreamCreated.class) == null;
+        try {
+            return streamService.load(Stream.DEFAULT_STREAM_ID) == null;
+        } catch (NotFoundException ignored) {
+            // if the stream cannot be found, recreate it
+            return true;
+        }
     }
 
     @Override

graylog2-server/src/main/java/org/graylog2/streams/config/DefaultStreamCreated.java → graylog2-server/src/main/java/org/graylog2/plugin/streams/DefaultStream.java

@@ -14,17 +14,17 @@
  * You should have received a copy of the GNU General Public License
  * along with Graylog.  If not, see <http://www.gnu.org/licenses/>.
  */
-package org.graylog2.streams.config;
+package org.graylog2.plugin.streams;
 
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.google.auto.value.AutoValue;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
 
-@JsonAutoDetect
-@AutoValue
-public abstract class DefaultStreamCreated {
-    @JsonCreator
-    public static DefaultStreamCreated create() {
-        return new AutoValue_DefaultStreamCreated();
-    }
+import javax.inject.Qualifier;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.PARAMETER)
+@Qualifier
+public @interface DefaultStream {
 }