Merge branch 'cold-tier' into stream-datawarehouse

Graylog2 · May 21, 2024 · f3b5851 · f3b5851
2 parents c1792be + 12addcd
commit f3b5851
Show file tree

Hide file tree

Showing 68 changed files with 789 additions and 474 deletions.
diff --git a/changelog/unreleased/issue-19139.toml b/changelog/unreleased/issue-19139.toml
@@ -1,5 +1,5 @@
 type="a"
-message="Adds export into csv, json, yaml and xml actions for aggregation widget. Also we change placment of message widget export action"
+message="Change placement of message widget export action"
 
 issues=["19139"]
 pulls=["19140"]
diff --git a/data-node/src/test/java/org/graylog/datanode/integration/DatanodeClusterIT.java b/data-node/src/test/java/org/graylog/datanode/integration/DatanodeClusterIT.java
@@ -16,21 +16,18 @@
  */
 package org.graylog.datanode.integration;
 
-import com.github.joschi.jadconfig.util.Duration;
 import com.github.rholder.retry.RetryException;
-import jakarta.inject.Provider;
 import jakarta.validation.constraints.NotNull;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.graylog.datanode.configuration.variants.KeystoreInformation;
-import org.graylog.datanode.restoperations.DatanodeOpensearchWait;
-import org.graylog.datanode.restoperations.DatanodeRestApiWait;
-import org.graylog.datanode.restoperations.DatanodeStatusChangeOperation;
-import org.graylog.datanode.restoperations.OpensearchTestIndexCreation;
-import org.graylog.datanode.restoperations.RestOperationParameters;
+import org.graylog.testing.restoperations.DatanodeOpensearchWait;
+import org.graylog.testing.restoperations.DatanodeRestApiWait;
+import org.graylog.testing.restoperations.DatanodeStatusChangeOperation;
+import org.graylog.testing.restoperations.OpensearchTestIndexCreation;
+import org.graylog.testing.restoperations.RestOperationParameters;
 import org.graylog.datanode.testinfra.DatanodeContainerizedBackend;
 import org.graylog.testing.containermatrix.MongodbServer;
 import org.graylog.testing.mongodb.MongoDBTestService;
-import org.graylog2.security.IndexerJwtAuthTokenProvider;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
@@ -41,7 +38,6 @@
 import org.testcontainers.containers.Network;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.security.GeneralSecurityException;
 import java.security.KeyStore;

diff --git a/data-node/src/test/java/org/graylog/datanode/integration/DatanodeLifecycleIT.java b/data-node/src/test/java/org/graylog/datanode/integration/DatanodeLifecycleIT.java
@@ -16,16 +16,13 @@
  */
 package org.graylog.datanode.integration;
 
-import com.github.joschi.jadconfig.util.Duration;
-import com.github.rholder.retry.RetryException;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.graylog.datanode.configuration.variants.KeystoreInformation;
-import org.graylog.datanode.restoperations.DatanodeRestApiWait;
-import org.graylog.datanode.restoperations.DatanodeStatusChangeOperation;
-import org.graylog.datanode.restoperations.RestOperationParameters;
+import org.graylog.testing.restoperations.DatanodeRestApiWait;
+import org.graylog.testing.restoperations.DatanodeStatusChangeOperation;
+import org.graylog.testing.restoperations.RestOperationParameters;
 import org.graylog.datanode.testinfra.DatanodeContainerizedBackend;
 import org.graylog2.plugin.Tools;
-import org.graylog2.security.IndexerJwtAuthTokenProvider;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -37,10 +34,8 @@
 import java.nio.file.Path;
 import java.security.GeneralSecurityException;
 import java.security.KeyStore;
-import java.util.concurrent.ExecutionException;
 
 import static org.graylog.datanode.testinfra.DatanodeContainerizedBackend.IMAGE_WORKING_DIR;
-import static org.graylog.testing.completebackend.ContainerizedGraylogBackend.ROOT_PASSWORD_PLAINTEXT;
 
 public class DatanodeLifecycleIT {
     private static final Logger LOG = LoggerFactory.getLogger(DatanodeLifecycleIT.class);

diff --git a/data-node/src/test/java/org/graylog/datanode/integration/DatanodeSearchableSnapshotsIT.java b/data-node/src/test/java/org/graylog/datanode/integration/DatanodeSearchableSnapshotsIT.java
@@ -19,8 +19,8 @@
 import com.github.rholder.retry.RetryException;
 import io.restassured.http.ContentType;
 import io.restassured.response.ValidatableResponse;
-import org.graylog.datanode.restoperations.DatanodeOpensearchWait;
-import org.graylog.datanode.restoperations.RestOperationParameters;
+import org.graylog.testing.restoperations.DatanodeOpensearchWait;
+import org.graylog.testing.restoperations.RestOperationParameters;
 import org.graylog.datanode.testinfra.DatanodeContainerizedBackend;
 import org.graylog.testing.completebackend.S3MinioContainer;
 import org.graylog.testing.containermatrix.MongodbServer;

diff --git a/data-node/src/test/java/org/graylog/datanode/integration/DatanodeSecuritySetupIT.java b/data-node/src/test/java/org/graylog/datanode/integration/DatanodeSecuritySetupIT.java
@@ -16,16 +16,14 @@
  */
 package org.graylog.datanode.integration;
 
-import com.github.joschi.jadconfig.util.Duration;
 import com.github.rholder.retry.RetryException;
 import io.restassured.response.ValidatableResponse;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.graylog.datanode.configuration.variants.KeystoreInformation;
-import org.graylog.datanode.restoperations.DatanodeRestApiWait;
-import org.graylog.datanode.restoperations.RestOperationParameters;
+import org.graylog.testing.restoperations.DatanodeRestApiWait;
+import org.graylog.testing.restoperations.RestOperationParameters;
 import org.graylog.datanode.testinfra.DatanodeContainerizedBackend;
 import org.graylog2.plugin.Tools;
-import org.graylog2.security.IndexerJwtAuthTokenProvider;
 import org.hamcrest.Matchers;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
@@ -35,7 +33,6 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.security.GeneralSecurityException;
 import java.security.KeyStore;

diff --git a/full-backend-tests/src/test/java/org/graylog/datanode/DatanodeProvisioningIT.java b/full-backend-tests/src/test/java/org/graylog/datanode/DatanodeProvisioningIT.java
@@ -17,6 +17,7 @@
 package org.graylog.datanode;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.github.joschi.jadconfig.util.Duration;
 import com.github.rholder.retry.RetryException;
 import com.github.rholder.retry.RetryerBuilder;
 import com.github.rholder.retry.StopStrategies;
@@ -29,20 +30,20 @@
 import org.apache.commons.lang.RandomStringUtils;
 import org.apache.http.HttpStatus;
 import org.assertj.core.api.Assertions;
-import org.graylog.datanode.restoperations.DatanodeOpensearchWait;
-import org.graylog.datanode.restoperations.RestOperationParameters;
-import org.graylog.datanode.testinfra.DatanodeContainerizedBackend;
-import org.graylog.datanode.testinfra.DatanodeDevContainerBuilder;
 import org.graylog.security.certutil.CertConstants;
 import org.graylog.security.certutil.CertutilCa;
 import org.graylog.security.certutil.console.TestableConsole;
+import org.graylog.testing.completebackend.ContainerizedGraylogBackend;
 import org.graylog.testing.completebackend.Lifecycle;
 import org.graylog.testing.completebackend.apis.GraylogApis;
 import org.graylog.testing.containermatrix.SearchServer;
 import org.graylog.testing.containermatrix.annotations.ContainerMatrixTest;
 import org.graylog.testing.containermatrix.annotations.ContainerMatrixTestsConfiguration;
+import org.graylog.testing.restoperations.DatanodeOpensearchWait;
+import org.graylog.testing.restoperations.RestOperationParameters;
 import org.graylog2.cluster.nodes.DataNodeStatus;
 import org.graylog2.cluster.preflight.DataNodeProvisioningConfig;
+import org.graylog2.security.IndexerJwtAuthTokenProvider;
 import org.junit.jupiter.api.io.TempDir;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -68,7 +69,7 @@
 
 @ContainerMatrixTestsConfiguration(serverLifecycle = Lifecycle.CLASS, searchVersions = SearchServer.DATANODE_DEV,
                                    additionalConfigurationParameters = {
-                                           @ContainerMatrixTestsConfiguration.ConfigurationParameter(key = DatanodeDevContainerBuilder.ENV_INSECURE_STARTUP, value = "false"),
+                                           @ContainerMatrixTestsConfiguration.ConfigurationParameter(key = "GRAYLOG_DATANODE_INSECURE_STARTUP", value = "false"),
                                            @ContainerMatrixTestsConfiguration.ConfigurationParameter(key = "GRAYLOG_ELASTICSEARCH_HOSTS", value = ""),
                                    })
 public class DatanodeProvisioningIT {
@@ -114,7 +115,7 @@ private void testEncryptedConnectionToOpensearch(KeyStore truststore) throws Exe
             new DatanodeOpensearchWait(RestOperationParameters.builder()
                     .port(getOpensearchPort())
                     .truststore(truststore)
-                    .jwtTokenProvider(DatanodeContainerizedBackend.JWT_AUTH_TOKEN_PROVIDER)
+                    .jwtTokenProvider(new IndexerJwtAuthTokenProvider(ContainerizedGraylogBackend.PASSWORD_SECRET, Duration.seconds(120), Duration.seconds(60)))
                     .build())
                     .waitForNodesCount(1);
         } catch (Exception e) {

diff --git a/...p/preflight/web/resources/CAResource.java → ...org/graylog/security/rest/CAResource.java b/...p/preflight/web/resources/CAResource.java → ...org/graylog/security/rest/CAResource.java
@@ -14,7 +14,7 @@
  * along with this program. If not, see
  * <http://www.mongodb.com/licensing/server-side-public-license>.
  */
-package org.graylog2.bootstrap.preflight.web.resources;
+package org.graylog.security.rest;
 
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
@@ -77,6 +77,7 @@ public CAResource(final CaService caService,
 
     @GET
     @ApiOperation("Returns the CA")
+    @RequiresPermissions(RestPermissions.GRAYLOG_CA_READ)
     public CA get() throws KeyStoreStorageException {
         return caService.get();
     }

diff --git a/...resources/CertificateRenewalResource.java → ...rity/rest/CertificateRenewalResource.java b/...resources/CertificateRenewalResource.java → ...rity/rest/CertificateRenewalResource.java
@@ -14,26 +14,24 @@
  * along with this program. If not, see
  * <http://www.mongodb.com/licensing/server-side-public-license>.
  */
-package org.graylog2.bootstrap.preflight.web.resources;
+package org.graylog.security.rest;
 
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiParam;
-import org.apache.shiro.authz.annotation.RequiresAuthentication;
-import org.apache.shiro.authz.annotation.RequiresPermissions;
-import org.graylog.security.certutil.CertRenewalService;
-import org.graylog2.audit.AuditEventTypes;
-import org.graylog2.audit.jersey.AuditEvent;
-import org.graylog2.plugin.rest.PluginRestResource;
-import org.graylog2.shared.security.RestPermissions;
-
 import jakarta.inject.Inject;
-
 import jakarta.ws.rs.GET;
 import jakarta.ws.rs.POST;
 import jakarta.ws.rs.Path;
 import jakarta.ws.rs.PathParam;
 import jakarta.ws.rs.Produces;
 import jakarta.ws.rs.core.MediaType;
+import org.apache.shiro.authz.annotation.RequiresAuthentication;
+import org.apache.shiro.authz.annotation.RequiresPermissions;
+import org.graylog.security.certutil.CertRenewalService;
+import org.graylog2.audit.AuditEventTypes;
+import org.graylog2.audit.jersey.AuditEvent;
+import org.graylog2.plugin.rest.PluginRestResource;
+import org.graylog2.shared.security.RestPermissions;
 
 import java.util.List;
 

diff --git a/graylog2-server/src/main/java/org/graylog2/bootstrap/preflight/PreflightWebModule.java b/graylog2-server/src/main/java/org/graylog2/bootstrap/preflight/PreflightWebModule.java
@@ -59,6 +59,9 @@
 public class PreflightWebModule extends Graylog2Module {
 
     public static final String FEATURE_FLAG_PREFLIGHT_WEB_ENABLED = "preflight_web";
+    public static final String PERMISSION_PREFLIGHT_ONLY = "preflight:only";
+    // this permission is never checked during preflight, but makes sure that the rest resources are not accidentally
+    // bound during regular startup of Graylog and available without permissions.
 
     private final Configuration configuration;
 

diff --git a/...java/org/graylog2/bootstrap/preflight/web/resources/CertificateRenewalPolicyResource.java b/...java/org/graylog2/bootstrap/preflight/web/resources/CertificateRenewalPolicyResource.java
@@ -16,20 +16,19 @@
  */
 package org.graylog2.bootstrap.preflight.web.resources;
 
-import org.graylog2.audit.jersey.NoAuditEvent;
-import org.graylog2.bootstrap.preflight.PreflightConstants;
-import org.graylog2.plugin.certificates.RenewalPolicy;
-import org.graylog2.plugin.cluster.ClusterConfigService;
-
 import jakarta.inject.Inject;
-
 import jakarta.validation.constraints.NotNull;
-
 import jakarta.ws.rs.GET;
 import jakarta.ws.rs.POST;
 import jakarta.ws.rs.Path;
 import jakarta.ws.rs.Produces;
 import jakarta.ws.rs.core.MediaType;
+import org.apache.shiro.authz.annotation.RequiresPermissions;
+import org.graylog2.audit.jersey.NoAuditEvent;
+import org.graylog2.bootstrap.preflight.PreflightConstants;
+import org.graylog2.bootstrap.preflight.PreflightWebModule;
+import org.graylog2.plugin.certificates.RenewalPolicy;
+import org.graylog2.plugin.cluster.ClusterConfigService;
 
 @Path(PreflightConstants.API_PREFIX + "renewal_policy")
 @Produces(MediaType.APPLICATION_JSON)
@@ -42,12 +41,14 @@ public CertificateRenewalPolicyResource(final ClusterConfigService clusterConfig
     }
 
     @GET
+    @RequiresPermissions(PreflightWebModule.PERMISSION_PREFLIGHT_ONLY)
     public RenewalPolicy get() {
         return this.clusterConfigService.get(RenewalPolicy.class);
     }
 
     @POST
-    @NoAuditEvent("No Audit Event needed")
+    @RequiresPermissions(PreflightWebModule.PERMISSION_PREFLIGHT_ONLY)
+    @NoAuditEvent("No Auditing during preflight")
     public void set(@NotNull RenewalPolicy renewalPolicy) {
         this.clusterConfigService.write(renewalPolicy);
     }

diff --git a/...ain/java/org/graylog2/bootstrap/preflight/web/resources/DataNodeProvisioningResource.java b/...ain/java/org/graylog2/bootstrap/preflight/web/resources/DataNodeProvisioningResource.java
diff --git a/...src/main/java/org/graylog2/bootstrap/preflight/web/resources/PreflightAssetsResource.java b/...src/main/java/org/graylog2/bootstrap/preflight/web/resources/PreflightAssetsResource.java
@@ -22,6 +22,7 @@
 import com.google.common.hash.HashCode;
 import com.google.common.hash.Hashing;
 import com.google.common.io.Resources;
+import org.apache.shiro.authz.annotation.RequiresPermissions;
 import org.graylog2.bootstrap.preflight.PreflightConstants;
 
 import javax.activation.MimetypesFileTypeMap;
@@ -40,6 +41,7 @@
 import jakarta.ws.rs.core.MediaType;
 import jakarta.ws.rs.core.Request;
 import jakarta.ws.rs.core.Response;
+import org.graylog2.bootstrap.preflight.PreflightWebModule;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -87,12 +89,14 @@ public FileSystem load(@Nonnull URI key) throws Exception {
 
     @Produces(MediaType.TEXT_HTML)
     @GET
+    @RequiresPermissions(PreflightWebModule.PERMISSION_PREFLIGHT_ONLY)
     public Response index(@Context Request request) {
         return this.get(request, "index.html");
     }
 
     @Path("/{filename}")
     @GET
+    @RequiresPermissions(PreflightWebModule.PERMISSION_PREFLIGHT_ONLY)
     public Response get(@Context Request request, @PathParam("filename") String filename) {
         try {
             final URL resourceUrl = getResourceUri(filename);