Avoid NPE in set_security_user without security (#52691)

If security was disabled (explicitly), then the SecurityContext would
be null, but the set_security_user processor was still registered.

Attempting to define a pipeline that used that processor would fail
with an (intentional) NPE. This behaviour, introduced in #52032, is a
regression from previous releases where the pipeline was allowed, but
was no usable.

This change restores the previous behaviour (with a new warning).

x-pack/plugin/security/qa/security-disabled/build.gradle

@@ -0,0 +1,29 @@
+/*
+ * This QA project tests the security plugin when security is explicitlt disabled.
+ * It is intended to cover security functionality which is supposed to 
+ * function in a specific way even if security is disabled on the cluster
+ * For example: If a cluster has a pipeline with the set_security_user processor
+ *              defined, it should be not fail
+ */
+
+apply plugin: 'elasticsearch.testclusters'
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+  testCompile project(path: xpackModule('core'), configuration: 'default')
+  testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
+  testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+}
+
+testClusters.integTest {
+  testDistribution = 'DEFAULT'
+  numberOfNodes = 2
+
+  setting 'xpack.ilm.enabled', 'false'
+  setting 'xpack.ml.enabled', 'false'
+  // We run with a trial license, but explicitly disable security.
+  // This means the security plugin is loaded and all feature are permitted, but they are not enabled
+  setting 'xpack.license.self_generated.type', 'trial'
+  setting 'xpack.security.enabled', 'false'
+}

x-pack/plugin/security/qa/security-disabled/src/test/java/org/elasticsearch/xpack/security/SetSecurityUserProcessorWithSecurityDisabledIT.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.test.rest.ESRestTestCase;
+
+import static org.hamcrest.Matchers.containsString;
+
+/**
+ * Tests that it is possible to <em>define</em> a pipeline with the
+ * {@link org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor} on a cluster with security disabled, but it is not possible
+ * to use that pipeline for ingestion.
+ */
+public class SetSecurityUserProcessorWithSecurityDisabledIT extends ESRestTestCase {
+
+    public void testDefineAndUseProcessor() throws Exception {
+        final String pipeline = "pipeline-" + getTestName();
+        final String index = "index-" + getTestName();
+        {
+            final Request putPipeline = new Request("PUT", "/_ingest/pipeline/" + pipeline);
+            putPipeline.setJsonEntity("{" +
+                " \"description\": \"Test pipeline (" + getTestName() + ")\"," +
+                " \"processors\":[{" +
+                "  \"set_security_user\":{ \"field\": \"user\" }" +
+                " }]" +
+                "}");
+            final Response response = client().performRequest(putPipeline);
+            assertOK(response);
+        }
+
+        {
+            final Request ingest = new Request("PUT", "/" + index + "/_doc/1?pipeline=" + pipeline);
+            ingest.setJsonEntity("{\"field\":\"value\"}");
+            final ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(ingest));
+            final Response response = ex.getResponse();
+            assertThat(EntityUtils.toString(response.getEntity()),
+                containsString("Security (authentication) is not enabled on this cluster"));
+        }
+    }
+
+}

x-pack/plugin/security/qa/security-not-enabled/build.gradle

@@ -0,0 +1,28 @@
+/*
+ * This QA project tests the security plugin when security is not enabled.
+ * It is intended to cover security functionality which is supposed to 
+ * function in a specific way even if security is not enabled on the cluster
+ * For example: If a cluster has a pipeline with the set_security_user processor
+ *              defined, it should be not fail
+ */
+
+apply plugin: 'elasticsearch.testclusters'
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+  testCompile project(path: xpackModule('core'), configuration: 'default')
+  testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
+  testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+}
+
+testClusters.integTest {
+  testDistribution = 'DEFAULT'
+  numberOfNodes = 2
+
+  setting 'xpack.ilm.enabled', 'false'
+  setting 'xpack.ml.enabled', 'false'
+  // We run with a trial license, but do not enable security.
+  // This means the security plugin is loaded and all feature are permitted, but they are not enabled
+  setting 'xpack.license.self_generated.type', 'trial'
+}

x-pack/plugin/security/qa/security-not-enabled/src/test/java/org/elasticsearch/xpack/security/SetSecurityUserProcessorWithSecurityNotEnabledIT.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.test.rest.ESRestTestCase;
+
+import static org.hamcrest.Matchers.containsString;
+
+/**
+ * Tests that it is possible to <em>define</em> a pipeline with the
+ * {@link org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor} on a cluster where security is not enabled,
+ * but it is not possible to use that pipeline for ingestion.
+ */
+public class SetSecurityUserProcessorWithSecurityNotEnabledIT extends ESRestTestCase {
+
+    public void testDefineAndUseProcessor() throws Exception {
+        final String pipeline = "pipeline-" + getTestName();
+        final String index = "index-" + getTestName();
+        {
+            final Request putPipeline = new Request("PUT", "/_ingest/pipeline/" + pipeline);
+            putPipeline.setJsonEntity("{" +
+                " \"description\": \"Test pipeline (" + getTestName() + ")\"," +
+                " \"processors\":[{" +
+                "  \"set_security_user\":{ \"field\": \"user\" }" +
+                " }]" +
+                "}");
+            final Response response = client().performRequest(putPipeline);
+            assertOK(response);
+        }
+
+        {
+            final Request ingest = new Request("PUT", "/" + index + "/_doc/1?pipeline=" + pipeline);
+            ingest.setJsonEntity("{\"field\":\"value\"}");
+            final ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(ingest));
+            final Response response = ex.getResponse();
+            assertThat(EntityUtils.toString(response.getEntity()),
+                containsString("Security (authentication) is not enabled on this cluster"));
+        }
+    }
+
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -816,7 +816,8 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
 
     @Override
     public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
-        return Collections.singletonMap(SetSecurityUserProcessor.TYPE, new SetSecurityUserProcessor.Factory(securityContext::get));
+        return Collections.singletonMap(SetSecurityUserProcessor.TYPE,
+            new SetSecurityUserProcessor.Factory(securityContext::get, this::getLicenseState));
     }
 
     /**

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java

@@ -5,9 +5,12 @@
  */
 package org.elasticsearch.xpack.security.ingest;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.user.User;
@@ -34,27 +37,51 @@ public final class SetSecurityUserProcessor extends AbstractProcessor {
 
     public static final String TYPE = "set_security_user";
 
+    private final Logger logger = LogManager.getLogger();
+
     private final SecurityContext securityContext;
+    private final XPackLicenseState licenseState;
     private final String field;
     private final Set<Property> properties;
 
-    public
-    SetSecurityUserProcessor(String tag, SecurityContext securityContext, String field, Set<Property> properties) {
+    public SetSecurityUserProcessor(String tag, SecurityContext securityContext, XPackLicenseState licenseState, String field,
+                                    Set<Property> properties) {
         super(tag);
-        this.securityContext = Objects.requireNonNull(securityContext, "security context must be provided");
+        this.securityContext = securityContext;
+        this.licenseState = Objects.requireNonNull(licenseState, "license state cannot be null");
+        if (licenseState.isAuthAllowed() == false) {
+            logger.warn("Creating processor [{}] (tag [{}]) on field [{}] but authentication is not currently enabled on this cluster " +
+                " - this processor is likely to fail at runtime if it is used", TYPE, tag, field);
+        } else if (this.securityContext == null) {
+            throw new IllegalArgumentException("Authentication is allowed on this cluster state, but there is no security context");
+        }
         this.field = field;
         this.properties = properties;
     }
 
     @Override
     public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
-        Authentication authentication = securityContext.getAuthentication();
-        if (authentication == null) {
-            throw new IllegalStateException("No user authenticated, only use this processor via authenticated user");
+        Authentication authentication = null;
+        User user = null;
+        if (this.securityContext != null) {
+            authentication = securityContext.getAuthentication();
+            if (authentication != null) {
+                user = authentication.getUser();
+            }
         }
-        User user = authentication.getUser();
+
         if (user == null) {
-            throw new IllegalStateException("No user for authentication");
+            logger.debug(
+                "Failed to find active user. SecurityContext=[{}] Authentication=[{}] User=[{}]", securityContext, authentication, user);
+            if (licenseState.isAuthAllowed()) {
+                // This shouldn't happen. If authentication is allowed (and active), then there _should_ always be an authenticated user.
+                // If we ever see this error message, then one of our assumptions are wrong.
+                throw new IllegalStateException("There is no authenticated user - the [" + TYPE
+                    + "] processor requires an authenticated user");
+            } else {
+                throw new IllegalStateException("Security (authentication) is not enabled on this cluster, so there is no active user - " +
+                    "the [" + TYPE + "] processor cannot be used without security");
+            }
         }
 
         Object fieldValue = ingestDocument.getFieldValue(field, Object.class, true);
@@ -155,9 +182,11 @@ Set<Property> getProperties() {
     public static final class Factory implements Processor.Factory {
 
         private final Supplier<SecurityContext> securityContext;
+        private final Supplier<XPackLicenseState> licenseState;
 
-        public Factory(Supplier<SecurityContext> securityContext) {
+        public Factory(Supplier<SecurityContext> securityContext, Supplier<XPackLicenseState> licenseState) {
             this.securityContext = securityContext;
+            this.licenseState = licenseState;
         }
 
         @Override
@@ -174,7 +203,7 @@ public SetSecurityUserProcessor create(Map<String, Processor.Factory> processorF
             } else {
                 properties = EnumSet.allOf(Property.class);
             }
-            return new SetSecurityUserProcessor(tag, securityContext.get(), field, properties);
+            return new SetSecurityUserProcessor(tag, securityContext.get(), licenseState.get(), field, properties);
         }
     }
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGetApiKeyAction.java

@@ -59,6 +59,7 @@ public RestResponse buildResponse(GetApiKeyResponse getApiKeyResponse, XContentB
                         }
                         return new BytesRestResponse(RestStatus.OK, builder);
                     }
+
                 });
     }
 

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorFactoryTests.java

@@ -8,31 +8,36 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor.Property;
 import org.junit.Before;
+import org.mockito.Mockito;
 
 import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.elasticsearch.test.TestMatchers.throwableWithMessage;
-import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.mockito.Mockito.when;
 
 public class SetSecurityUserProcessorFactoryTests extends ESTestCase {
 
     private SecurityContext securityContext;
+    private XPackLicenseState licenseState;
 
     @Before
     public void setupContext() {
         securityContext = new SecurityContext(Settings.EMPTY, new ThreadContext(Settings.EMPTY));
+        licenseState = Mockito.mock(XPackLicenseState.class);
+        when(licenseState.isAuthAllowed()).thenReturn(true);
     }
 
     public void testProcessor() throws Exception {
-        SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(() -> securityContext);
+        SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(() -> securityContext, () -> licenseState);
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         SetSecurityUserProcessor processor = factory.create(null, "_tag", config);
@@ -41,7 +46,7 @@ public void testProcessor() throws Exception {
     }
 
     public void testProcessor_noField() throws Exception {
-        SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(() -> securityContext);
+        SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(() -> securityContext, () -> licenseState);
         Map<String, Object> config = new HashMap<>();
         ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", config));
         assertThat(e.getMetadata("es.property_name").get(0), equalTo("field"));
@@ -50,7 +55,7 @@ public void testProcessor_noField() throws Exception {
     }
 
     public void testProcessor_validProperties() throws Exception {
-        SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(() -> securityContext);
+        SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(() -> securityContext, () -> licenseState);
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("properties", Arrays.asList(Property.USERNAME.name(), Property.ROLES.name()));
@@ -60,7 +65,7 @@ public void testProcessor_validProperties() throws Exception {
     }
 
     public void testProcessor_invalidProperties() throws Exception {
-        SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(() -> securityContext);
+        SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(() -> securityContext, () -> licenseState);
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("properties", Arrays.asList("invalid"));
@@ -70,12 +75,13 @@ public void testProcessor_invalidProperties() throws Exception {
         assertThat(e.getMetadata("es.processor_tag").get(0), equalTo("_tag"));
     }
 
-    public void testNullSecurityContextThrowsException() throws Exception {
-        SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(() -> null);
+    public void testCanConstructorProcessorWithoutSecurityEnabled() throws Exception {
+        when(licenseState.isAuthAllowed()).thenReturn(false);
+        SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(() -> null, () -> licenseState);
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        NullPointerException e = expectThrows(NullPointerException.class, () -> factory.create(null, "_tag", config));
-        assertThat(e, throwableWithMessage(containsString("security context")));
+        final SetSecurityUserProcessor processor = factory.create(null, "_tag", config);
+        assertThat(processor, notNullValue());
     }
 
 }