Allow native users/roles to be disabled via setting (#98654)

This adds 2 new internal settings that can be used to disable security management APIs without disabling security.
- xpack.security.authc.native_users.enabled (default true) controls native user management
- xpack.security.authc.native_roles.enabled (default true) controls native role management

Neither setting is registered to be available in external config - both of these must be managed by a separate plugin.

If native user management is disabled then:
- Native user APIs (/_security/user/) return 401 (gone)
- The default_native realm is not registered
- It is not possible to configure a native realm (the factory is disabled)

If native role management is disabled then:
- Native role APIs (/_security/role/) other than GET return 401 (gone)
- The native roles store never attempts to resolve roles.

The disabling of native user APIs may affect management of reserved users as well, and it is intended that the reserve realm be disabled (xpack.security.authc.reserved_realm.enabled) if native users are disabled.

docs/changelog/98654.yaml

@@ -0,0 +1,5 @@
+pr: 98654
+summary: Allow native users/roles to be disabled via setting
+area: Authentication
+type: enhancement
+issues: []

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/esnative/NativeRealmSettings.java

@@ -17,6 +17,19 @@ public final class NativeRealmSettings {
     public static final String TYPE = "native";
     public static final String DEFAULT_NAME = "default_native";
 
+    /**
+     * This setting is never registered by the security plugin - in order to disable the native user APIs
+     * another plugin must register it as a boolean setting and cause it to be set to `false`.
+     *
+     * If this setting is set to <code>false</code> then
+     * <ul>
+     *     <li>the Rest APIs for native user management are disabled.</li>
+     *     <li>the default native realm will <em>not</em> be automatically configured.</li>
+     *     <li>it is not possible to configure a native realm.</li>
+     * </ul>
+     */
+    public static final String NATIVE_USERS_ENABLED = "xpack.security.authc.native_users.enabled";
+
     private NativeRealmSettings() {}
 
     /**

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalRealms.java

@@ -143,11 +143,7 @@ public static Map<String, Realm.Factory> getFactories(
             config -> new FileRealm(config, resourceWatcherService, threadPool),
             // native realm
             NativeRealmSettings.TYPE,
-            config -> {
-                final NativeRealm nativeRealm = new NativeRealm(config, nativeUsersStore, threadPool);
-                securityIndex.addStateListener(nativeRealm::onSecurityIndexStateChange);
-                return nativeRealm;
-            },
+            config -> buildNativeRealm(threadPool, settings, nativeUsersStore, securityIndex, config),
             // active directory realm
             LdapRealmSettings.AD_TYPE,
             config -> new LdapRealm(config, sslService, resourceWatcherService, nativeRoleMappingStore, threadPool),
@@ -172,6 +168,27 @@ public static Map<String, Realm.Factory> getFactories(
         );
     }
 
+    private static NativeRealm buildNativeRealm(
+        ThreadPool threadPool,
+        Settings settings,
+        NativeUsersStore nativeUsersStore,
+        SecurityIndexManager securityIndex,
+        RealmConfig config
+    ) {
+        if (settings.getAsBoolean(NativeRealmSettings.NATIVE_USERS_ENABLED, true) == false) {
+            throw new IllegalArgumentException(
+                "Cannot configure a ["
+                    + NativeRealmSettings.TYPE
+                    + "] realm when ["
+                    + NativeRealmSettings.NATIVE_USERS_ENABLED
+                    + "] is false"
+            );
+        }
+        final NativeRealm nativeRealm = new NativeRealm(config, nativeUsersStore, threadPool);
+        securityIndex.addStateListener(nativeRealm::onSecurityIndexStateChange);
+        return nativeRealm;
+    }
+
     private InternalRealms() {}
 
     public static List<BootstrapCheck> getBootstrapChecks(final Settings globalSettings, final Environment env) {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java

@@ -385,23 +385,26 @@ private void maybeAddBasicRealms(List<Realm> realms, List<RealmConfig> realmConf
         final Set<String> realmTypes = realms.stream().map(Realm::type).collect(Collectors.toUnmodifiableSet());
         // Add native realm first so that file realm will be added before it
         if (false == disabledBasicRealmTypes.contains(NativeRealmSettings.TYPE) && false == realmTypes.contains(NativeRealmSettings.TYPE)) {
+            boolean enabled = settings.getAsBoolean(NativeRealmSettings.NATIVE_USERS_ENABLED, true);
             ensureRealmNameIsAvailable(realms, NativeRealmSettings.DEFAULT_NAME);
             var nativeRealmId = new RealmConfig.RealmIdentifier(NativeRealmSettings.TYPE, NativeRealmSettings.DEFAULT_NAME);
             var realmConfig = new RealmConfig(
                 nativeRealmId,
-                ensureOrderSetting(settings, nativeRealmId, Integer.MIN_VALUE),
+                buildSettingsforDefaultRealm(settings, nativeRealmId, Integer.MIN_VALUE, enabled),
                 env,
                 threadContext
             );
             realmConfigs.add(realmConfig);
-            realms.add(0, factories.get(NativeRealmSettings.TYPE).create(realmConfig));
+            if (enabled) {
+                realms.add(0, factories.get(NativeRealmSettings.TYPE).create(realmConfig));
+            }
         }
         if (false == disabledBasicRealmTypes.contains(FileRealmSettings.TYPE) && false == realmTypes.contains(FileRealmSettings.TYPE)) {
             ensureRealmNameIsAvailable(realms, FileRealmSettings.DEFAULT_NAME);
             var fileRealmId = new RealmConfig.RealmIdentifier(FileRealmSettings.TYPE, FileRealmSettings.DEFAULT_NAME);
             var realmConfig = new RealmConfig(
                 fileRealmId,
-                ensureOrderSetting(settings, fileRealmId, Integer.MIN_VALUE),
+                buildSettingsforDefaultRealm(settings, fileRealmId, Integer.MIN_VALUE, true),
                 env,
                 threadContext
             );
@@ -433,9 +436,18 @@ private void ensureRealmNameIsAvailable(List<Realm> realms, String realmName) {
         }
     }
 
-    private static Settings ensureOrderSetting(Settings settings, RealmConfig.RealmIdentifier realmIdentifier, int order) {
-        String orderSettingKey = RealmSettings.realmSettingPrefix(realmIdentifier) + "order";
-        return Settings.builder().put(settings).put(orderSettingKey, order).build();
+    private static Settings buildSettingsforDefaultRealm(
+        Settings settings,
+        RealmConfig.RealmIdentifier realmIdentifier,
+        int order,
+        boolean enabled
+    ) {
+        final String prefix = RealmSettings.realmSettingPrefix(realmIdentifier);
+        final Settings.Builder builder = Settings.builder().put(settings).put(prefix + "order", order);
+        if (enabled == false) {
+            builder.put(prefix + "enabled", false);
+        }
+        return builder.build();
     }
 
     private static void checkUniqueOrders(Map<Integer, Set<String>> orderToRealmName) {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java

@@ -82,11 +82,24 @@
  */
 public class NativeRolesStore implements BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>> {
 
+    /**
+     * This setting is never registered by the security plugin - in order to disable the native role APIs
+     * another plugin must register it as a boolean setting and cause it to be set to `false`.
+     *
+     * If this setting is set to <code>false</code> then
+     * <ul>
+     *     <li>the Rest APIs for native role management are disabled.</li>
+     *     <li>The native roles store will not resolve any roles</li>
+     * </ul>
+     */
+    public static final String NATIVE_ROLES_ENABLED = "xpack.security.authc.native_roles.enabled";
+
     private static final Logger logger = LogManager.getLogger(NativeRolesStore.class);
 
     private final Settings settings;
     private final Client client;
     private final XPackLicenseState licenseState;
+    private final boolean enabled;
 
     private final SecurityIndexManager securityIndex;
 
@@ -104,6 +117,7 @@ public NativeRolesStore(
         this.licenseState = licenseState;
         this.securityIndex = securityIndex;
         this.clusterService = clusterService;
+        this.enabled = settings.getAsBoolean(NATIVE_ROLES_ENABLED, true);
     }
 
     @Override
@@ -115,6 +129,11 @@ public void accept(Set<String> names, ActionListener<RoleRetrievalResult> listen
      * Retrieve a list of roles, if rolesToGet is null or empty, fetch all roles
      */
     public void getRoleDescriptors(Set<String> names, final ActionListener<RoleRetrievalResult> listener) {
+        if (enabled == false) {
+            listener.onResponse(RoleRetrievalResult.success(Set.of()));
+            return;
+        }
+
         final SecurityIndexManager frozenSecurityIndex = this.securityIndex.freeze();
         if (frozenSecurityIndex.indexExists() == false) {
             // TODO remove this short circuiting and fix tests that fail without this!
@@ -185,6 +204,11 @@ public void getRoleDescriptors(Set<String> names, final ActionListener<RoleRetri
     }
 
     public void deleteRole(final DeleteRoleRequest deleteRoleRequest, final ActionListener<Boolean> listener) {
+        if (enabled == false) {
+            listener.onFailure(new IllegalStateException("Native role management is disabled"));
+            return;
+        }
+
         final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze();
         if (frozenSecurityIndex.indexExists() == false) {
             listener.onResponse(false);
@@ -221,6 +245,11 @@ public void onFailure(Exception e) {
     }
 
     public void putRole(final PutRoleRequest request, final RoleDescriptor role, final ActionListener<Boolean> listener) {
+        if (enabled == false) {
+            listener.onFailure(new IllegalStateException("Native role management is disabled"));
+            return;
+        }
+
         if (role.isUsingDocumentOrFieldLevelSecurity() && DOCUMENT_LEVEL_SECURITY_FEATURE.checkWithoutTracking(licenseState) == false) {
             listener.onFailure(LicenseUtils.newComplianceException("field and document level security"));
         } else if (role.hasRemoteIndicesPrivileges()

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestDelegatePkiAuthenticationAction.java

@@ -50,7 +50,7 @@ public List<Route> routes() {
     }
 
     @Override
-    protected Exception innerCheckFeatureAvailable() {
+    protected Exception innerCheckFeatureAvailable(RestRequest request) {
         if (Security.PKI_REALM_FEATURE.checkWithoutTracking(licenseState)) {
             return null;
         } else {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/SecurityBaseRestHandler.java

@@ -35,7 +35,7 @@ protected SecurityBaseRestHandler(Settings settings, XPackLicenseState licenseSt
     }
 
     /**
-     * Calls the {@link #checkFeatureAvailable()} method to check whether the feature is available based
+     * Calls the {@link #checkFeatureAvailable(RestRequest)} method to check whether the feature is available based
      * on settings and license state. If allowed, the result from
      * {@link #innerPrepareRequest(RestRequest, NodeClient)} is returned, otherwise a default error
      * response will be returned indicating that security is not licensed.
@@ -44,7 +44,7 @@ protected SecurityBaseRestHandler(Settings settings, XPackLicenseState licenseSt
      * trip the unused parameters check
      */
     protected final RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        final Exception failedFeature = checkFeatureAvailable();
+        final Exception failedFeature = checkFeatureAvailable(request);
         if (failedFeature == null) {
             return innerPrepareRequest(request, client);
         } else {
@@ -58,31 +58,31 @@ protected final RestChannelConsumer prepareRequest(RestRequest request, NodeClie
      * Check whether the given request is allowed within the current license state and setup,
      * and return the name of any unlicensed feature.
      * By default this returns an exception if security is not enabled.
-     * Sub-classes can override {@link #innerCheckFeatureAvailable()} if they have additional requirements.
+     * Sub-classes can override {@link #innerCheckFeatureAvailable(RestRequest)} if they have additional requirements.
      *
      * @return {@code null} if all required features are available, otherwise an exception to be
      * sent to the requester
      */
-    public final Exception checkFeatureAvailable() {
+    public final Exception checkFeatureAvailable(RestRequest request) {
         if (XPackSettings.SECURITY_ENABLED.get(settings) == false) {
             return new IllegalStateException("Security is not enabled but a security rest handler is registered");
         } else {
-            return innerCheckFeatureAvailable();
+            return innerCheckFeatureAvailable(request);
         }
     }
 
     /**
      * Implementers should implement this method when sub-classes have additional license requirements.
      */
-    protected Exception innerCheckFeatureAvailable() {
+    protected Exception innerCheckFeatureAvailable(RestRequest request) {
         return null;
     }
 
     /**
      * Implementers should implement this method as they normally would for
      * {@link BaseRestHandler#prepareRequest(RestRequest, NodeClient)} and ensure that all request
      * parameters are consumed prior to returning a value. This method is executed only if the
-     * check from {@link #checkFeatureAvailable()} passes.
+     * check from {@link #checkFeatureAvailable(RestRequest)} passes.
      */
     protected abstract RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException;
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/ApiKeyBaseRestHandler.java

@@ -9,6 +9,7 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 import org.elasticsearch.xpack.security.support.FeatureNotEnabledException;
@@ -19,7 +20,7 @@ abstract class ApiKeyBaseRestHandler extends SecurityBaseRestHandler {
     }
 
     @Override
-    protected Exception innerCheckFeatureAvailable() {
+    protected Exception innerCheckFeatureAvailable(RestRequest request) {
         if (XPackSettings.API_KEY_SERVICE_ENABLED_SETTING.get(settings) == false) {
             return new FeatureNotEnabledException(FeatureNotEnabledException.Feature.API_KEY_SERVICE, "api keys are not enabled");
         } else {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateCrossClusterApiKeyAction.java

@@ -85,7 +85,7 @@ protected RestChannelConsumer innerPrepareRequest(final RestRequest request, fin
     }
 
     @Override
-    protected Exception innerCheckFeatureAvailable() {
+    protected Exception innerCheckFeatureAvailable(RestRequest request) {
         if (ADVANCED_REMOTE_CLUSTER_SECURITY_FEATURE.checkWithoutTracking(licenseState)) {
             return null;
         } else {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateCrossClusterApiKeyAction.java

@@ -67,7 +67,7 @@ protected RestChannelConsumer innerPrepareRequest(final RestRequest request, fin
     }
 
     @Override
-    protected Exception innerCheckFeatureAvailable() {
+    protected Exception innerCheckFeatureAvailable(RestRequest request) {
         if (ADVANCED_REMOTE_CLUSTER_SECURITY_FEATURE.checkWithoutTracking(licenseState)) {
             return null;
         } else {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/enrollment/EnrollmentBaseRestHandler.java

@@ -10,6 +10,7 @@
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
@@ -24,7 +25,7 @@ public EnrollmentBaseRestHandler(Settings settings, XPackLicenseState licenseSta
     }
 
     @Override
-    protected Exception innerCheckFeatureAvailable() {
+    protected Exception innerCheckFeatureAvailable(RestRequest request) {
         if (XPackSettings.ENROLLMENT_ENABLED.get(settings) == false) {
             return new ElasticsearchSecurityException(
                 "Enrollment mode is not enabled. Set ["

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oauth2/TokenBaseRestHandler.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.xpack.security.Security;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
@@ -27,7 +28,7 @@ abstract class TokenBaseRestHandler extends SecurityBaseRestHandler {
     }
 
     @Override
-    protected Exception innerCheckFeatureAvailable() {
+    protected Exception innerCheckFeatureAvailable(RestRequest request) {
         if (Security.TOKEN_SERVICE_FEATURE.check(licenseState)) {
             return null;
         } else {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oidc/OpenIdConnectBaseRestHandler.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.xpack.core.security.authc.oidc.OpenIdConnectRealmSettings;
 import org.elasticsearch.xpack.security.authc.Realms;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
@@ -29,7 +30,7 @@ protected OpenIdConnectBaseRestHandler(Settings settings, XPackLicenseState lice
     }
 
     @Override
-    protected Exception innerCheckFeatureAvailable() {
+    protected Exception innerCheckFeatureAvailable(RestRequest request) {
         if (Realms.isRealmTypeAvailable(licenseState, OIDC_REALM_TYPE)) {
             return null;
         } else {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/profile/RestSuggestProfilesAction.java

@@ -100,7 +100,7 @@ protected RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClien
     }
 
     @Override
-    protected Exception innerCheckFeatureAvailable() {
+    protected Exception innerCheckFeatureAvailable(RestRequest request) {
         if (Security.USER_PROFILE_COLLABORATION_FEATURE.check(licenseState)) {
             return null;
         } else {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/NativeRoleBaseRestHandler.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.rest.action.role;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
+import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
+
+abstract class NativeRoleBaseRestHandler extends SecurityBaseRestHandler {
+
+    private final Logger logger = LogManager.getLogger(NativeRoleBaseRestHandler.class);
+
+    NativeRoleBaseRestHandler(Settings settings, XPackLicenseState licenseState) {
+        super(settings, licenseState);
+    }
+
+    @Override
+    protected Exception innerCheckFeatureAvailable(RestRequest request) {
+        final Boolean nativeRolesEnabled = settings.getAsBoolean(NativeRolesStore.NATIVE_ROLES_ENABLED, true);
+        if (nativeRolesEnabled == false) {
+            logger.debug(
+                "Attempt to call [{} {}] but [{}] is [{}]",
+                request.method(),
+                request.rawPath(),
+                NativeRolesStore.NATIVE_ROLES_ENABLED,
+                settings.get(NativeRolesStore.NATIVE_ROLES_ENABLED)
+            );
+            return new ElasticsearchStatusException(
+                "Native role management is not enabled in this Elasticsearch instance",
+                RestStatus.GONE
+            );
+        } else {
+            return null;
+        }
+
+    }
+}