[7.x] Mirror privileges over data streams to their backing indices (#…

…58991)

rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml

@@ -45,15 +45,6 @@
       indices.delete:
         index: logs-foobar
 
-  - do:
-      indices.create:
-        index: logs-foobarbaz
-
-  - do:
-      catch: bad_request
-      indices.close:
-        index: logs-*
-
   - do:
       indices.delete_data_stream:
         name: logs-foobar

server/src/main/java/org/elasticsearch/action/IndicesRequest.java

@@ -40,6 +40,15 @@ public interface IndicesRequest {
      */
     IndicesOptions indicesOptions();
 
+    /**
+     * Determines whether the request should be applied to data streams. When {@code false}, none of the names or
+     * wildcard expressions in {@link #indices} should be applied to or expanded to any data streams. All layers
+     * involved in the request's fulfillment including security, name resolution, etc., should respect this flag.
+     */
+    default boolean includeDataStreams() {
+        return false;
+    }
+
     interface Replaceable extends IndicesRequest {
         /**
          * Sets the indices that the action relates to.

server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java

@@ -112,6 +112,11 @@ public ClusterSearchShardsRequest indicesOptions(IndicesOptions indicesOptions)
         return this;
     }
 
+    @Override
+    public boolean includeDataStreams() {
+        return true;
+    }
+
     /**
      * A comma separated list of routing values to control the shards the search will be executed on.
      */

server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java

@@ -290,4 +290,9 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         flags.writeTo(out);
     }
+
+    @Override
+    public boolean includeDataStreams() {
+        return true;
+    }
 }

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java

@@ -157,6 +157,11 @@ public IndicesOptions indicesOptions() {
         return indicesOptions;
     }
 
+    @Override
+    public boolean includeDataStreams() {
+        return true;
+    }
+
     public boolean includeUnmapped() {
         return includeUnmapped;
     }

server/src/main/java/org/elasticsearch/action/search/SearchRequest.java

@@ -356,6 +356,11 @@ public SearchRequest indicesOptions(IndicesOptions indicesOptions) {
         return this;
     }
 
+    @Override
+    public boolean includeDataStreams() {
+        return true;
+    }
+
     /**
      * Returns whether network round-trips should be minimized when executing cross-cluster search requests.
      * Defaults to <code>true</code>.

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java

@@ -194,7 +194,7 @@ public Automaton allowedActionsMatcher(String index) {
      * Authorizes the provided action against the provided indices, given the current cluster metadata
      */
     public Map<String, IndicesAccessControl.IndexAccessControl> authorize(String action, Set<String> requestedIndicesOrAliases,
-                                                                          Map<String, IndexAbstraction> allAliasesAndIndices,
+                                                                          Map<String, IndexAbstraction> lookup,
                                                                           FieldPermissionsCache fieldPermissionsCache) {
         // now... every index that is associated with the request, must be granted
         // by at least one indices permission group
@@ -205,15 +205,20 @@ public Map<String, IndicesAccessControl.IndexAccessControl> authorize(String act
         for (String indexOrAlias : requestedIndicesOrAliases) {
             boolean granted = false;
             Set<String> concreteIndices = new HashSet<>();
-            IndexAbstraction indexAbstraction = allAliasesAndIndices.get(indexOrAlias);
+            IndexAbstraction indexAbstraction = lookup.get(indexOrAlias);
             if (indexAbstraction != null) {
                 for (IndexMetadata indexMetadata : indexAbstraction.getIndices()) {
                     concreteIndices.add(indexMetadata.getIndex().getName());
                 }
             }
 
             for (Group group : groups) {
-                if (group.check(action, indexOrAlias)) {
+                // check for privilege granted directly on the requested index/alias
+                if (group.check(action, indexOrAlias) ||
+                    // check for privilege granted on parent data stream if a backing index
+                    (indexAbstraction != null && indexAbstraction.getType() == IndexAbstraction.Type.CONCRETE_INDEX &&
+                        indexAbstraction.getParentDataStream() != null &&
+                        group.check(action, indexAbstraction.getParentDataStream().getName()))) {
                     granted = true;
                     for (String index : concreteIndices) {
                         Set<FieldPermissions> fieldPermissions = fieldPermissionsByIndex.computeIfAbsent(index, (k) -> new HashSet<>());

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java

@@ -137,7 +137,7 @@ ResolvedIndices resolveIndicesAndAliases(IndicesRequest indicesRequest, Metadata
             if (IndexNameExpressionResolver.isAllIndices(indicesList(indicesRequest.indices()))) {
                 if (replaceWildcards) {
                     for (String authorizedIndex : authorizedIndices) {
-                        if (isIndexVisible("*", authorizedIndex, indicesOptions, metadata)) {
+                        if (isIndexVisible("*", authorizedIndex, indicesOptions, metadata, indicesRequest.includeDataStreams())) {
                             resolvedIndicesBuilder.addLocal(authorizedIndex);
                         }
                     }
@@ -152,7 +152,7 @@ ResolvedIndices resolveIndicesAndAliases(IndicesRequest indicesRequest, Metadata
                     split = new ResolvedIndices(Arrays.asList(indicesRequest.indices()), Collections.emptyList());
                 }
                 List<String> replaced = replaceWildcardsWithAuthorizedIndices(split.getLocal(), indicesOptions, metadata,
-                        authorizedIndices, replaceWildcards);
+                        authorizedIndices, replaceWildcards, indicesRequest.includeDataStreams());
                 if (indicesOptions.ignoreUnavailable()) {
                     //out of all the explicit names (expanded from wildcards and original ones that were left untouched)
                     //remove all the ones that the current user is not authorized for and ignore them
@@ -344,7 +344,8 @@ private boolean containsWildcards(IndicesRequest indicesRequest) {
 
     //TODO Investigate reusing code from vanilla es to resolve index names and wildcards
     private List<String> replaceWildcardsWithAuthorizedIndices(Iterable<String> indices, IndicesOptions indicesOptions, Metadata metadata,
-                                                               List<String> authorizedIndices, boolean replaceWildcards) {
+                                                               List<String> authorizedIndices, boolean replaceWildcards,
+                                                               boolean includeDataStreams) {
         //the order matters when it comes to exclusions
         List<String> finalIndices = new ArrayList<>();
         boolean wildcardSeen = false;
@@ -366,7 +367,7 @@ private List<String> replaceWildcardsWithAuthorizedIndices(Iterable<String> indi
                     // continue
                     aliasOrIndex = dateMathName;
                 } else if (authorizedIndices.contains(dateMathName) &&
-                    isIndexVisible(aliasOrIndex, dateMathName, indicesOptions, metadata, true)) {
+                    isIndexVisible(aliasOrIndex, dateMathName, indicesOptions, metadata, includeDataStreams, true)) {
                     if (minus) {
                         finalIndices.remove(dateMathName);
                     } else {
@@ -384,7 +385,7 @@ private List<String> replaceWildcardsWithAuthorizedIndices(Iterable<String> indi
                 Set<String> resolvedIndices = new HashSet<>();
                 for (String authorizedIndex : authorizedIndices) {
                     if (Regex.simpleMatch(aliasOrIndex, authorizedIndex) &&
-                        isIndexVisible(aliasOrIndex, authorizedIndex, indicesOptions, metadata)) {
+                        isIndexVisible(aliasOrIndex, authorizedIndex, indicesOptions, metadata, includeDataStreams)) {
                         resolvedIndices.add(authorizedIndex);
                     }
                 }
@@ -419,13 +420,17 @@ private List<String> replaceWildcardsWithAuthorizedIndices(Iterable<String> indi
         return finalIndices;
     }
 
-    private static boolean isIndexVisible(String expression, String index, IndicesOptions indicesOptions, Metadata metadata) {
-        return isIndexVisible(expression, index, indicesOptions, metadata, false);
+    private static boolean isIndexVisible(String expression, String index, IndicesOptions indicesOptions, Metadata metadata,
+                                          boolean includeDataStreams) {
+        return isIndexVisible(expression, index, indicesOptions, metadata, includeDataStreams, false);
     }
 
     private static boolean isIndexVisible(String expression, String index, IndicesOptions indicesOptions, Metadata metadata,
-                                          boolean dateMathExpression) {
+                                          boolean includeDataStreams, boolean dateMathExpression) {
         IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(index);
+        if (indexAbstraction == null) {
+            throw new IllegalStateException("could not resolve index abstraction [" + index + "]");
+        }
         final boolean isHidden = indexAbstraction.isHidden();
         if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS) {
             //it's an alias, ignore expandWildcardsOpen and expandWildcardsClosed.
@@ -440,12 +445,7 @@ private static boolean isIndexVisible(String expression, String index, IndicesOp
             }
         }
         if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) {
-            // If indicesOptions.includeDataStreams() returns false then we fail later in IndexNameExpressionResolver.
-            if (isHidden == false || indicesOptions.expandWildcardsHidden()) {
-                return true;
-            } else {
-                return false;
-            }
+            return includeDataStreams;
         }
         assert indexAbstraction.getIndices().size() == 1 : "concrete index must point to a single index";
         IndexMetadata indexMetadata = indexAbstraction.getIndices().get(0);

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java

@@ -86,6 +86,7 @@
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.Strings.arrayToCommaDelimitedString;
 import static org.elasticsearch.xpack.security.action.user.TransportHasPrivilegesAction.getApplicationNames;
@@ -335,7 +336,7 @@ public void loadAuthorizedIndices(RequestInfo requestInfo, AuthorizationInfo aut
                                       Map<String, IndexAbstraction> indicesLookup, ActionListener<List<String>> listener) {
         if (authorizationInfo instanceof RBACAuthorizationInfo) {
             final Role role = ((RBACAuthorizationInfo) authorizationInfo).getRole();
-            listener.onResponse(resolveAuthorizedIndicesFromRole(role, requestInfo.getAction(), indicesLookup));
+            listener.onResponse(resolveAuthorizedIndicesFromRole(role, requestInfo, indicesLookup));
         } else {
             listener.onFailure(
                 new IllegalArgumentException("unsupported authorization info:" + authorizationInfo.getClass().getSimpleName()));
@@ -492,18 +493,29 @@ GetUserPrivilegesResponse buildUserPrivilegesResponseObject(Role userRole) {
         return new GetUserPrivilegesResponse(cluster, conditionalCluster, indices, application, runAs);
     }
 
-    static List<String> resolveAuthorizedIndicesFromRole(Role role, String action, Map<String, IndexAbstraction> aliasAndIndexLookup) {
-        Predicate<String> predicate = role.allowedIndicesMatcher(action);
+    static List<String> resolveAuthorizedIndicesFromRole(Role role, RequestInfo requestInfo, Map<String, IndexAbstraction> lookup) {
+        Predicate<String> predicate = role.allowedIndicesMatcher(requestInfo.getAction());
 
-        List<String> indicesAndAliases = new ArrayList<>();
+        // do not include data streams for actions that do not operate on data streams
+        TransportRequest request = requestInfo.getRequest();
+        boolean includeDataStreams = (request instanceof IndicesRequest) && ((IndicesRequest) request).includeDataStreams();
+
+        Set<String> indicesAndAliases = new HashSet<>();
         // TODO: can this be done smarter? I think there are usually more indices/aliases in the cluster then indices defined a roles?
-        for (Map.Entry<String, IndexAbstraction> entry : aliasAndIndexLookup.entrySet()) {
-            String aliasOrIndex = entry.getKey();
-            if (predicate.test(aliasOrIndex)) {
-                indicesAndAliases.add(aliasOrIndex);
+        for (Map.Entry<String, IndexAbstraction> entry : lookup.entrySet()) {
+            String indexAbstraction = entry.getKey();
+            if (predicate.test(indexAbstraction)) {
+                if (entry.getValue().getType() != IndexAbstraction.Type.DATA_STREAM) {
+                    indicesAndAliases.add(indexAbstraction);
+                } else if (includeDataStreams) {
+                    // add data stream and its backing indices for any authorized data streams
+                    indicesAndAliases.addAll(entry.getValue().getIndices().stream()
+                        .map(i -> i.getIndex().getName()).collect(Collectors.toList()));
+                    indicesAndAliases.add(indexAbstraction);
+                }
             }
         }
-        return Collections.unmodifiableList(indicesAndAliases);
+        return Collections.unmodifiableList(new ArrayList<>(indicesAndAliases));
     }
 
     private void buildIndicesAccessControl(Authentication authentication, String action,

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java

@@ -14,6 +14,8 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
@@ -36,7 +38,7 @@ public class AuthorizedIndicesTests extends ESTestCase {
 
     public void testAuthorizedIndicesUserWithoutRoles() {
         List<String> authorizedIndices =
-            RBACEngine.resolveAuthorizedIndicesFromRole(Role.EMPTY, "", Metadata.EMPTY_METADATA.getIndicesLookup());
+            RBACEngine.resolveAuthorizedIndicesFromRole(Role.EMPTY, getRequestInfo(""), Metadata.EMPTY_METADATA.getIndicesLookup());
         assertTrue(authorizedIndices.isEmpty());
     }
 
@@ -72,7 +74,7 @@ public void testAuthorizedIndicesUserWithSomeRoles() {
         CompositeRolesStore.buildRoleFromDescriptors(descriptors, new FieldPermissionsCache(Settings.EMPTY), null, future);
         Role roles = future.actionGet();
         List<String> list =
-            RBACEngine.resolveAuthorizedIndicesFromRole(roles, SearchAction.NAME, metadata.getIndicesLookup());
+            RBACEngine.resolveAuthorizedIndicesFromRole(roles, getRequestInfo(SearchAction.NAME), metadata.getIndicesLookup());
         assertThat(list, containsInAnyOrder("a1", "a2", "aaaaaa", "b", "ab"));
         assertFalse(list.contains("bbbbb"));
         assertFalse(list.contains("ba"));
@@ -82,16 +84,18 @@ public void testAuthorizedIndicesUserWithSomeRoles() {
 
     public void testAuthorizedIndicesUserWithSomeRolesEmptyMetadata() {
         Role role = Role.builder("role").add(IndexPrivilege.ALL, "*").build();
-        List<String> authorizedIndices =
-            RBACEngine.resolveAuthorizedIndicesFromRole(role, SearchAction.NAME, Metadata.EMPTY_METADATA.getIndicesLookup());
+        List<String> authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(role, getRequestInfo(SearchAction.NAME),
+            Metadata.EMPTY_METADATA.getIndicesLookup());
         assertTrue(authorizedIndices.isEmpty());
     }
 
     public void testSecurityIndicesAreRemovedFromRegularUser() {
-        Role role = Role.builder("user_role").add(IndexPrivilege.ALL, "*").cluster(Collections.singleton("all"), Collections.emptySet())
+        Role role = Role.builder("user_role").add(IndexPrivilege.ALL, "*").cluster(
+            org.elasticsearch.common.collect.Set.of("all"),
+            org.elasticsearch.common.collect.Set.of())
             .build();
-        List<String> authorizedIndices =
-            RBACEngine.resolveAuthorizedIndicesFromRole(role, SearchAction.NAME, Metadata.EMPTY_METADATA.getIndicesLookup());
+        List<String> authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(role, getRequestInfo(SearchAction.NAME),
+            Metadata.EMPTY_METADATA.getIndicesLookup());
         assertTrue(authorizedIndices.isEmpty());
     }
 
@@ -116,7 +120,7 @@ public void testSecurityIndicesAreRestrictedForDefaultRole() {
                 .build();
 
         List<String> authorizedIndices =
-            RBACEngine.resolveAuthorizedIndicesFromRole(role, SearchAction.NAME, metadata.getIndicesLookup());
+            RBACEngine.resolveAuthorizedIndicesFromRole(role, getRequestInfo(SearchAction.NAME), metadata.getIndicesLookup());
         assertThat(authorizedIndices, containsInAnyOrder("an-index", "another-index"));
         assertThat(authorizedIndices, not(contains(internalSecurityIndex)));
         assertThat(authorizedIndices, not(contains(RestrictedIndicesNames.SECURITY_MAIN_ALIAS)));
@@ -142,13 +146,21 @@ public void testSecurityIndicesAreNotRemovedFromUnrestrictedRole() {
                 .build();
 
         List<String> authorizedIndices =
-            RBACEngine.resolveAuthorizedIndicesFromRole(role, SearchAction.NAME, metadata.getIndicesLookup());
+            RBACEngine.resolveAuthorizedIndicesFromRole(role, getRequestInfo(SearchAction.NAME), metadata.getIndicesLookup());
         assertThat(authorizedIndices, containsInAnyOrder(
             "an-index", "another-index", RestrictedIndicesNames.SECURITY_MAIN_ALIAS, internalSecurityIndex));
 
         List<String> authorizedIndicesSuperUser =
-            RBACEngine.resolveAuthorizedIndicesFromRole(role, SearchAction.NAME, metadata.getIndicesLookup());
+            RBACEngine.resolveAuthorizedIndicesFromRole(role, getRequestInfo(SearchAction.NAME), metadata.getIndicesLookup());
         assertThat(authorizedIndicesSuperUser, containsInAnyOrder(
             "an-index", "another-index", RestrictedIndicesNames.SECURITY_MAIN_ALIAS, internalSecurityIndex));
     }
+
+    public static AuthorizationEngine.RequestInfo getRequestInfo(String action) {
+        return getRequestInfo(TransportRequest.Empty.INSTANCE, action);
+    }
+
+    public static AuthorizationEngine.RequestInfo getRequestInfo(TransportRequest request, String action) {
+        return new AuthorizationEngine.RequestInfo(null, request, action);
+    }
 }