From 5bbfa24760c5248b9b5eafc450f8f7759a86acbe Mon Sep 17 00:00:00 2001 From: Fabian Morgan Date: Mon, 30 Mar 2026 18:02:34 -0700 Subject: [PATCH 1/3] handle latent inconsistencies in s3 api acl checks --- .../src/main/proto/OmClientProtocol.proto | 8 ++ .../om/ratis/OzoneManagerStateMachine.java | 28 +++++++ .../ozone/om/request/OMClientRequest.java | 43 ++++++++++- .../bucket/acl/OMBucketSetAclRequest.java | 9 ++- .../ozone/security/STSSecurityUtil.java | 41 ++++++++++ .../TestOMClientRequestWithUserInfo.java | 66 ++++++++++++++++ .../ozone/security/TestSTSSecurityUtil.java | 75 +++++++++++++++++++ 7 files changed, 263 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 9bb0d801ee7b..c3a0dc26a9f8 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -2311,6 +2311,14 @@ message S3Authentication { // If present, indicates this request uses STS temporary credentials // and carries the base64-encoded session token for validation. optional string sessionToken = 4; + // The following fields are resolved from the STS session token by OM. + // They are used to enforce STS session policies during Ratis apply. + // They must be written or cleared by the OM leader when the token is validated. + optional string resolvedStsSessionPolicy = 5; + optional string resolvedStsRoleArn = 6; + optional string resolvedStsOriginalAccessKeyId = 7; + optional string resolvedStsTempAccessKeyId = 8; + optional string resolvedStsSecretKeyId = 9; } message RecoverLeaseRequest { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java index 09a530ab6cc7..d17ad9e62e46 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java @@ -55,6 +55,8 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import org.apache.hadoop.ozone.protocolPB.OzoneManagerRequestHandler; import org.apache.hadoop.ozone.protocolPB.RequestHandler; +import org.apache.hadoop.ozone.security.STSSecurityUtil; +import org.apache.hadoop.ozone.security.STSTokenIdentifier; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.concurrent.HadoopExecutors; @@ -617,7 +619,29 @@ public void close() { */ @VisibleForTesting OMResponse runCommand(OMRequest request, TermIndex termIndex) { + boolean isStsThreadLocalSet = false; try { + if (ozoneManager.isSecurityEnabled() && request.hasS3Authentication()) { + // STS token verification runs on the leader RPC path so we don't need to recheck here on the apply + // after the log is committed + STSSecurityUtil.ensureResolvedStsFieldsInvariants(request); + + final OzoneManagerProtocolProtos.S3Authentication s3Auth = request.getS3Authentication(); + if (s3Auth.hasSessionToken() && !s3Auth.getSessionToken().isEmpty()) { + // ThreadLocal carries session policy for OmMetadataReader + final STSTokenIdentifier rehydratedTokenIdentifier = new STSTokenIdentifier( + s3Auth.hasResolvedStsTempAccessKeyId() ? s3Auth.getResolvedStsTempAccessKeyId() : "", + s3Auth.hasResolvedStsOriginalAccessKeyId() ? s3Auth.getResolvedStsOriginalAccessKeyId() : "", + s3Auth.hasResolvedStsRoleArn() ? s3Auth.getResolvedStsRoleArn() : "", + java.time.Instant.MAX, // ensure it deterministically is not expired + "", // no secretAccessKey needed + s3Auth.hasResolvedStsSessionPolicy() ? s3Auth.getResolvedStsSessionPolicy() : "", + null // no encryption key needed + ); + OzoneManager.setStsTokenIdentifier(rehydratedTokenIdentifier); + isStsThreadLocalSet = true; + } + } ExecutionContext context = ExecutionContext.of(termIndex.getIndex(), termIndex); final OMClientResponse omClientResponse = handler.handleWriteRequest( request, context, ozoneManagerDoubleBuffer); @@ -636,6 +660,10 @@ OMResponse runCommand(OMRequest request, TermIndex termIndex) { // For any Runtime exceptions, terminate OM. String errorMessage = "Request " + request + " failed with exception"; ExitUtils.terminate(1, errorMessage, e, LOG); + } finally { + if (isStsThreadLocalSet) { + OzoneManager.setStsTokenIdentifier(null); + } } return null; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java index 0420eef2fd5d..1d4816008454 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java @@ -27,6 +27,7 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; +import java.util.UUID; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.ipc_.ProtobufRpcEngine; @@ -54,6 +55,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LayoutVersion; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.security.STSTokenIdentifier; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.apache.hadoop.ozone.security.acl.OzoneObj; import org.apache.hadoop.ozone.security.acl.OzoneObjInfo; @@ -112,15 +114,50 @@ public OMClientRequest(OMRequest omRequest) { */ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { - LayoutVersion layoutVersion = LayoutVersion.newBuilder() + final LayoutVersion layoutVersion = LayoutVersion.newBuilder() .setVersion(ozoneManager.getVersionManager().getMetadataLayoutVersion()) .build(); - omRequest = getOmRequest().toBuilder() + + final OMRequest.Builder requestBuilder = getOmRequest().toBuilder() .setUserInfo(getUserIfNotExists(ozoneManager)) - .setLayoutVersion(layoutVersion).build(); + .setLayoutVersion(layoutVersion); + + if (requestBuilder.hasS3Authentication()) { + requestBuilder.setS3Authentication( + resolveS3Authentication(requestBuilder.getS3Authentication(), OzoneManager.getStsTokenIdentifier())); + } + + omRequest = requestBuilder.build(); return omRequest; } + private static OzoneManagerProtocolProtos.S3Authentication resolveS3Authentication( + OzoneManagerProtocolProtos.S3Authentication s3Auth, STSTokenIdentifier stsTokenIdentifier) { + final OzoneManagerProtocolProtos.S3Authentication.Builder s3AuthBuilder = s3Auth.toBuilder(); + + if (s3Auth.hasSessionToken() && !s3Auth.getSessionToken().isEmpty() && stsTokenIdentifier != null) { + s3AuthBuilder.setResolvedStsSessionPolicy( + StringUtils.defaultString(stsTokenIdentifier.getSessionPolicy())); + s3AuthBuilder.setResolvedStsRoleArn( + StringUtils.defaultString(stsTokenIdentifier.getRoleArn())); + s3AuthBuilder.setResolvedStsOriginalAccessKeyId( + StringUtils.defaultString(stsTokenIdentifier.getOriginalAccessKeyId())); + s3AuthBuilder.setResolvedStsTempAccessKeyId( + StringUtils.defaultString(stsTokenIdentifier.getTempAccessKeyId())); + final UUID secretKeyId = stsTokenIdentifier.getSecretKeyId(); + s3AuthBuilder.setResolvedStsSecretKeyId( + secretKeyId != null ? secretKeyId.toString() : ""); + } else { + s3AuthBuilder.clearResolvedStsSessionPolicy(); + s3AuthBuilder.clearResolvedStsRoleArn(); + s3AuthBuilder.clearResolvedStsOriginalAccessKeyId(); + s3AuthBuilder.clearResolvedStsTempAccessKeyId(); + s3AuthBuilder.clearResolvedStsSecretKeyId(); + } + + return s3AuthBuilder.build(); + } + /** * Performs any request specific failure handling during request * submission. An example of this would be an undo of any steps diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketSetAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketSetAclRequest.java index 97dca83c1978..678c4ba0dc86 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketSetAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketSetAclRequest.java @@ -54,14 +54,15 @@ public class OMBucketSetAclRequest extends OMBucketAclRequest { @Override public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { - long modificationTime = Time.now(); - OzoneManagerProtocolProtos.SetAclRequest.Builder setAclRequestBuilder = + final long modificationTime = Time.now(); + final OzoneManagerProtocolProtos.SetAclRequest.Builder setAclRequestBuilder = getOmRequest().getSetAclRequest().toBuilder() .setModificationTime(modificationTime); - return getOmRequest().toBuilder() + // super.preExecute resolves S3Authentication (STS) for Ratis apply. Merge SetAclRequest changes on top. + final OMRequest request = super.preExecute(ozoneManager); + return request.toBuilder() .setSetAclRequest(setAclRequestBuilder) - .setUserInfo(getUserInfo()) .build(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSSecurityUtil.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSSecurityUtil.java index c414708cebee..2212ad6db797 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSSecurityUtil.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSSecurityUtil.java @@ -31,7 +31,9 @@ import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey; import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMTokenProto; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication; import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.Token; @@ -179,5 +181,44 @@ static void ensureEssentialFieldsArePresentInToken(STSTokenIdentifier stsTokenId throw new SecretManager.InvalidToken("Invalid STS token - secretAccessKey is null/empty"); } } + + /** + * Ensures STS-related {@link S3Authentication} fields are structurally consistent on the Ratis + * apply path. Cryptographic validation (signature, expiry, secret key lookup) runs on the leader + * RPC path (e.g. {@code S3SecurityUtil.validateS3Credential}). This method performs no crypto and does + * not contact {@link SecretKeyClient}, keeping the apply thread deterministic and lightweight. + * + * @param request OM request possibly containing S3 authentication + * @throws OMException if resolved fields and session token presence are inconsistent + */ + public static void ensureResolvedStsFieldsInvariants(OzoneManagerProtocolProtos.OMRequest request) + throws OMException { + if (!request.hasS3Authentication()) { + return; + } + + final S3Authentication s3Auth = request.getS3Authentication(); + final boolean hasSessionToken = s3Auth.hasSessionToken() && !s3Auth.getSessionToken().isEmpty(); + + if (!hasSessionToken) { + // If sessionToken is missing/empty, resolved fields must be empty. + if (s3Auth.hasResolvedStsSessionPolicy() || s3Auth.hasResolvedStsRoleArn() || + s3Auth.hasResolvedStsOriginalAccessKeyId() || s3Auth.hasResolvedStsTempAccessKeyId() || + s3Auth.hasResolvedStsSecretKeyId()) { + throw new OMException("Resolved STS fields must be empty when sessionToken is not present", INVALID_TOKEN); + } + return; + } + + ensureResolvedFieldsArePresent(s3Auth); + } + + private static void ensureResolvedFieldsArePresent(S3Authentication s3Auth) throws OMException { + if (!s3Auth.hasResolvedStsSessionPolicy() || !s3Auth.hasResolvedStsRoleArn() || + !s3Auth.hasResolvedStsOriginalAccessKeyId() || !s3Auth.hasResolvedStsTempAccessKeyId() || + !s3Auth.hasResolvedStsSecretKeyId()) { + throw new OMException("Resolved STS fields must be present when sessionToken is present", INVALID_TOKEN); + } + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java index 6d3621cc0098..4baef5ac9a04 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java @@ -210,6 +210,72 @@ public void testUserInfoWithSTSToken() throws IOException { } } + @Test + public void testPreExecuteOverwritesResolvedStsFields() throws Exception { + try (MockedStatic mockedRpcServer = mockStatic(Server.class)) { + mockedRpcServer.when(Server::getRemoteUser).thenReturn(userGroupInformation); + mockedRpcServer.when(Server::getRemoteIp).thenReturn(inetAddress); + mockedRpcServer.when(Server::getRemoteAddress).thenReturn(inetAddress.toString()); + + final String accessId = "ASIA12345"; + final String signature = "Signature"; + final String stringToSign = "StringToSign"; + final String sessionToken = "SessionToken"; + final String originalAccessKeyId = "AKIAORIGINAL"; + final String roleArn = "arn:aws:iam::123456789012:role/test-role"; + final String sessionPolicy = "test-session-policy"; + final UUID secretKeyId = UUID.randomUUID(); + + final STSTokenIdentifier stsTokenIdentifier = mock(STSTokenIdentifier.class); + when(stsTokenIdentifier.getSessionPolicy()).thenReturn(sessionPolicy); + when(stsTokenIdentifier.getRoleArn()).thenReturn(roleArn); + when(stsTokenIdentifier.getOriginalAccessKeyId()).thenReturn(originalAccessKeyId); + when(stsTokenIdentifier.getTempAccessKeyId()).thenReturn(accessId); + when(stsTokenIdentifier.getSecretKeyId()).thenReturn(secretKeyId); + + final S3Authentication s3Authentication = S3Authentication.newBuilder() + .setAccessId(accessId) + .setSignature(signature) + .setStringToSign(stringToSign) + .setSessionToken(sessionToken) + .setResolvedStsSessionPolicy("client-session-policy") + .setResolvedStsRoleArn("client-role") + .setResolvedStsOriginalAccessKeyId("client-original-access-key-id") + .setResolvedStsTempAccessKeyId("client-temp-access-key-id") + .setResolvedStsSecretKeyId("client-secret-key-id") + .build(); + + OzoneManager.setS3Auth(s3Authentication); + OzoneManager.setStsTokenIdentifier(stsTokenIdentifier); + + try { + final String bucketName = UUID.randomUUID().toString(); + final String volumeName = UUID.randomUUID().toString(); + final BucketInfo.Builder bucketInfo = + newBucketInfoBuilder(bucketName, volumeName) + .setIsVersionEnabled(true) + .setStorageType(StorageTypeProto.DISK); + + final OMRequest omRequest = newCreateBucketRequest(bucketInfo) + .setS3Authentication(s3Authentication) + .build(); + + final OMBucketCreateRequest omBucketCreateRequest = new OMBucketCreateRequest(omRequest); + final OMRequest modifiedRequest = omBucketCreateRequest.preExecute(ozoneManager); + final S3Authentication modifiedS3Auth = modifiedRequest.getS3Authentication(); + + assertEquals(sessionPolicy, modifiedS3Auth.getResolvedStsSessionPolicy()); + assertEquals(roleArn, modifiedS3Auth.getResolvedStsRoleArn()); + assertEquals(originalAccessKeyId, modifiedS3Auth.getResolvedStsOriginalAccessKeyId()); + assertEquals(accessId, modifiedS3Auth.getResolvedStsTempAccessKeyId()); + assertEquals(secretKeyId.toString(), modifiedS3Auth.getResolvedStsSecretKeyId()); + } finally { + OzoneManager.setStsTokenIdentifier(null); + OzoneManager.setS3Auth(null); + } + } + } + @Test public void testUserInfoWithSTSAccessKeyMissingSessionToken() { final String accessId = "ASIA12345"; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSSecurityUtil.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSSecurityUtil.java index c93df8a49009..8decf4fd316f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSSecurityUtil.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSSecurityUtil.java @@ -35,7 +35,10 @@ import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient; import org.apache.hadoop.io.Text; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMTokenProto; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.Token; import org.apache.ozone.test.TestClock; @@ -374,4 +377,76 @@ public void testEnsureEssentialFieldsArePresentInTokenMissingSecretAccessKey() { .isInstanceOf(SecretManager.InvalidToken.class) .hasMessage("Invalid STS token - secretAccessKey is null/empty"); } + + @Test + public void testEnsureResolvedStsFieldsInvariantsSuccess() throws Exception { + final String tokenString = tokenSecretManager.createSTSTokenString( + TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, DURATION_SECONDS, SECRET_ACCESS_KEY, SESSION_POLICY, clock); + + final S3Authentication s3Auth = S3Authentication.newBuilder() + .setSessionToken(tokenString) + .setResolvedStsSessionPolicy(SESSION_POLICY) + .setResolvedStsRoleArn(ROLE_ARN) + .setResolvedStsOriginalAccessKeyId(ORIGINAL_ACCESS_KEY) + .setResolvedStsTempAccessKeyId(TEMP_ACCESS_KEY) + .setResolvedStsSecretKeyId(secretKeyId.toString()) + .build(); + + final OMRequest request = OMRequest.newBuilder() + .setCmdType(Type.CreateBucket) + .setClientId("client-id") + .setS3Authentication(s3Auth) + .build(); + + STSSecurityUtil.ensureResolvedStsFieldsInvariants(request); + } + + @Test + public void testEnsureResolvedStsFieldsInvariantsMissingSessionToken() { + final S3Authentication s3Auth = S3Authentication.newBuilder() + .setResolvedStsSessionPolicy(SESSION_POLICY) + .build(); + + final OMRequest request = OMRequest.newBuilder() + .setCmdType(Type.CreateBucket) + .setClientId("client-id") + .setS3Authentication(s3Auth) + .build(); + + assertThatThrownBy(() -> STSSecurityUtil.ensureResolvedStsFieldsInvariants(request)) + .isInstanceOf(OMException.class) + .hasMessageContaining("Resolved STS fields must be empty when sessionToken is not present"); + } + + @Test + public void testEnsureResolvedStsFieldsInvariantsMissingResolvedFields() throws Exception { + final String tokenString = tokenSecretManager.createSTSTokenString( + TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, DURATION_SECONDS, + SECRET_ACCESS_KEY, SESSION_POLICY, clock); + + final S3Authentication s3Auth = S3Authentication.newBuilder() + .setSessionToken(tokenString) + .build(); + + final OMRequest request = OMRequest.newBuilder() + .setCmdType(Type.CreateBucket) + .setClientId("client-id") + .setS3Authentication(s3Auth) + .build(); + + assertThatThrownBy(() -> STSSecurityUtil.ensureResolvedStsFieldsInvariants(request)) + .isInstanceOf(OMException.class) + .hasMessageContaining("Resolved STS fields must be present when sessionToken is present"); + } + + @Test + public void testEnsureResolvedStsFieldsInvariantsNoS3Auth() throws Exception { + final OMRequest request = OMRequest.newBuilder() + .setCmdType(Type.CreateBucket) + .setClientId("client-id") + .build(); + + // Should not throw + STSSecurityUtil.ensureResolvedStsFieldsInvariants(request); + } } From 7b59609ed7d73da33cb8ff543b9898685a05ef68 Mon Sep 17 00:00:00 2001 From: Fabian Morgan Date: Tue, 5 May 2026 15:36:15 -0700 Subject: [PATCH 2/3] ensure each S3 API has an associated S3 Action --- .../hadoop/ozone/om/protocol/S3Auth.java | 10 ++ ...ManagerProtocolClientSideTranslatorPB.java | 8 +- .../src/main/proto/OmClientProtocol.proto | 4 +- .../src/main/resources/proto.lock | 2 +- .../hadoop/ozone/om/OmMetadataReader.java | 35 +++-- .../om/ratis/OzoneManagerStateMachine.java | 8 + .../hadoop/ozone/om/TestOMMetadataReader.java | 122 +++++++++++----- .../ozone/s3/endpoint/BucketEndpoint.java | 27 ++-- .../ozone/s3/endpoint/EndpointBase.java | 36 +++++ .../ozone/s3/endpoint/ObjectEndpoint.java | 137 +++++++++--------- .../ozone/s3/endpoint/RootEndpoint.java | 8 +- .../ozone/s3/endpoint/S3RequestContext.java | 2 + .../ozone/s3/util/S3GActionIamMapper.java | 92 ++++++++++++ .../s3/endpoint/TestCopyActionsAudit.java | 133 +++++++++++++++++ .../ozone/s3/util/TestS3GActionIamMapper.java | 68 +++++++++ .../hadoop/ozone/s3/util/package-info.java | 21 +++ 16 files changed, 577 insertions(+), 136 deletions(-) create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3GActionIamMapper.java create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestCopyActionsAudit.java create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3GActionIamMapper.java create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/package-info.java diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/S3Auth.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/S3Auth.java index fa023dfc8119..577339c96ac3 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/S3Auth.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/S3Auth.java @@ -29,6 +29,8 @@ public class S3Auth { private String userPrincipal; // Optional STS session token when using temporary credentials private String sessionToken; + // S3 action without s3: prefix (e.g. PutObject), set by S3 Gateway for use in finer-grained STS permissions. + private String s3Action; public S3Auth(final String stringToSign, final String signature, @@ -67,4 +69,12 @@ public String getSessionToken() { public void setSessionToken(String sessionToken) { this.sessionToken = sessionToken; } + + public String getS3Action() { + return s3Action; + } + + public void setS3Action(String s3Action) { + this.s3Action = s3Action; + } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index b42b0e6ddf87..7559bbf94bd2 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -328,6 +328,9 @@ private OMResponse submitRequest(OMRequest omRequest) if (threadLocalS3Auth.get().getSessionToken() != null) { s3AuthBuilder.setSessionToken(threadLocalS3Auth.get().getSessionToken()); } + if (threadLocalS3Auth.get().getS3Action() != null) { + s3AuthBuilder.setS3Action(threadLocalS3Auth.get().getS3Action()); + } builder.setS3Authentication(s3AuthBuilder.build()); } @@ -1703,10 +1706,7 @@ public OmMultipartCommitUploadPartInfo commitMultipartUploadPart( handleError(submitRequest(omRequest)) .getCommitMultiPartUploadResponse(); - OmMultipartCommitUploadPartInfo info = new - OmMultipartCommitUploadPartInfo(response.getPartName(), - response.getETag()); - return info; + return new OmMultipartCommitUploadPartInfo(response.getPartName(), response.getETag()); } @Override diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index c3a0dc26a9f8..5f752dbcbd50 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1572,7 +1572,6 @@ message CommitKeyRequest { } message CommitKeyResponse { - } message AllocateBlockRequest { @@ -2319,6 +2318,9 @@ message S3Authentication { optional string resolvedStsOriginalAccessKeyId = 7; optional string resolvedStsTempAccessKeyId = 8; optional string resolvedStsSecretKeyId = 9; + // S3 action without the s3: prefix for this request (e.g. GetObject), set by S3 Gateway for use + // in finer-grained STS permissions. + optional string s3Action = 10; } message RecoverLeaseRequest { diff --git a/hadoop-ozone/interface-client/src/main/resources/proto.lock b/hadoop-ozone/interface-client/src/main/resources/proto.lock index 0271bd8a20f1..14dd43d747ca 100644 --- a/hadoop-ozone/interface-client/src/main/resources/proto.lock +++ b/hadoop-ozone/interface-client/src/main/resources/proto.lock @@ -8656,4 +8656,4 @@ } } ] -} \ No newline at end of file +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java index b14f01cf6ba0..1ef815d67555 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java @@ -55,6 +55,7 @@ import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.protocolPB.grpc.GrpcClientConstants; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication; import org.apache.hadoop.ozone.security.STSTokenIdentifier; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType; @@ -236,9 +237,7 @@ public List listStatus(OmKeyArgs args, boolean recursive, try { if (isAclEnabled) { if (isStsS3Request()) { - // We need to be able to tell the difference between being able to download a file and merely seeing the file - // name in a list. Use READ for download ability and LIST (here) for listing. - // When listPrefix is set (original S3 ListObjects prefix), authorize LIST on that prefix for the whole + // When listPrefix is set (original S3 ListObjects prefix), authorize READ on that prefix for the whole // listing, including FSO traversal where keyName is an internal directory (e.g. userA) under prefix user. final String listPrefix = args.getListPrefix(); final String keyName = args.getKeyName(); @@ -258,7 +257,7 @@ public List listStatus(OmKeyArgs args, boolean recursive, } else { aclKey = "*"; } - checkAcls(ResourceType.KEY, StoreType.OZONE, ACLType.LIST, bucket.realVolume(), bucket.realBucket(), aclKey); + checkAcls(ResourceType.KEY, StoreType.OZONE, ACLType.READ, bucket.realVolume(), bucket.realBucket(), aclKey); } else { checkAcls(getResourceType(args), StoreType.OZONE, ACLType.READ, bucket, args.getKeyName()); @@ -304,12 +303,7 @@ public OzoneFileStatus getFileStatus(OmKeyArgs args) throws IOException { try { if (isAclEnabled) { - if (isStsS3Request()) { - checkAcls(getResourceType(args), StoreType.OZONE, ACLType.LIST, bucket, args.getKeyName()); - } else { - checkAcls(getResourceType(args), StoreType.OZONE, ACLType.READ, - bucket, args.getKeyName()); - } + checkAcls(getResourceType(args), StoreType.OZONE, ACLType.READ, bucket, args.getKeyName()); } metrics.incNumGetFileStatus(); return keyManager.getFileStatus(args, getClientAddress()); @@ -384,7 +378,7 @@ public ListKeysResult listKeys(String volumeName, String bucketName, final String aclKey = (keyPrefix == null || keyPrefix.isEmpty()) ? "*" : keyPrefix; captureLatencyNs( perfMetrics.getListKeysAclCheckLatencyNs(), () -> checkAcls( - ResourceType.KEY, StoreType.OZONE, ACLType.LIST, bucket.realVolume(), bucket.realBucket(), aclKey)); + ResourceType.KEY, StoreType.OZONE, ACLType.READ, bucket.realVolume(), bucket.realBucket(), aclKey)); } else { captureLatencyNs(perfMetrics.getListKeysAclCheckLatencyNs(), () -> checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.LIST, @@ -634,7 +628,8 @@ public boolean checkAcls(ResourceType resType, StoreType storeType, public boolean checkAcls(OzoneObj obj, RequestContext context, boolean throwIfPermissionDenied) throws OMException { - final RequestContext normalizedRequestContext = maybeAttachSessionPolicyFromThreadLocal(context); + final RequestContext normalizedRequestContext = maybeAttachS3ActionFromThreadLocal( + maybeAttachSessionPolicyFromThreadLocal(context)); if (!captureLatencyNs(perfMetrics::setCheckAccessLatencyNs, () -> accessAuthorizer.checkAccess(obj, normalizedRequestContext))) { @@ -692,6 +687,22 @@ private RequestContext maybeAttachSessionPolicyFromThreadLocal(RequestContext co .build(); } + /** + * Attaches s3 action to RequestContext if an S3Authentication is found in the Ozone Manager thread local, + * and it has an s3 action. Otherwise, returns the RequestContext as it was before. + * @param context the original RequestContext + * @return RequestContext as before or with s3 action embedded + */ + private RequestContext maybeAttachS3ActionFromThreadLocal(RequestContext context) { + final S3Authentication s3Authentication = OzoneManager.getS3Auth(); + if (s3Authentication == null || !s3Authentication.hasS3Action()) { + return context; + } + return context.toBuilder() + .setS3Action(s3Authentication.getS3Action()) + .build(); + } + static String getClientAddress() { String clientMachine = Server.getRemoteAddress(); if (clientMachine == null) { //not a RPC client diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java index d17ad9e62e46..58f70af236c8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java @@ -619,6 +619,7 @@ public void close() { */ @VisibleForTesting OMResponse runCommand(OMRequest request, TermIndex termIndex) { + boolean isS3AuthThreadLocalSet = false; boolean isStsThreadLocalSet = false; try { if (ozoneManager.isSecurityEnabled() && request.hasS3Authentication()) { @@ -627,6 +628,10 @@ OMResponse runCommand(OMRequest request, TermIndex termIndex) { STSSecurityUtil.ensureResolvedStsFieldsInvariants(request); final OzoneManagerProtocolProtos.S3Authentication s3Auth = request.getS3Authentication(); + // ThreadLocal carries S3 action for OmMetadataReader. + OzoneManager.setS3Auth(s3Auth); + isS3AuthThreadLocalSet = true; + if (s3Auth.hasSessionToken() && !s3Auth.getSessionToken().isEmpty()) { // ThreadLocal carries session policy for OmMetadataReader final STSTokenIdentifier rehydratedTokenIdentifier = new STSTokenIdentifier( @@ -661,6 +666,9 @@ OMResponse runCommand(OMRequest request, TermIndex termIndex) { String errorMessage = "Request " + request + " failed with exception"; ExitUtils.terminate(1, errorMessage, e, LOG); } finally { + if (isS3AuthThreadLocalSet) { + OzoneManager.setS3Auth(null); + } if (isStsThreadLocalSet) { OzoneManager.setStsTokenIdentifier(null); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMetadataReader.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMetadataReader.java index 8403d2203e01..6a6dea551f51 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMetadataReader.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMetadataReader.java @@ -23,7 +23,6 @@ import static org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType.KEY; import static org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType.VOLUME; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -78,8 +77,9 @@ public class TestOMMetadataReader { private static final long MAX_KEYS = 100L; @AfterEach - public void clearStsThreadLocal() { + public void clearOmThreadLocals() { OzoneManager.setStsTokenIdentifier(null); + OzoneManager.setS3Auth(null); } @Test @@ -145,7 +145,58 @@ public void testNoSessionPolicyWhenThreadLocalIsNull() throws Exception { } @Test - public void testListStatusUsesListAclForStsS3Request() throws Exception { + public void testCheckAclsAttachesS3ActionFromThreadLocal() throws Exception { + OzoneManager.setS3Auth(S3Authentication.newBuilder() + .setAccessId(ACCESS_KEY_ID) + .setS3Action("GetObject") + .build()); + + final IAccessAuthorizer accessAuthorizer = createMockIAccessAuthorizerReturningTrue(); + final OmMetadataReader omMetadataReader = createMetadataReader(accessAuthorizer); + + final RequestContext contextWithoutS3Action = createTestRequestContext(); + final OzoneObj obj = createTestOzoneObj(); + + assertTrue(omMetadataReader.checkAcls(obj, contextWithoutS3Action, true)); + + verifyS3ActionPassedToAuthorizer(accessAuthorizer, obj, "GetObject"); + } + + @Test + public void testCheckAclsLeavesS3ActionUnsetWhenS3AuthThreadLocalNull() throws Exception { + final IAccessAuthorizer accessAuthorizer = createMockIAccessAuthorizerReturningTrue(); + final OmMetadataReader omMetadataReader = createMetadataReader(accessAuthorizer); + + final RequestContext contextWithoutS3Action = createTestRequestContext(); + final OzoneObj obj = createTestOzoneObj(); + + assertTrue(omMetadataReader.checkAcls(obj, contextWithoutS3Action, true)); + + verifyS3ActionPassedToAuthorizer(accessAuthorizer, obj, null); + } + + @Test + public void testCheckAclsAttachesSessionPolicyAndS3ActionFromThreadLocals() throws Exception { + setupStsTokenIdentifier(); + + OzoneManager.setS3Auth(S3Authentication.newBuilder() + .setAccessId(ACCESS_KEY_ID) + .setS3Action("PutObject") + .build()); + + final IAccessAuthorizer accessAuthorizer = createMockIAccessAuthorizerReturningTrue(); + final OmMetadataReader omMetadataReader = createMetadataReader(accessAuthorizer); + + final RequestContext baseContext = createTestRequestContext(); + final OzoneObj obj = createTestOzoneObj(); + + assertTrue(omMetadataReader.checkAcls(obj, baseContext, true)); + + verifySessionPolicyAndS3ActionPassedToAuthorizer(accessAuthorizer, obj); + } + + @Test + public void testListStatusUsesReadAclForStsS3Request() throws Exception { setupStsS3Request(); final IAccessAuthorizer accessAuthorizer = createMockIAccessAuthorizerReturningTrue(); @@ -161,10 +212,9 @@ public void testListStatusUsesListAclForStsS3Request() throws Exception { // For STS S3 requests, listStatus() performs these checks: // 1. Volume READ (for volume access) - // 2) Key LIST (for the specific prefix being listed) - we need LIST permission for STS in order to tell whether the - // file should be listed only or downloadable (downloadable would be READ) + // 2) Key READ (for the specific prefix being listed) assertContainsVolumeReadCheck(checks); - assertContainsKeyListCheckWithName(checks, KEY_PREFIX); + assertContainsKeyReadCheckWithName(checks, KEY_PREFIX); } @Test @@ -186,7 +236,6 @@ public void testListStatusUsesReadAclForNonStsRequest() throws Exception { assertContainsVolumeReadCheck(checks); // We want to ensure the current behavior for non-STS requests remains the same assertContainsKeyReadCheckWithName(checks); - assertDoesNotContainKeyListCheck(checks); } @Test @@ -209,7 +258,7 @@ public void testListStatusUsesListPrefixForAclWhenKeyNameEmptyAndListPrefixSet() final List checks = captureAclChecks(accessAuthorizer, 2); assertContainsVolumeReadCheck(checks); - assertContainsKeyListCheckWithName(checks, "userA/"); + assertContainsKeyReadCheckWithName(checks, "userA/"); } @Test @@ -231,7 +280,7 @@ public void testListStatusUsesWildcardForAclWhenKeyNameAndListPrefixEmpty() thro final List checks = captureAclChecks(accessAuthorizer, 2); assertContainsVolumeReadCheck(checks); - assertContainsKeyListCheckWithName(checks, "*"); + assertContainsKeyReadCheckWithName(checks, "*"); } @Test @@ -254,7 +303,7 @@ public void testListStatusUsesListPrefixForAclWhenKeyNameIsDescendantOfListPrefi final List checks = captureAclChecks(accessAuthorizer, 2); assertContainsVolumeReadCheck(checks); - assertContainsKeyListCheckWithName(checks, "user"); + assertContainsKeyReadCheckWithName(checks, "user"); } @Test @@ -277,7 +326,7 @@ public void testListStatusUsesListPrefixForAclWhenKeyNameIsAncestorOfListPrefix( final List checks = captureAclChecks(accessAuthorizer, 2); assertContainsVolumeReadCheck(checks); - assertContainsKeyListCheckWithName(checks, "user/foo"); + assertContainsKeyReadCheckWithName(checks, "user/foo"); } @Test @@ -301,7 +350,7 @@ public void testListStatusThrowsWhenStsKeyNameNotUnderListPrefix() throws Except } @Test - public void testGetFileStatusUsesListAclForStsS3Request() throws Exception { + public void testGetFileStatusUsesReadAclForStsS3Request() throws Exception { setupStsS3Request(); final IAccessAuthorizer accessAuthorizer = createMockIAccessAuthorizerReturningTrue(); @@ -314,8 +363,7 @@ public void testGetFileStatusUsesListAclForStsS3Request() throws Exception { final List checks = captureAclChecks(accessAuthorizer, 2); assertContainsVolumeReadCheck(checks); - assertContainsKeyListCheckWithName(checks, KEY_PREFIX); - assertDoesNotContainKeyReadCheck(checks); + assertContainsKeyReadCheckWithName(checks, KEY_PREFIX); } @Test @@ -333,7 +381,6 @@ public void testGetFileStatusUsesReadAclForNonStsS3Request() throws Exception { final List checks = captureAclChecks(accessAuthorizer, 2); assertContainsVolumeReadCheck(checks); assertContainsKeyReadCheckWithName(checks); - assertDoesNotContainKeyListCheck(checks); } @Test @@ -350,7 +397,7 @@ public void testListKeysUsesPrefixCheckForStsS3Request() throws Exception { List checks = captureAclChecks(accessAuthorizer, 4); assertContainsBucketListCheck(checks); - assertContainsKeyListCheckWithName(checks, "userA/"); + assertContainsKeyReadCheckWithName(checks, "userA/"); // Reset to make case 2 assertions independent of case 1 captures. reset(accessAuthorizer); @@ -361,7 +408,7 @@ public void testListKeysUsesPrefixCheckForStsS3Request() throws Exception { checks = captureAclChecks(accessAuthorizer, 4); assertContainsBucketListCheck(checks); - assertContainsKeyListCheckWithName(checks, "*"); + assertContainsKeyReadCheckWithName(checks, "*"); } private OmMetadataReader createMetadataReader(IAccessAuthorizer accessAuthorizer) throws IOException { @@ -505,6 +552,27 @@ private void verifySessionPolicyPassedToAuthorizer(IAccessAuthorizer accessAutho assertEquals(expectedSessionPolicy, captor.getValue().getSessionPolicy()); } + /** + * Verifies that the accessAuthorizer received a call to checkAccess with the expected s3 action. + * @param accessAuthorizer the mock authorizer to verify + * @param expectedObj the expected OzoneObj + * @param expectedS3Action the expected s3 action (could be null) + */ + private void verifyS3ActionPassedToAuthorizer(IAccessAuthorizer accessAuthorizer, OzoneObj expectedObj, + String expectedS3Action) throws OMException { + final ArgumentCaptor captor = ArgumentCaptor.forClass(RequestContext.class); + verify(accessAuthorizer).checkAccess(eq(expectedObj), captor.capture()); + assertEquals(expectedS3Action, captor.getValue().getS3Action()); + } + + private void verifySessionPolicyAndS3ActionPassedToAuthorizer(IAccessAuthorizer accessAuthorizer, + OzoneObj expectedObj) throws OMException { + final ArgumentCaptor captor = ArgumentCaptor.forClass(RequestContext.class); + verify(accessAuthorizer).checkAccess(eq(expectedObj), captor.capture()); + assertEquals("session-policy-from-thread-local", captor.getValue().getSessionPolicy()); + assertEquals("PutObject", captor.getValue().getS3Action()); + } + private List captureAclChecks(IAccessAuthorizer accessAuthorizer, int expectedCheckCount) throws OMException { final ArgumentCaptor objCaptor = ArgumentCaptor.forClass(OzoneObj.class); @@ -538,12 +606,12 @@ private void assertContainsBucketListCheck(List checks) { "Expected a BUCKET LIST ACL check"); } - private void assertContainsKeyListCheckWithName(List checks, String keyName) { + private void assertContainsKeyReadCheckWithName(List checks, String keyName) { assertTrue( checks.stream().anyMatch( - check -> check.getObj().getResourceType() == KEY && check.getContext().getAclRights() == LIST && + check -> check.getObj().getResourceType() == KEY && check.getContext().getAclRights() == READ && keyName.equals(check.getObj().getKeyName())), - "Expected a KEY LIST ACL check for key '" + keyName + "'"); + "Expected a KEY READ ACL check for key '" + keyName + "'"); } private void assertContainsKeyReadCheckWithName(List checks) { @@ -554,20 +622,6 @@ private void assertContainsKeyReadCheckWithName(List checks) { "Expected a KEY READ ACL check for key '" + TestOMMetadataReader.KEY_PREFIX + "'"); } - private void assertDoesNotContainKeyReadCheck(List checks) { - assertFalse( - checks.stream().anyMatch( - check -> check.getObj().getResourceType() == KEY && check.getContext().getAclRights() == READ), - "Did not expect a KEY READ ACL check"); - } - - private void assertDoesNotContainKeyListCheck(List checks) { - assertFalse( - checks.stream().anyMatch( - check -> check.getObj().getResourceType() == KEY && check.getContext().getAclRights() == LIST), - "Did not expect a KEY LIST ACL check"); - } - private static final class AclCheck { private final OzoneObj obj; private final RequestContext context; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 799af5d7fa95..8325017c7b5f 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -100,7 +100,6 @@ public Response get( S3RequestContext context = new S3RequestContext(this, S3GAction.GET_BUCKET); long startNanos = context.getStartNanos(); - S3GAction s3GAction = context.getAction(); PerformanceStringBuilder perf = context.getPerf(); // Chain of responsibility: let each handler try to handle the request @@ -128,7 +127,7 @@ public Response get( try { final String uploads = queryParams().get(QueryParams.UPLOADS); if (uploads != null) { - s3GAction = S3GAction.LIST_MULTIPART_UPLOAD; + context.setAction(S3GAction.LIST_MULTIPART_UPLOAD); final String uploadIdMarker = queryParams().get(QueryParams.UPLOAD_ID_MARKER); final String keyMarker = queryParams().get(QueryParams.KEY_MARKER); return listMultipartUploads(bucketName, prefix, keyMarker, uploadIdMarker, maxUploads); @@ -161,12 +160,12 @@ public Response get( ozoneKeyIterator = bucket.listKeys(prefix, prevKey, shallow); } catch (OMException ex) { - auditReadFailure(s3GAction, ex); + auditReadFailure(context.getAction(), ex); getMetrics().updateGetBucketFailureStats(startNanos); handleOMException(ex, bucketName, prefix); } catch (Exception ex) { getMetrics().updateGetBucketFailureStats(startNanos); - auditReadFailure(s3GAction, ex); + auditReadFailure(context.getAction(), ex); throw ex; } @@ -255,11 +254,11 @@ public Response get( } } catch (RuntimeException ex) { getMetrics().updateGetBucketFailureStats(startNanos); - auditReadFailure(s3GAction, ex); + auditReadFailure(context.getAction(), ex); if (ex.getCause() instanceof OMException) { final OMException omException = (OMException) ex.getCause(); if (omException.getResult() == ResultCodes.FILE_NOT_FOUND) { - throw ex; + throw ex; } handleOMException(omException, bucketName, prefix); } else { @@ -289,7 +288,7 @@ public Response get( getMetrics().incListKeyCount(keyCount); perf.appendCount(keyCount); perf.appendOpLatencyNanos(opLatencyNs); - auditReadSuccess(s3GAction, perf); + auditReadSuccess(context.getAction(), perf); response.setKeyCount(keyCount); return Response.ok(response).build(); } @@ -390,16 +389,16 @@ public Response listMultipartUploads( @HEAD public Response head(@PathParam(BUCKET) String bucketName) throws OS3Exception, IOException { - long startNanos = Time.monotonicNowNanos(); - S3GAction s3GAction = S3GAction.HEAD_BUCKET; + S3RequestContext context = new S3RequestContext(this, S3GAction.HEAD_BUCKET); + long startNanos = context.getStartNanos(); try { OzoneBucket bucket = getBucket(bucketName); S3Owner.verifyBucketOwnerCondition(getHeaders(), bucketName, bucket.getOwner()); - auditReadSuccess(s3GAction); + auditReadSuccess(context.getAction()); getMetrics().updateHeadBucketSuccessStats(startNanos); return Response.ok().build(); } catch (Exception e) { - auditReadFailure(s3GAction, e); + auditReadFailure(context.getAction(), e); throw e; } } @@ -438,7 +437,7 @@ public MultiDeleteResponse multiDelete( @QueryParam(QueryParams.DELETE) String delete, MultiDeleteRequest request ) throws OS3Exception, IOException { - S3GAction s3GAction = S3GAction.MULTI_DELETE; + S3RequestContext context = new S3RequestContext(this, S3GAction.MULTI_DELETE); OzoneBucket bucket = getBucket(bucketName); MultiDeleteResponse result = new MultiDeleteResponse(); @@ -449,7 +448,7 @@ public MultiDeleteResponse multiDelete( for (DeleteObject keyToDelete : request.getObjects()) { deleteKeys.add(keyToDelete.getKey()); } - long startNanos = Time.monotonicNowNanos(); + long startNanos = context.getStartNanos(); try { S3Owner.verifyBucketOwnerCondition(getHeaders(), bucketName, bucket.getOwner()); undeletedKeyResultMap = bucket.deleteKeys(deleteKeys, true); @@ -477,7 +476,7 @@ public MultiDeleteResponse multiDelete( } } - AuditMessage.Builder message = auditMessageFor(s3GAction); + AuditMessage.Builder message = auditMessageFor(context.getAction()); message.getParams().put("failedDeletes", deleteKeys.toString()); if (!result.getErrors().isEmpty()) { diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 20ad21e23f11..5b31ffbc7716 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -87,6 +87,7 @@ import org.apache.hadoop.ozone.audit.AuditLogger.PerformanceStringBuilder; import org.apache.hadoop.ozone.audit.AuditLoggerType; import org.apache.hadoop.ozone.audit.AuditMessage; +import org.apache.hadoop.ozone.audit.S3GAction; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientUtils; @@ -106,11 +107,13 @@ import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics; import org.apache.hadoop.ozone.s3.signature.SignatureInfo; import org.apache.hadoop.ozone.s3.util.AuditUtils; +import org.apache.hadoop.ozone.s3.util.S3GActionIamMapper; import org.apache.hadoop.ozone.s3.util.S3Utils; import org.apache.hadoop.ozone.web.utils.OzoneUtils; import org.apache.hadoop.util.Time; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URLEncodedUtils; +import org.apache.ratis.util.function.CheckedSupplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -252,6 +255,39 @@ protected void init() { // hook method } + /** + * Sets the IAM S3 action on thread-local {@link S3Auth} for fine-grained STS authorization. + * Called when the handler resolves the {@link S3GAction}. + */ + protected void applyS3Action(S3GAction action) { + if (s3Auth != null) { + s3Auth.setS3Action(S3GActionIamMapper.toS3ActionString(action)); + } + } + + /** + * Temporarily override the S3 action string set on {@link S3Auth} for authorization. + *

+ * This does not change S3G auditing (which is based on {@link S3GAction}). + * The action string is the IAM-style S3 action name without the {@code s3:} prefix (for example + * {@code GetObject}, {@code PutObject}, {@code GetObjectTagging}). + * This is used for special case APIs like CopyObject that don't have a 1-1 s3 action mapping, but + * requires GetObject on the source file and PutObject on the destination file. + */ + protected T runWithS3ActionString(String s3Action, CheckedSupplier checkedSupplier) + throws E { + if (s3Auth == null) { + return checkedSupplier.get(); + } + final String originalS3Action = s3Auth.getS3Action(); + s3Auth.setS3Action(s3Action); + try { + return checkedSupplier.get(); + } finally { + s3Auth.setS3Action(originalS3Action); + } + } + protected OzoneBucket getBucket(String bucketName) throws OS3Exception, IOException { OzoneBucket bucket; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index d97c514f9ae6..53629016efa1 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -539,8 +539,8 @@ static void addTagCountIfAny( public Response head( @PathParam(BUCKET) String bucketName, @PathParam(PATH) String keyPath) throws IOException, OS3Exception { - long startNanos = Time.monotonicNowNanos(); - S3GAction s3GAction = S3GAction.HEAD_KEY; + ObjectRequestContext context = new ObjectRequestContext(S3GAction.HEAD_KEY, bucketName); + long startNanos = context.getStartNanos(); OzoneKey key; try { @@ -553,7 +553,7 @@ public Response head( isFile(keyPath, key); // TODO: return the specified range bytes of this object. } catch (OMException ex) { - auditReadFailure(s3GAction, ex); + auditReadFailure(context.getAction(), ex); getMetrics().updateHeadKeyFailureStats(startNanos); if (ex.getResult() == ResultCodes.KEY_NOT_FOUND) { // Just return 404 with no content @@ -568,7 +568,7 @@ public Response head( throw ex; } } catch (Exception ex) { - auditReadFailure(s3GAction, ex); + auditReadFailure(context.getAction(), ex); throw ex; } @@ -596,7 +596,7 @@ public Response head( addLastModifiedDate(response, key); addCustomMetadataHeaders(response, key); getMetrics().updateHeadKeySuccessStats(startNanos); - auditReadSuccess(s3GAction); + auditReadSuccess(context.getAction()); return response.build(); } @@ -691,8 +691,8 @@ public Response initializeMultipartUpload( @PathParam(BUCKET) String bucket, @PathParam(PATH) String key ) throws IOException, OS3Exception { - long startNanos = Time.monotonicNowNanos(); - S3GAction s3GAction = S3GAction.INIT_MULTIPART_UPLOAD; + ObjectRequestContext context = new ObjectRequestContext(S3GAction.INIT_MULTIPART_UPLOAD, bucket); + long startNanos = context.getStartNanos(); try { OzoneBucket ozoneBucket = getBucket(bucket); @@ -715,12 +715,12 @@ public Response initializeMultipartUpload( multipartUploadInitiateResponse.setKey(key); multipartUploadInitiateResponse.setUploadID(multipartInfo.getUploadID()); - auditWriteSuccess(s3GAction); + auditWriteSuccess(context.getAction()); getMetrics().updateInitMultipartUploadSuccessStats(startNanos); return Response.status(Status.OK).entity( multipartUploadInitiateResponse).build(); } catch (OMException ex) { - auditWriteFailure(s3GAction, ex); + auditWriteFailure(context.getAction(), ex); getMetrics().updateInitMultipartUploadFailureStats(startNanos); if (isExpiredToken(ex)) { throw newError(S3ErrorTable.EXPIRED_TOKEN, key, ex); @@ -730,7 +730,7 @@ public Response initializeMultipartUpload( } throw ex; } catch (Exception ex) { - auditWriteFailure(s3GAction, ex); + auditWriteFailure(context.getAction(), ex); getMetrics().updateInitMultipartUploadFailureStats(startNanos); throw ex; } @@ -746,9 +746,9 @@ public Response completeMultipartUpload( @PathParam(PATH) String key, CompleteMultipartUploadRequest multipartUploadRequest ) throws IOException, OS3Exception { + ObjectRequestContext context = new ObjectRequestContext(S3GAction.COMPLETE_MULTIPART_UPLOAD, bucket); final String uploadID = queryParams().get(QueryParams.UPLOAD_ID, ""); - long startNanos = Time.monotonicNowNanos(); - S3GAction s3GAction = S3GAction.COMPLETE_MULTIPART_UPLOAD; + long startNanos = context.getStartNanos(); OzoneVolume volume = getVolume(); // Using LinkedHashMap to preserve ordering of parts list. Map partsMap = new LinkedHashMap<>(); @@ -776,12 +776,12 @@ public Response completeMultipartUpload( wrapInQuotes(omMultipartUploadCompleteInfo.getHash())); // Location also setting as bucket name. completeMultipartUploadResponse.setLocation(bucket); - auditWriteSuccess(s3GAction); + auditWriteSuccess(context.getAction()); getMetrics().updateCompleteMultipartUploadSuccessStats(startNanos); return Response.status(Status.OK).entity(completeMultipartUploadResponse) .build(); } catch (OMException ex) { - auditWriteFailure(s3GAction, ex); + auditWriteFailure(context.getAction(), ex); getMetrics().updateCompleteMultipartUploadFailureStats(startNanos); if (ex.getResult() == ResultCodes.INVALID_PART) { throw newError(S3ErrorTable.INVALID_PART, key, ex); @@ -814,7 +814,7 @@ public Response completeMultipartUpload( } throw ex; } catch (Exception ex) { - auditWriteFailure(s3GAction, ex); + auditWriteFailure(context.getAction(), ex); throw ex; } } @@ -854,8 +854,8 @@ private Response createMultipartKey(OzoneVolume volume, OzoneBucket ozoneBucket, uploadID, getChunkSize(), multiDigestInputStream, perf, getHeaders()); } // OmMultipartCommitUploadPartInfo can only be gotten after the - // OzoneOutputStream is closed, so we need to save the OzoneOutputStream - final OzoneOutputStream outputStream; + // OzoneOutputStream is closed, so we need to get and save the commit info. + final OmMultipartCommitUploadPartInfo omMultipartCommitUploadPartInfo; long metadataLatencyNs; if (copyHeader != null) { Pair result = parseSourceHeader(copyHeader); @@ -867,8 +867,8 @@ private Response createMultipartKey(OzoneVolume volume, OzoneBucket ozoneBucket, ozoneBucket.getOwner()); } - OzoneKeyDetails sourceKeyDetails = getClientProtocol().getKeyDetails( - volume.getName(), sourceBucket, sourceKey); + final OzoneKeyDetails sourceKeyDetails = runWithS3ActionString( + "GetObject", () -> getClientProtocol().getKeyDetails(volume.getName(), sourceBucket, sourceKey)); String range = getHeaders().getHeaderString(COPY_SOURCE_HEADER_RANGE); RangeHeader rangeHeader = null; @@ -893,7 +893,8 @@ private Response createMultipartKey(OzoneVolume volume, OzoneBucket ozoneBucket, } try (OzoneInputStream sourceObject = sourceKeyDetails.getContent()) { - long copyLength; + final long[] copyLengthHolder = new long[1]; + final long[] metadataLatencyHolder = new long[1]; if (range != null) { final long skipped = sourceObject.skip(rangeHeader.getStartOffset()); @@ -903,52 +904,55 @@ private Response createMultipartKey(OzoneVolume volume, OzoneBucket ozoneBucket, + rangeHeader.getStartOffset() + " actual: " + skipped); } } - try (OzoneOutputStream ozoneOutputStream = getClientProtocol() - .createMultipartKey(volume.getName(), bucketName, key, length, - partNumber, uploadID)) { - metadataLatencyNs = - getMetrics().updateCopyKeyMetadataStats(startNanos); - copyLength = IOUtils.copyLarge(sourceObject, ozoneOutputStream, 0, length, - new byte[getIOBufferSize(length)]); - ozoneOutputStream.getMetadata() - .putAll(sourceKeyDetails.getMetadata()); - String raw = ozoneOutputStream.getMetadata().get(OzoneConsts.ETAG); - if (raw != null) { - ozoneOutputStream.getMetadata().put(OzoneConsts.ETAG, stripQuotes(raw)); + final long finalLength = length; + final long bytesToCopy = length; + omMultipartCommitUploadPartInfo = runWithS3ActionString("PutObject", () -> { + final OzoneOutputStream ozoneOutputStream = getClientProtocol().createMultipartKey( + volume.getName(), bucketName, key, finalLength, partNumber, uploadID); + try (OzoneOutputStream ignored = ozoneOutputStream) { + metadataLatencyHolder[0] = getMetrics().updateCopyKeyMetadataStats(startNanos); + copyLengthHolder[0] = IOUtils.copyLarge( + sourceObject, ozoneOutputStream, 0, bytesToCopy, new byte[getIOBufferSize(bytesToCopy)]); + ozoneOutputStream.getMetadata() + .putAll(sourceKeyDetails.getMetadata()); + final String raw = ozoneOutputStream.getMetadata().get(OzoneConsts.ETAG); + if (raw != null) { + ozoneOutputStream.getMetadata().put(OzoneConsts.ETAG, stripQuotes(raw)); + } } - outputStream = ozoneOutputStream; - } - getMetrics().incCopyObjectSuccessLength(copyLength); - perf.appendSizeBytes(copyLength); + return ozoneOutputStream.getCommitUploadPartInfo(); + }); + metadataLatencyNs = metadataLatencyHolder[0]; + getMetrics().incCopyObjectSuccessLength(copyLengthHolder[0]); + perf.appendSizeBytes(copyLengthHolder[0]); } } else { - long putLength; - try (OzoneOutputStream ozoneOutputStream = getClientProtocol() + final long putLength; + final OzoneOutputStream ozoneOutputStream = getClientProtocol() .createMultipartKey(volume.getName(), bucketName, key, length, - partNumber, uploadID)) { + partNumber, uploadID); + try (OzoneOutputStream ignored = ozoneOutputStream) { metadataLatencyNs = getMetrics().updatePutKeyMetadataStats(startNanos); putLength = IOUtils.copyLarge(multiDigestInputStream, ozoneOutputStream, 0, length, new byte[getIOBufferSize(length)]); - byte[] digest = multiDigestInputStream.getMessageDigest(OzoneConsts.MD5_HASH).digest(); - String md5Hash = DatatypeConverter.printHexBinary(digest).toLowerCase(); - String clientContentMD5 = getHeaders().getHeaderString(S3Consts.CHECKSUM_HEADER); + final byte[] digest = multiDigestInputStream.getMessageDigest(OzoneConsts.MD5_HASH).digest(); + final String md5Hash = DatatypeConverter.printHexBinary(digest).toLowerCase(); + final String clientContentMD5 = getHeaders().getHeaderString(S3Consts.CHECKSUM_HEADER); if (clientContentMD5 != null) { - CheckedRunnable checkContentMD5Hook = () -> { + final CheckedRunnable checkContentMD5Hook = () -> { S3Utils.validateContentMD5(clientContentMD5, md5Hash, key); }; ozoneOutputStream.getKeyOutputStream().setPreCommits(Collections.singletonList(checkContentMD5Hook)); } ozoneOutputStream.getMetadata().put(OzoneConsts.ETAG, md5Hash); - outputStream = ozoneOutputStream; } + omMultipartCommitUploadPartInfo = ozoneOutputStream.getCommitUploadPartInfo(); getMetrics().incPutKeySuccessLength(putLength); perf.appendSizeBytes(putLength); } perf.appendMetaLatencyNanos(metadataLatencyNs); - OmMultipartCommitUploadPartInfo omMultipartCommitUploadPartInfo = - outputStream.getCommitUploadPartInfo(); String eTag = omMultipartCommitUploadPartInfo.getETag(); // If the OmMultipartCommitUploadPartInfo does not contain eTag, // fall back to MPU part name for compatibility in case the (old) OM @@ -1018,7 +1022,7 @@ srcKeyLen > getDatastreamMinLength()) { getMetrics().updateCopyKeyMetadataStats(startNanos); perf.appendMetaLatencyNanos(metadataLatencyNs); copyLength = IOUtils.copyLarge(src, dest, 0, srcKeyLen, new byte[getIOBufferSize(srcKeyLen)]); - String md5Hash = DatatypeConverter.printHexBinary(src.getMessageDigest().digest()).toLowerCase(); + final String md5Hash = DatatypeConverter.printHexBinary(src.getMessageDigest().digest()).toLowerCase(); dest.getMetadata().put(OzoneConsts.ETAG, md5Hash); } } @@ -1039,7 +1043,7 @@ private CopyObjectResponse copyObject(OzoneVolume volume, String sourceBucket = result.getLeft(); String sourceKey = result.getRight(); - DigestInputStream sourceDigestInputStream = null; + final MessageDigest md5Digest = getMD5DigestInstance(); if (S3Owner.hasBucketOwnershipVerificationConditions(getHeaders())) { String sourceBucketOwner = volume.getBucket(sourceBucket).getOwner(); @@ -1047,8 +1051,8 @@ private CopyObjectResponse copyObject(OzoneVolume volume, S3Owner.verifyBucketOwnerConditionOnCopyOperation(getHeaders(), sourceBucket, sourceBucketOwner, null, null); } try { - OzoneKeyDetails sourceKeyDetails = getClientProtocol().getKeyDetails( - volume.getName(), sourceBucket, sourceKey); + final OzoneKeyDetails sourceKeyDetails = runWithS3ActionString( + "GetObject", () -> getClientProtocol().getKeyDetails(volume.getName(), sourceBucket, sourceKey)); // Checking whether we trying to copying to it self. if (sourceBucket.equals(destBucket) && sourceKey .equals(destkey)) { @@ -1110,22 +1114,25 @@ private CopyObjectResponse copyObject(OzoneVolume volume, throw ex; } - try (OzoneInputStream src = getClientProtocol().getKey(volume.getName(), - sourceBucket, sourceKey)) { + try (OzoneInputStream src = runWithS3ActionString( + "GetObject", () -> getClientProtocol().getKey(volume.getName(), sourceBucket, sourceKey)); + DigestInputStream sourceDigestInputStream = new DigestInputStream(src, md5Digest)) { getMetrics().updateCopyKeyMetadataStats(startNanos); - sourceDigestInputStream = new DigestInputStream(src, getMD5DigestInstance()); - copy(volume, sourceDigestInputStream, sourceKeyLen, destkey, destBucket, replicationConfig, - customMetadata, perf, startNanos, tags); - } + runWithS3ActionString("PutObject", () -> { + copy(volume, sourceDigestInputStream, sourceKeyLen, destkey, destBucket, + replicationConfig, customMetadata, perf, startNanos, tags); + return null; + }); - final OzoneKeyDetails destKeyDetails = getClientProtocol().getKeyDetails( - volume.getName(), destBucket, destkey); + final OzoneKeyDetails destKeyDetails = getClientProtocol().getKeyDetails( + volume.getName(), destBucket, destkey); - getMetrics().updateCopyObjectSuccessStats(startNanos); - CopyObjectResponse copyObjectResponse = new CopyObjectResponse(); - copyObjectResponse.setETag(wrapInQuotes(destKeyDetails.getMetadata().get(OzoneConsts.ETAG))); - copyObjectResponse.setLastModified(destKeyDetails.getModificationTime()); - return copyObjectResponse; + getMetrics().updateCopyObjectSuccessStats(startNanos); + CopyObjectResponse copyObjectResponse = new CopyObjectResponse(); + copyObjectResponse.setETag(wrapInQuotes(destKeyDetails.getMetadata().get(OzoneConsts.ETAG))); + copyObjectResponse.setLastModified(destKeyDetails.getModificationTime()); + return copyObjectResponse; + } } catch (OMException ex) { if (ex.getResult() == ResultCodes.KEY_NOT_FOUND) { throw newError(S3ErrorTable.NO_SUCH_KEY, sourceKey, ex); @@ -1141,9 +1148,7 @@ private CopyObjectResponse copyObject(OzoneVolume volume, } finally { // Reset the thread-local message digest instance in case of exception // and MessageDigest#digest is never called - if (sourceDigestInputStream != null) { - sourceDigestInputStream.getMessageDigest().reset(); - } + md5Digest.reset(); } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java index 9e638a112a76..8fdb80d9a488 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java @@ -26,7 +26,6 @@ import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.s3.commontypes.BucketMetadata; import org.apache.hadoop.ozone.s3.exception.OS3Exception; -import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +47,8 @@ public class RootEndpoint extends EndpointBase { @GET public Response get() throws OS3Exception, IOException { - long startNanos = Time.monotonicNowNanos(); + S3RequestContext context = new S3RequestContext(this, S3GAction.LIST_S3_BUCKETS); + long startNanos = context.getStartNanos(); boolean auditSuccess = true; try { ListBucketResponse response = new ListBucketResponse(); @@ -73,11 +73,11 @@ public Response get() return Response.ok(response).build(); } catch (Exception ex) { auditSuccess = false; - auditReadFailure(S3GAction.LIST_S3_BUCKETS, ex); + auditReadFailure(context.getAction(), ex); throw ex; } finally { if (auditSuccess) { - auditReadSuccess(S3GAction.LIST_S3_BUCKETS); + auditReadSuccess(context.getAction()); } } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3RequestContext.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3RequestContext.java index 4130feaf6fdb..ebcb773cef45 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3RequestContext.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3RequestContext.java @@ -36,6 +36,7 @@ class S3RequestContext { this.startNanos = Time.monotonicNowNanos(); this.perf = new PerformanceStringBuilder(); this.action = action; + endpoint.applyS3Action(action); } long getStartNanos() { @@ -59,6 +60,7 @@ S3GAction getAction() { void setAction(S3GAction action) { this.action = action; + endpoint.applyS3Action(action); } /** diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3GActionIamMapper.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3GActionIamMapper.java new file mode 100644 index 000000000000..9953ebe2020b --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3GActionIamMapper.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3.util; + +import jakarta.annotation.Nullable; +import org.apache.hadoop.ozone.audit.S3GAction; + +/** + * Maps S3 Gateway operations to AWS IAM S3 action names. Values align with + * {@code org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver} so STS session + * policies and Ranger policy conditions use the same vocabulary. + */ +public final class S3GActionIamMapper { + + private S3GActionIamMapper() { + } + + /** + * @return S3 action string, or null if not applicable to IAM S3 + */ + public static @Nullable String toS3ActionString(@Nullable S3GAction action) { + if (action == null) { + return null; + } + switch (action) { + case GET_BUCKET: + case HEAD_BUCKET: + return "ListBucket"; + case CREATE_BUCKET: + return "CreateBucket"; + case DELETE_BUCKET: + return "DeleteBucket"; + case GET_ACL: + return "GetBucketAcl"; + case PUT_ACL: + return "PutBucketAcl"; + case LIST_MULTIPART_UPLOAD: + return "ListBucketMultipartUploads"; + case MULTI_DELETE: + case DELETE_KEY: + return "DeleteObject"; + case LIST_S3_BUCKETS: + return "ListAllMyBuckets"; + case CREATE_MULTIPART_KEY: + case CREATE_KEY: + case INIT_MULTIPART_UPLOAD: + case COMPLETE_MULTIPART_UPLOAD: + case CREATE_DIRECTORY: + return "PutObject"; + case LIST_PARTS: + return "ListMultipartUploadParts"; + case GET_KEY: + case HEAD_KEY: + return "GetObject"; + case ABORT_MULTIPART_UPLOAD: + return "AbortMultipartUpload"; + case GET_OBJECT_TAGGING: + return "GetObjectTagging"; + case PUT_OBJECT_TAGGING: + return "PutObjectTagging"; + case DELETE_OBJECT_TAGGING: + return "DeleteObjectTagging"; + case PUT_OBJECT_ACL: + return "PutObjectAcl"; + case COPY_OBJECT: + case CREATE_MULTIPART_KEY_BY_COPY: + // CopyObject / UploadPartCopy require distinct source (GetObject) and destination (PutObject) + // authorization. The endpoint code explicitly sets the IAM action string for each phase. + return null; + case GENERATE_SECRET: + case REVOKE_SECRET: + case ASSUME_ROLE: + default: + return null; + } + } +} diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestCopyActionsAudit.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestCopyActionsAudit.java new file mode 100644 index 000000000000..380a7780e9e6 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestCopyActionsAudit.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3.endpoint; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayInputStream; +import java.io.OutputStream; +import java.util.HashMap; +import javax.ws.rs.core.HttpHeaders; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.audit.AuditLogger.PerformanceStringBuilder; +import org.apache.hadoop.ozone.audit.S3GAction; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneClientStub; +import org.apache.hadoop.ozone.s3.endpoint.ObjectEndpoint.ObjectRequestContext; +import org.apache.hadoop.ozone.s3.util.S3Consts; +import org.junit.jupiter.api.Test; + +/** + * Verifies audit logging action for copy operations even if S3 action authorization strings are overridden internally. + * For example, S3G.COPY_OBJECT must use S3G.COPY_OBJECT as the audit action, even though internally the S3 actions + * checked are GetObject and PutObject. + */ +public class TestCopyActionsAudit { + + @Test + public void testCopyObjectAuditActionRemainsCopyObject() throws Exception { + final String bucketName = OzoneConsts.S3_BUCKET; + final String srcKey = "src.txt"; + final String destKey = "dest.txt"; + + final OzoneClient client = new OzoneClientStub(); + client.getObjectStore().createS3Bucket(bucketName); + final OzoneBucket bucket = client.getObjectStore().getS3Bucket(bucketName); + + try (OutputStream out = bucket.createKey( + srcKey, 3, ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), + new HashMap<>())) { + out.write("src".getBytes(UTF_8)); + } + + final HttpHeaders headers = mock(HttpHeaders.class); + when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn("STANDARD"); + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature"); + when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(bucketName + "/" + srcKey); + when(headers.getHeaderString(HttpHeaders.CONTENT_LENGTH)).thenReturn("0"); + + final ObjectEndpoint endpoint = newEndpoint(client, headers); + final AuditingObjectOperationHandler auditing = spy(new AuditingObjectOperationHandler(endpoint)); + + final ObjectRequestContext requestContext = endpoint.new ObjectRequestContext(S3GAction.CREATE_KEY, bucketName); + + auditing.handlePutRequest(requestContext, destKey, new ByteArrayInputStream(new byte[0])); + + verify(auditing).auditWriteSuccess(eq(S3GAction.COPY_OBJECT), any(PerformanceStringBuilder.class)); + } + + @Test + public void testUploadPartCopyAuditActionRemainsCreateMultipartKeyByCopy() throws Exception { + final String bucketName = OzoneConsts.S3_BUCKET; + final String srcKey = "src-part.txt"; + final String destKey = "dest-mpu.txt"; + + final OzoneClient client = new OzoneClientStub(); + client.getObjectStore().createS3Bucket(bucketName); + final OzoneBucket bucket = client.getObjectStore().getS3Bucket(bucketName); + + try (OutputStream out = bucket.createKey( + srcKey, 4, ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), + new HashMap<>())) { + out.write("part".getBytes(UTF_8)); + } + + final HttpHeaders headers = mock(HttpHeaders.class); + when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn("STANDARD"); + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature"); + when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(bucketName + "/" + srcKey); + when(headers.getHeaderString(HttpHeaders.CONTENT_LENGTH)).thenReturn("0"); + + final ObjectEndpoint endpoint = newEndpoint(client, headers); + + final String uploadId = EndpointTestUtils.initiateMultipartUpload(endpoint, bucketName, destKey); + assertNotNull(uploadId); + + endpoint.queryParamsForTest().set(S3Consts.QueryParams.UPLOAD_ID, uploadId); + endpoint.queryParamsForTest().setInt(S3Consts.QueryParams.PART_NUMBER, 1); + + final AuditingObjectOperationHandler auditing = spy(new AuditingObjectOperationHandler(endpoint)); + final ObjectRequestContext requestContext = endpoint.new ObjectRequestContext(S3GAction.CREATE_KEY, bucketName); + + auditing.handlePutRequest(requestContext, destKey, new ByteArrayInputStream(new byte[0])); + + verify(auditing).auditWriteSuccess(eq(S3GAction.CREATE_MULTIPART_KEY_BY_COPY), any(PerformanceStringBuilder.class)); + } + + private static ObjectEndpoint newEndpoint(OzoneClient client, HttpHeaders headers) { + return EndpointBuilder.newObjectEndpointBuilder() + .setClient(client) + .setHeaders(headers) + .build(); + } +} + diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3GActionIamMapper.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3GActionIamMapper.java new file mode 100644 index 000000000000..c7ae9e4e924c --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3GActionIamMapper.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.apache.hadoop.ozone.audit.S3GAction; +import org.junit.jupiter.api.Test; + +/** Unit tests for {@link S3GActionIamMapper}. */ +public class TestS3GActionIamMapper { + + @Test + public void mapsCoreObjectActions() { + assertEquals("ListBucket", S3GActionIamMapper.toS3ActionString(S3GAction.GET_BUCKET)); + assertEquals("ListBucket", S3GActionIamMapper.toS3ActionString(S3GAction.HEAD_BUCKET)); + assertEquals("CreateBucket", S3GActionIamMapper.toS3ActionString(S3GAction.CREATE_BUCKET)); + assertEquals("DeleteBucket", S3GActionIamMapper.toS3ActionString(S3GAction.DELETE_BUCKET)); + assertEquals("GetBucketAcl", S3GActionIamMapper.toS3ActionString(S3GAction.GET_ACL)); + assertEquals("PutBucketAcl", S3GActionIamMapper.toS3ActionString(S3GAction.PUT_ACL)); + assertEquals("ListBucketMultipartUploads", S3GActionIamMapper.toS3ActionString(S3GAction.LIST_MULTIPART_UPLOAD)); + assertEquals("DeleteObject", S3GActionIamMapper.toS3ActionString(S3GAction.MULTI_DELETE)); + assertEquals("DeleteObject", S3GActionIamMapper.toS3ActionString(S3GAction.DELETE_KEY)); + assertEquals("ListAllMyBuckets", S3GActionIamMapper.toS3ActionString(S3GAction.LIST_S3_BUCKETS)); + assertEquals("PutObject", S3GActionIamMapper.toS3ActionString(S3GAction.CREATE_MULTIPART_KEY)); + assertEquals("PutObject", S3GActionIamMapper.toS3ActionString(S3GAction.CREATE_KEY)); + assertEquals("PutObject", S3GActionIamMapper.toS3ActionString(S3GAction.INIT_MULTIPART_UPLOAD)); + assertEquals("PutObject", S3GActionIamMapper.toS3ActionString(S3GAction.COMPLETE_MULTIPART_UPLOAD)); + assertEquals("PutObject", S3GActionIamMapper.toS3ActionString(S3GAction.CREATE_DIRECTORY)); + assertEquals("ListMultipartUploadParts", S3GActionIamMapper.toS3ActionString(S3GAction.LIST_PARTS)); + assertEquals("GetObject", S3GActionIamMapper.toS3ActionString(S3GAction.GET_KEY)); + assertEquals("GetObject", S3GActionIamMapper.toS3ActionString(S3GAction.HEAD_KEY)); + assertEquals("AbortMultipartUpload", S3GActionIamMapper.toS3ActionString(S3GAction.ABORT_MULTIPART_UPLOAD)); + assertEquals("GetObjectTagging", S3GActionIamMapper.toS3ActionString(S3GAction.GET_OBJECT_TAGGING)); + assertEquals("PutObjectTagging", S3GActionIamMapper.toS3ActionString(S3GAction.PUT_OBJECT_TAGGING)); + assertEquals("DeleteObjectTagging", S3GActionIamMapper.toS3ActionString(S3GAction.DELETE_OBJECT_TAGGING)); + assertEquals("PutObjectAcl", S3GActionIamMapper.toS3ActionString(S3GAction.PUT_OBJECT_ACL)); + } + + @Test + public void copyActionsReturnNull() { + assertNull(S3GActionIamMapper.toS3ActionString(S3GAction.COPY_OBJECT)); + assertNull(S3GActionIamMapper.toS3ActionString(S3GAction.CREATE_MULTIPART_KEY_BY_COPY)); + } + + @Test + public void nonIamActionsReturnNull() { + assertNull(S3GActionIamMapper.toS3ActionString(S3GAction.ASSUME_ROLE)); + assertNull(S3GActionIamMapper.toS3ActionString(S3GAction.GENERATE_SECRET)); + assertNull(S3GActionIamMapper.toS3ActionString(S3GAction.REVOKE_SECRET)); + } +} diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/package-info.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/package-info.java new file mode 100644 index 000000000000..4b7b37a574a1 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Unit tests for s3 utilities. + */ +package org.apache.hadoop.ozone.s3.util; From 8e717e048bfd3470bf0c907e0293e7e7b7fc121f Mon Sep 17 00:00:00 2001 From: Fabian Morgan Date: Tue, 5 May 2026 15:39:39 -0700 Subject: [PATCH 3/3] update IamSessionPolicyResolver to return S3 Actions --- .../acl/iam/IamSessionPolicyResolver.java | 250 ++++-- .../acl/iam/TestIamSessionPolicyResolver.java | 811 +++++++++++------- 2 files changed, 692 insertions(+), 369 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/iam/IamSessionPolicyResolver.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/iam/IamSessionPolicyResolver.java index 7421aab3253a..4e591d14b5f0 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/iam/IamSessionPolicyResolver.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/iam/IamSessionPolicyResolver.java @@ -135,6 +135,7 @@ public static Set resolve(String policyJson, Strin // Accumulate ACLs across ALL statements using a single map to allow // cross-statement deduplication and ALL-permission collapsing. final Map> objToAclsMap = new LinkedHashMap<>(); + final Map> objToActionsMap = new LinkedHashMap<>(); // Parse JSON into set of statements final Set statements = parseJsonAndRetrieveStatements(policyJson); @@ -168,11 +169,12 @@ public static Set resolve(String policyJson, Strin final Set resourceSpecs = validateAndCategorizeResources(authorizerType, resources); // For each action, map to Ozone objects (paths) and acls based on resource specs and prefixes - createPathsAndPermissions(volumeName, authorizerType, filteredS3Actions, resourceSpecs, condition, objToAclsMap); + createPathsAndPermissions( + volumeName, authorizerType, filteredS3Actions, resourceSpecs, condition, objToAclsMap, objToActionsMap); } - // Group accumulated objects by their ACL sets to create final result - return groupObjectsByAcls(objToAclsMap); + // Group accumulated objects by their ACL sets and S3 actions to create final result + return groupObjectsByAclsAndActions(objToAclsMap, objToActionsMap); } /** @@ -356,7 +358,8 @@ static Set mapPolicyActionsToS3Actions(Set actions) { final Set mappedActions = new LinkedHashSet<>(); for (String action : actions) { if ("s3:*".equalsIgnoreCase(action)) { - return EnumSet.of(S3Action.ALL_S3); + // Expand into all supported concrete actions + return EnumSet.allOf(S3Action.class); } // Unsupported actions are silently ignored @@ -377,7 +380,7 @@ private static Set filterActionsWhenConditionPresent(Set map return mappedS3Actions; } - if (mappedS3Actions.contains(S3Action.LIST_BUCKET) || mappedS3Actions.contains(S3Action.ALL_S3)) { + if (mappedS3Actions.contains(S3Action.LIST_BUCKET)) { final Set filteredActions = new HashSet<>(); filteredActions.add(S3Action.LIST_BUCKET); return filteredActions; @@ -457,30 +460,68 @@ static Set validateAndCategorizeResources(AuthorizerType authorize */ @VisibleForTesting static void createPathsAndPermissions(String volumeName, AuthorizerType authorizerType, Set mappedS3Actions, - Set resourceSpecs, Condition condition, Map> objToAclsMap) { + Set resourceSpecs, Condition condition, Map> objToAclsMap, + Map> objToActionsMap) { // Process each resource spec with the given actions for (ResourceSpec resourceSpec : resourceSpecs) { processResourceSpecWithActions( - volumeName, authorizerType, mappedS3Actions, resourceSpec, condition, objToAclsMap); + volumeName, authorizerType, mappedS3Actions, resourceSpec, condition, objToAclsMap, objToActionsMap); } } /** - * Groups objects by their ACL sets. + * Groups objects by their ACL sets and S3 actions. */ @VisibleForTesting - static Set groupObjectsByAcls(Map> objToAclsMap) { - final Map, Set> groupMap = new LinkedHashMap<>(); + static Set groupObjectsByAclsAndActions(Map> objToAclsMap, + Map> objToActionsMap) { - // Group objects by their ACL sets only (across resource types) - objToAclsMap.forEach((obj, acls) -> - groupMap.computeIfAbsent(acls, k -> new LinkedHashSet<>()).add(obj)); + // Composite key to group by both ACLs and S3 actions + class GrantKey { + private final Set acls; + private final Set actions; + + GrantKey(Set acls, Set actions) { + this.acls = acls; + this.actions = actions; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + GrantKey grantKey = (GrantKey) o; + return Objects.equals(acls, grantKey.acls) && Objects.equals(actions, grantKey.actions); + } + + @Override + public int hashCode() { + return Objects.hash(acls, actions); + } + } + + final Map> groupMap = new LinkedHashMap<>(); + + // Group objects by their ACL sets and S3 actions + objToAclsMap.forEach((obj, acls) -> { + Set actions = objToActionsMap.getOrDefault(obj, Collections.emptySet()); + if (actions.size() == S3Action.values().length) { + // An empty set of actions means all actions are allowed + actions = Collections.emptySet(); + } + final GrantKey key = new GrantKey(acls, actions); + groupMap.computeIfAbsent(key, k -> new LinkedHashSet<>()).add(obj); + }); // Convert to result format, filtering out entries with empty ACLs final Set result = new LinkedHashSet<>(); groupMap.forEach((key, objs) -> { - if (!key.isEmpty()) { - result.add(new AssumeRoleRequest.OzoneGrant(objs, key)); + if (!key.acls.isEmpty()) { + result.add(new AssumeRoleRequest.OzoneGrant(objs, key.acls, key.actions)); } }); @@ -493,7 +534,7 @@ static Set groupObjectsByAcls(Map mappedS3Actions, ResourceSpec resourceSpec, Condition condition, - Map> objToAclsMap) { + Map> objToAclsMap, Map> objToActionsMap) { // Process based on ResourceSpec type switch (resourceSpec.type) { @@ -501,31 +542,35 @@ private static void processResourceSpecWithActions(String volumeName, Authorizer Preconditions.checkArgument( authorizerType != AuthorizerType.NATIVE, "ResourceSpec type ANY not supported for OzoneNativeAuthorizer"); - processResourceTypeAny(volumeName, authorizerType, mappedS3Actions, condition, objToAclsMap); + processResourceTypeAny(volumeName, authorizerType, mappedS3Actions, condition, objToAclsMap, objToActionsMap); break; case BUCKET: - processBucketResource(volumeName, mappedS3Actions, resourceSpec, condition, authorizerType, objToAclsMap); + processBucketResource( + volumeName, mappedS3Actions, resourceSpec, condition, authorizerType, objToAclsMap, objToActionsMap); break; case BUCKET_WILDCARD: Preconditions.checkArgument( authorizerType != AuthorizerType.NATIVE, "ResourceSpec type BUCKET_WILDCARD not supported for OzoneNativeAuthorizer"); - processBucketResource(volumeName, mappedS3Actions, resourceSpec, condition, authorizerType, objToAclsMap); + processBucketResource( + volumeName, mappedS3Actions, resourceSpec, condition, authorizerType, objToAclsMap, objToActionsMap); break; case OBJECT_EXACT: - processObjectExactResource(volumeName, mappedS3Actions, resourceSpec, objToAclsMap); + processObjectExactResource(volumeName, mappedS3Actions, resourceSpec, objToAclsMap, objToActionsMap); break; case OBJECT_PREFIX: Preconditions.checkArgument( authorizerType != AuthorizerType.RANGER, "ResourceSpec type OBJECT_PREFIX not supported for RangerOzoneAuthorizer"); - processObjectPrefixResource(volumeName, authorizerType, mappedS3Actions, resourceSpec, objToAclsMap); + processObjectPrefixResource( + volumeName, authorizerType, mappedS3Actions, resourceSpec, objToAclsMap, objToActionsMap); break; case OBJECT_PREFIX_WILDCARD: Preconditions.checkArgument( authorizerType != AuthorizerType.NATIVE, "ResourceSpec type OBJECT_PREFIX_WILDCARD not supported for OzoneNativeAuthorizer"); - processObjectPrefixResource(volumeName, authorizerType, mappedS3Actions, resourceSpec, objToAclsMap); + processObjectPrefixResource( + volumeName, authorizerType, mappedS3Actions, resourceSpec, objToAclsMap, objToActionsMap); break; default: throw new IllegalStateException("Unexpected resourceSpec type found: " + resourceSpec.type); @@ -537,22 +582,34 @@ private static void processResourceSpecWithActions(String volumeName, Authorizer * Example: "Resource": "*" */ private static void processResourceTypeAny(String volumeName, AuthorizerType authorizerType, - Set mappedS3Actions, Condition condition, Map> objToAclsMap) { + Set mappedS3Actions, Condition condition, Map> objToAclsMap, + Map> objToActionsMap) { + final IOzoneObj volumeObj = volumeObj(volumeName); + final IOzoneObj bucketObj = bucketObj(volumeName, "*"); + final IOzoneObj keyObj = keyObj(volumeName, "*", "*"); for (S3Action action : mappedS3Actions) { - addAclsForObj(objToAclsMap, volumeObj(volumeName), action.volumePerms); - addAclsForObj(objToAclsMap, bucketObj(volumeName, "*"), action.bucketPerms); + addAclsForObj(objToAclsMap, volumeObj, action.volumePerms); + addAclsForObj(objToAclsMap, bucketObj, action.bucketPerms); + if (condition != null && condition.prefixes != null && !condition.prefixes.isEmpty() && - (action == S3Action.LIST_BUCKET || action == S3Action.ALL_S3)) { + action == S3Action.LIST_BUCKET) { + + // Ensure the volume and bucket get the action + addActionForKind(objToActionsMap, action, volumeObj, bucketObj, null); + for (String prefix : condition.prefixes) { // If operator is StringEquals, ignore wildcard prefixes - this is AWS behavior if (STRING_EQUALS.equals(condition.operator) && hasWildcard(prefix)) { continue; } - createObjectResourcesFromConditionPrefix( + + final IOzoneObj listObj = createObjectResourcesFromConditionPrefix( volumeName, authorizerType, ResourceSpec.any(), prefix, objToAclsMap, EnumSet.of(READ)); + addActionForKind(objToActionsMap, action, null, null, listObj); } } else { - addAclsForObj(objToAclsMap, keyObj(volumeName, "*", "*"), action.objectPerms); + addAclsForObj(objToAclsMap, keyObj, action.objectPerms); + addActionForKind(objToActionsMap, action, volumeObj, bucketObj, keyObj); } } } @@ -564,7 +621,9 @@ private static void processResourceTypeAny(String volumeName, AuthorizerType aut */ private static void processBucketResource(String volumeName, Set mappedS3Actions, ResourceSpec resourceSpec, Condition condition, AuthorizerType authorizerType, - Map> objToAclsMap) { + Map> objToAclsMap, Map> objToActionsMap) { + final IOzoneObj volumeObj = volumeObj(volumeName); + final IOzoneObj bucketObj = bucketObj(volumeName, resourceSpec.bucket); for (S3Action action : mappedS3Actions) { // The s3:ListAllMyBuckets action can use either "*" or // "arn:aws:s3:::*" as its Resource. The former is already handled via the @@ -574,36 +633,31 @@ private static void processBucketResource(String volumeName, Set mappe // actions (currently s3:ListAllMyBuckets). if (action.kind == ActionKind.BUCKET || (action.kind == ActionKind.VOLUME && "*".equals(resourceSpec.bucket))) { // this handles s3:ListAllMyBuckets - addAclsForObj(objToAclsMap, volumeObj(volumeName), action.volumePerms); - addAclsForObj(objToAclsMap, bucketObj(volumeName, resourceSpec.bucket), action.bucketPerms); - } else if (action == S3Action.ALL_S3) { - // For s3:*, ALL should only apply at the bucket level; grant READ at volume for navigation - // However, resource "arn:aws:s3:::*" can apply to volume as well (as explained above) - // If the bucket is "*", include the volumePerms, otherwise just include READ for navigation. - if ("*".equals(resourceSpec.bucket)) { - addAclsForObj(objToAclsMap, volumeObj(volumeName), action.volumePerms); - } else { - addAclsForObj(objToAclsMap, volumeObj(volumeName), EnumSet.of(READ)); - } - addAclsForObj(objToAclsMap, bucketObj(volumeName, resourceSpec.bucket), action.bucketPerms); + addAclsForObj(objToAclsMap, volumeObj, action.volumePerms); + addAclsForObj(objToAclsMap, bucketObj, action.bucketPerms); + addActionForKind(objToActionsMap, action, volumeObj, bucketObj, null); } - if (action == S3Action.LIST_BUCKET || action == S3Action.ALL_S3) { + if (action == S3Action.LIST_BUCKET) { // If condition prefixes are present, these would constrain the object permissions if the action - // is s3:ListBucket or s3:* (which includes s3:ListBucket) + // is s3:ListBucket if (condition != null && condition.prefixes != null && !condition.prefixes.isEmpty()) { for (String prefix : condition.prefixes) { // If operator is StringEquals, we should ignore any prefix containing wildcards - this is AWS behavior if (STRING_EQUALS.equals(condition.operator) && hasWildcard(prefix)) { continue; } - createObjectResourcesFromConditionPrefix( + final IOzoneObj listObj = createObjectResourcesFromConditionPrefix( volumeName, authorizerType, resourceSpec, prefix, objToAclsMap, EnumSet.of(READ)); + // Add action for the key/prefix + addActionForKind(objToActionsMap, action, null, null, listObj); } } else if (condition == null) { // No condition prefixes, but we need READ access to all objects, so use "*" as the prefix - createObjectResourcesFromConditionPrefix( + final IOzoneObj readObj = createObjectResourcesFromConditionPrefix( volumeName, authorizerType, resourceSpec, "*", objToAclsMap, EnumSet.of(READ)); + // Add action for the key/prefix + addActionForKind(objToActionsMap, action, null, null, readObj); } } } @@ -614,17 +668,17 @@ private static void processBucketResource(String volumeName, Set mappe * Example: "Resource": "arn:aws:s3:::my-bucket/file.txt" */ private static void processObjectExactResource(String volumeName, Set mappedS3Actions, - ResourceSpec resourceSpec, Map> objToAclsMap) { + ResourceSpec resourceSpec, Map> objToAclsMap, + Map> objToActionsMap) { + final IOzoneObj volumeObj = volumeObj(volumeName); + final IOzoneObj bucketObj = bucketObj(volumeName, resourceSpec.bucket); + final IOzoneObj keyObj = keyObj(volumeName, resourceSpec.bucket, resourceSpec.key); for (S3Action action : mappedS3Actions) { if (action.kind == ActionKind.OBJECT) { - addAclsForObj(objToAclsMap, volumeObj(volumeName), action.volumePerms); - addAclsForObj(objToAclsMap, bucketObj(volumeName, resourceSpec.bucket), action.bucketPerms); - addAclsForObj(objToAclsMap, keyObj(volumeName, resourceSpec.bucket, resourceSpec.key), action.objectPerms); - } else if (action == S3Action.ALL_S3) { - addAclsForObj(objToAclsMap, volumeObj(volumeName), EnumSet.of(READ)); - // For s3:*, ALL should only apply at the object level; grant READ at bucket level for navigation - addAclsForObj(objToAclsMap, bucketObj(volumeName, resourceSpec.bucket), EnumSet.of(READ)); - addAclsForObj(objToAclsMap, keyObj(volumeName, resourceSpec.bucket, resourceSpec.key), action.objectPerms); + addAclsForObj(objToAclsMap, volumeObj, action.volumePerms); + addAclsForObj(objToAclsMap, bucketObj, action.bucketPerms); + addAclsForObj(objToAclsMap, keyObj, action.objectPerms); + addActionForKind(objToActionsMap, action, volumeObj, bucketObj, keyObj); } } } @@ -635,22 +689,20 @@ private static void processObjectExactResource(String volumeName, Set * Example: "Resource": "arn:aws:s3:::my-bucket/path/folder" */ private static void processObjectPrefixResource(String volumeName, AuthorizerType authorizerType, - Set mappedS3Actions, ResourceSpec resourceSpec, Map> objToAclsMap) { + Set mappedS3Actions, ResourceSpec resourceSpec, Map> objToAclsMap, + Map> objToActionsMap) { + final IOzoneObj volumeObj = volumeObj(volumeName); + final IOzoneObj bucketObj = bucketObj(volumeName, resourceSpec.bucket); for (S3Action action : mappedS3Actions) { // Object actions apply to prefix/key resources - ensure to add the acls only for the appropriate action type if (action.kind == ActionKind.OBJECT) { - addAclsForObj(objToAclsMap, volumeObj(volumeName), action.volumePerms); - addAclsForObj(objToAclsMap, bucketObj(volumeName, resourceSpec.bucket), action.bucketPerms); + addAclsForObj(objToAclsMap, volumeObj, action.volumePerms); + addAclsForObj(objToAclsMap, bucketObj, action.bucketPerms); // Handle the resource prefix itself (e.g., my-bucket/*) createObjectResourcesFromResourcePrefix( - volumeName, authorizerType, resourceSpec, objToAclsMap, action.objectPerms); - } else if (action == S3Action.ALL_S3) { - addAclsForObj(objToAclsMap, volumeObj(volumeName), EnumSet.of(READ)); - // For s3:*, ALL should only apply at the object/prefix level; grant READ at bucket level for navigation - addAclsForObj(objToAclsMap, bucketObj(volumeName, resourceSpec.bucket), EnumSet.of(READ)); - // Handle the resource prefix itself (e.g., my-bucket/*) - createObjectResourcesFromResourcePrefix( - volumeName, authorizerType, resourceSpec, objToAclsMap, action.objectPerms); + volumeName, authorizerType, resourceSpec, objToAclsMap, objToActionsMap, action.objectPerms, action.name); + // Object-level action was already applied inside createObjectResourcesFromResourcePrefix. + addActionForKind(objToActionsMap, action, volumeObj, bucketObj, null); } } } @@ -659,20 +711,23 @@ private static void processObjectPrefixResource(String volumeName, AuthorizerTyp * Creates object resources from resource prefix (e.g., my-bucket/*). */ private static void createObjectResourcesFromResourcePrefix(String volumeName, AuthorizerType authorizerType, - ResourceSpec resourceSpec, Map> objToAclsMap, Set acls) { + ResourceSpec resourceSpec, Map> objToAclsMap, + Map> objToActionsMap, Set acls, String actionName) { if (authorizerType == AuthorizerType.NATIVE) { final IOzoneObj prefixObj = prefixObj(volumeName, resourceSpec.bucket, resourceSpec.prefix); addAclsForObj(objToAclsMap, prefixObj, acls); + addActionForObj(objToActionsMap, prefixObj, actionName); } else { final IOzoneObj keyObj = keyObj(volumeName, resourceSpec.bucket, resourceSpec.prefix); addAclsForObj(objToAclsMap, keyObj, acls); + addActionForObj(objToActionsMap, keyObj, actionName); } } /** * Creates object resources from condition prefixes (i.e. the s3:prefix conditions). */ - private static void createObjectResourcesFromConditionPrefix(String volumeName, AuthorizerType authorizerType, + private static IOzoneObj createObjectResourcesFromConditionPrefix(String volumeName, AuthorizerType authorizerType, ResourceSpec resourceSpec, String conditionPrefix, Map> objToAclsMap, Set acls) { if (authorizerType == AuthorizerType.NATIVE) { // For native authorizer, use PREFIX resource type with normalized prefix. @@ -686,12 +741,40 @@ private static void createObjectResourcesFromConditionPrefix(String volumeName, } final IOzoneObj prefixObj = prefixObj(volumeName, resourceSpec.bucket, normalizedPrefix); addAclsForObj(objToAclsMap, prefixObj, acls); + return prefixObj; } else { // For Ranger authorizer, use KEY resource type with original prefix // Map "x" in condition list prefix to "x". Map "x/*" in condition list prefix to "x/*". // Map "* in condition list prefix to "*". final IOzoneObj keyObj = keyObj(volumeName, resourceSpec.bucket, conditionPrefix); addAclsForObj(objToAclsMap, keyObj, acls); + return keyObj; + } + } + + private static void addActionForKind(Map> objToActionsMap, S3Action action, + IOzoneObj volumeObj, IOzoneObj bucketObj, IOzoneObj objectObj) { + if (action == null) { + return; + } + final ActionKind kind = action.kind; + if (kind == ActionKind.VOLUME) { + if (volumeObj != null) { + addActionForObj(objToActionsMap, volumeObj, action.name); + } + return; + } + + if (volumeObj != null) { + addActionForObj(objToActionsMap, volumeObj, action.name); + } + if (bucketObj != null) { + addActionForObj(objToActionsMap, bucketObj, action.name); + } + if (objectObj != null) { + if (kind == ActionKind.OBJECT || action == S3Action.LIST_BUCKET) { + addActionForObj(objToActionsMap, objectObj, action.name); + } } } @@ -720,6 +803,31 @@ private static void addAclsForObj(Map> objToAclsMap, IOz } } + /** + * Helper method to add an S3 action for an IOzoneObj. It basically strips off + * the s3: prefix, such that s3:GetObject becomes GetObject. + */ + private static String normalizeS3ActionForGrant(String action) { + if (action == null) { + return null; + } + if (action.isEmpty()) { + return action; + } + if (action.regionMatches(true, 0, "s3:", 0, 3)) { + return action.substring(3); + } + return action; + } + + private static void addActionForObj(Map> objToActionsMap, IOzoneObj obj, String action) { + final String normalizedAction = normalizeS3ActionForGrant(action); + if (normalizedAction != null && !normalizedAction.isEmpty()) { + final OzoneObj ozoneObj = (OzoneObj) obj; + objToActionsMap.computeIfAbsent(ozoneObj, k -> new LinkedHashSet<>()).add(normalizedAction); + } + } + /** * The authorizer type, whether for OzoneNativeAuthorizer or RangerOzoneAuthorizer. * The IOzoneObjs generated differ in certain cases depending on the type. @@ -736,8 +844,7 @@ public enum AuthorizerType { private enum ActionKind { VOLUME, BUCKET, - OBJECT, - ALL + OBJECT } /** @@ -896,10 +1003,7 @@ enum S3Action { PUT_OBJECT("s3:PutObject", ActionKind.OBJECT, EnumSet.of(READ), EnumSet.of(READ), EnumSet.of(CREATE, ACLType.WRITE)), PUT_OBJECT_TAGGING("s3:PutObjectTagging", ActionKind.OBJECT, EnumSet.of(READ), EnumSet.of(READ), - EnumSet.of(ACLType.WRITE)), - - // Wildcard all - ALL_S3("s3:*", ActionKind.ALL, EnumSet.of(READ, LIST), EnumSet.of(ACLType.ALL), EnumSet.of(ACLType.ALL)); + EnumSet.of(ACLType.WRITE)); private final String name; private final ActionKind kind; diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/acl/iam/TestIamSessionPolicyResolver.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/acl/iam/TestIamSessionPolicyResolver.java index 1d9765146325..217fc341b1c2 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/acl/iam/TestIamSessionPolicyResolver.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/acl/iam/TestIamSessionPolicyResolver.java @@ -22,7 +22,6 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION; import static org.apache.hadoop.ozone.security.acl.AssumeRoleRequest.OzoneGrant; import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType; -import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.ALL; import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.CREATE; import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.DELETE; import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.LIST; @@ -32,16 +31,32 @@ import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.WRITE_ACL; import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.AuthorizerType.NATIVE; import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.AuthorizerType.RANGER; +import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3Action.ABORT_MULTIPART_UPLOAD; +import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3Action.CREATE_BUCKET; +import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3Action.DELETE_BUCKET; +import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3Action.DELETE_OBJECT; +import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3Action.DELETE_OBJECT_TAGGING; +import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3Action.GET_BUCKET_ACL; +import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3Action.GET_OBJECT; +import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3Action.GET_OBJECT_TAGGING; +import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3Action.LIST_ALL_MY_BUCKETS; +import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3Action.LIST_BUCKET; +import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3Action.LIST_BUCKET_MULTIPART_UPLOADS; +import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3Action.LIST_MULTIPART_UPLOAD_PARTS; +import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3Action.PUT_BUCKET_ACL; +import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3Action.PUT_OBJECT; +import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3Action.PUT_OBJECT_TAGGING; import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3ResourceType; import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.buildCaseInsensitiveS3ActionMap; import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.createPathsAndPermissions; -import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.groupObjectsByAcls; +import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.groupObjectsByAclsAndActions; import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.mapPolicyActionsToS3Actions; import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.resolve; import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.validateAndCategorizeResources; import static org.assertj.core.api.Assertions.assertThat; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.Map; @@ -61,6 +76,36 @@ public class TestIamSessionPolicyResolver { private static final String VOLUME = "s3v"; + private static final Set ALL_OBJECT_ACTIONS = strSet( + "AbortMultipartUpload", "DeleteObject", "DeleteObjectTagging", "GetObject", "GetObjectTagging", + "ListMultipartUploadParts", "PutObject", "PutObjectTagging"); + + private static final Set ALL_BUCKET_ACTIONS = strSet( + "CreateBucket", "DeleteBucket", "GetBucketAcl", "ListBucket", "ListBucketMultipartUploads", "PutBucketAcl"); + + private static final Set ALL_BUCKET_ACTIONS_WITH_LIST_ALL_MY_BUCKETS; + + static { + final Set tempSet = new HashSet<>(ALL_BUCKET_ACTIONS); + tempSet.add("ListAllMyBuckets"); + ALL_BUCKET_ACTIONS_WITH_LIST_ALL_MY_BUCKETS = Collections.unmodifiableSet(tempSet); + } + + private static final Set ALL_BUCKET_AND_OBJECT_ACTIONS; + + static { + final Set tempSet = new HashSet<>(ALL_BUCKET_ACTIONS); + tempSet.addAll(ALL_OBJECT_ACTIONS); + ALL_BUCKET_AND_OBJECT_ACTIONS = Collections.unmodifiableSet(tempSet); + } + + private static final Set ALL_OBJECT_ACTIONS_WITH_LIST_BUCKET; + + static { + final Set tempSet = new HashSet<>(ALL_OBJECT_ACTIONS); + tempSet.add("ListBucket"); + ALL_OBJECT_ACTIONS_WITH_LIST_BUCKET = Collections.unmodifiableSet(tempSet); + } @Test public void testUnsupportedConditionOperatorThrows() { @@ -302,42 +347,38 @@ public void testBuildCaseInsensitiveS3ActionMap() { // Verify that wildcard actions are present assertThat(caseInsensitiveS3ActionMap).containsKeys( - "s3:*", "s3:get*", "s3:put*", "s3:list*", "s3:delete*", "s3:create*"); + "s3:get*", "s3:put*", "s3:list*", "s3:delete*", "s3:create*"); // Verify s3:Get* contains Get actions final Set getActions = caseInsensitiveS3ActionMap.get("s3:get*"); - assertThat(getActions).containsOnly( - S3Action.GET_OBJECT, S3Action.GET_BUCKET_ACL, S3Action.GET_OBJECT_TAGGING); + assertThat(getActions).containsOnly(GET_OBJECT, GET_BUCKET_ACL, GET_OBJECT_TAGGING); // Verify s3:Put* contains Put actions final Set putActions = caseInsensitiveS3ActionMap.get("s3:put*"); - assertThat(putActions).containsOnly( - S3Action.PUT_OBJECT, S3Action.PUT_OBJECT_TAGGING, S3Action.PUT_BUCKET_ACL); + assertThat(putActions).containsOnly(PUT_OBJECT, PUT_OBJECT_TAGGING, PUT_BUCKET_ACL); // Verify s3:List* contains List actions final Set listActions = caseInsensitiveS3ActionMap.get("s3:list*"); assertThat(listActions).containsOnly( - S3Action.LIST_BUCKET, S3Action.LIST_ALL_MY_BUCKETS, S3Action.LIST_BUCKET_MULTIPART_UPLOADS, - S3Action.LIST_MULTIPART_UPLOAD_PARTS); + LIST_BUCKET, LIST_ALL_MY_BUCKETS, LIST_BUCKET_MULTIPART_UPLOADS, LIST_MULTIPART_UPLOAD_PARTS); // Verify s3:Delete* contains Delete actions final Set deleteActions = caseInsensitiveS3ActionMap.get("s3:delete*"); - assertThat(deleteActions).containsOnly( - S3Action.DELETE_OBJECT, S3Action.DELETE_BUCKET, S3Action.DELETE_OBJECT_TAGGING); + assertThat(deleteActions).containsOnly(DELETE_OBJECT, DELETE_BUCKET, DELETE_OBJECT_TAGGING); // Verify s3:Create* contains Create actions final Set createActions = caseInsensitiveS3ActionMap.get("s3:create*"); - assertThat(createActions).containsOnly(S3Action.CREATE_BUCKET); + assertThat(createActions).containsOnly(CREATE_BUCKET); } @Test public void testBuildCaseInsensitiveS3ActionMapIndividualActionsContainSingleEntry() { final Map> actionMap = buildCaseInsensitiveS3ActionMap(); - + // Individual actions should map to a set with exactly one entry final Set listBucketAction = actionMap.get("s3:listbucket"); assertThat(listBucketAction).hasSize(1); - + final Set getObjectAction = actionMap.get("s3:getobject"); assertThat(getObjectAction).hasSize(1); } @@ -357,50 +398,53 @@ public void testMapPolicyActionsToS3ActionsWithEmptyListReturnsEmpty() { @Test public void testMapPolicyActionsToS3ActionsWithSingleActionMapsCorrectly() { final Set listBucket = mapPolicyActionsToS3Actions(Collections.singleton("s3:ListBucket")); - assertThat(listBucket).containsOnly(S3Action.LIST_BUCKET); + assertThat(listBucket).containsOnly(LIST_BUCKET); // Ensure case-insensitive action works final Set listBucketCi = mapPolicyActionsToS3Actions(Collections.singleton("S3:ListBuCKet")); - assertThat(listBucketCi).containsOnly(S3Action.LIST_BUCKET); + assertThat(listBucketCi).containsOnly(LIST_BUCKET); final Set deleteObject = mapPolicyActionsToS3Actions(Collections.singleton("s3:DeleteObject")); - assertThat(deleteObject).containsOnly(S3Action.DELETE_OBJECT); + assertThat(deleteObject).containsOnly(DELETE_OBJECT); // Ensure case-insensitive action works final Set deleteObjectCi = mapPolicyActionsToS3Actions(Collections.singleton("S3:DeLETeObjeCT")); - assertThat(deleteObjectCi).containsOnly(S3Action.DELETE_OBJECT); + assertThat(deleteObjectCi).containsOnly(DELETE_OBJECT); } @Test public void testMapPolicyActionsToS3ActionsWithMultipleActionsMapAllCorrectly() { final Set result = mapPolicyActionsToS3Actions(strSet("s3:ListBucket", "s3:GetObject", "s3:PutObject")); - assertThat(result).containsOnly(S3Action.LIST_BUCKET, S3Action.GET_OBJECT, S3Action.PUT_OBJECT); + assertThat(result).containsOnly(LIST_BUCKET, GET_OBJECT, PUT_OBJECT); } @Test public void testMapPolicyActionsToS3ActionsWithWildcardExpansion() { final Set result = mapPolicyActionsToS3Actions(Collections.singleton("s3:Get*")); - assertThat(result).containsOnly(S3Action.GET_OBJECT, S3Action.GET_BUCKET_ACL, S3Action.GET_OBJECT_TAGGING); + assertThat(result).containsOnly(GET_OBJECT, GET_BUCKET_ACL, GET_OBJECT_TAGGING); // Ensure it is case-insensitive final Set resultCi = mapPolicyActionsToS3Actions(Collections.singleton("s3:gET*")); - assertThat(resultCi).containsOnly(S3Action.GET_OBJECT, S3Action.GET_BUCKET_ACL, S3Action.GET_OBJECT_TAGGING); + assertThat(resultCi).containsOnly(GET_OBJECT, GET_BUCKET_ACL, GET_OBJECT_TAGGING); } @Test public void testMapPolicyActionsToS3ActionsWithS3StarReturnsAll() { final Set result = mapPolicyActionsToS3Actions(Collections.singleton("s3:*")); - assertThat(result).containsOnly(S3Action.ALL_S3); + assertThat(result).containsOnly( + LIST_ALL_MY_BUCKETS, CREATE_BUCKET, DELETE_BUCKET, GET_BUCKET_ACL, LIST_BUCKET, LIST_BUCKET_MULTIPART_UPLOADS, + PUT_BUCKET_ACL, ABORT_MULTIPART_UPLOAD, DELETE_OBJECT, DELETE_OBJECT_TAGGING, GET_OBJECT, GET_OBJECT_TAGGING, + LIST_MULTIPART_UPLOAD_PARTS, PUT_OBJECT, PUT_OBJECT_TAGGING); final Set resultCi = mapPolicyActionsToS3Actions(Collections.singleton("S3:*")); - assertThat(resultCi).containsOnly(S3Action.ALL_S3); + assertThat(resultCi).isEqualTo(result); } @Test public void testMapPolicyActionsToS3ActionsIgnoresUnsupportedActions() { final Set result = mapPolicyActionsToS3Actions(strSet("s3:GetAccelerateConfiguration", "s3:GetObject")); // Unsupported action should be silently ignored - assertThat(result).containsOnly(S3Action.GET_OBJECT); + assertThat(result).containsOnly(GET_OBJECT); } @Test @@ -413,22 +457,24 @@ public void testMapPolicyActionsToS3ActionsWithOnlyUnsupportedActionsReturnsEmpt @Test public void testMapPolicyActionsToS3ActionsDeduplicatesResults() { final Set result = mapPolicyActionsToS3Actions(strSet("s3:Get*", "s3:GetObject", "s3:GetBucketAcl")); - assertThat(result).containsOnly(S3Action.GET_OBJECT, S3Action.GET_BUCKET_ACL, S3Action.GET_OBJECT_TAGGING); + assertThat(result).containsOnly(GET_OBJECT, GET_BUCKET_ACL, GET_OBJECT_TAGGING); } @Test public void testMapPolicyActionsToS3ActionsHandlesMultipleWildcards() { final Set result = mapPolicyActionsToS3Actions(strSet("s3:Get*", "s3:Put*")); assertThat(result).containsOnly( - S3Action.GET_OBJECT, S3Action.GET_BUCKET_ACL, S3Action.GET_OBJECT_TAGGING, S3Action.PUT_OBJECT, - S3Action.PUT_OBJECT_TAGGING, S3Action.PUT_BUCKET_ACL); + GET_OBJECT, GET_BUCKET_ACL, GET_OBJECT_TAGGING, PUT_OBJECT, PUT_OBJECT_TAGGING, PUT_BUCKET_ACL); } @Test public void testMapPolicyActionsToS3ActionsWithS3StarIgnoresOtherActions() { final Set result = mapPolicyActionsToS3Actions(strSet("s3:*", "s3:GetObject", "s3:PutObject")); - // When s3:* is present, it should return only the ALL_S3 action - assertThat(result).containsOnly(S3Action.ALL_S3); + // When s3:* is present, it should return all supported concrete actions + assertThat(result).containsOnly( + LIST_ALL_MY_BUCKETS, CREATE_BUCKET, DELETE_BUCKET, GET_BUCKET_ACL, LIST_BUCKET, LIST_BUCKET_MULTIPART_UPLOADS, + PUT_BUCKET_ACL, ABORT_MULTIPART_UPLOAD, DELETE_OBJECT, DELETE_OBJECT_TAGGING, GET_OBJECT, GET_OBJECT_TAGGING, + LIST_MULTIPART_UPLOAD_PARTS, PUT_OBJECT, PUT_OBJECT_TAGGING); } @Test @@ -699,86 +745,98 @@ public void testValidateAndCategorizeResourcesWithNoResourcesThrows() { @Test public void testCreatePathsAndPermissionsWithResourceAny() { // This also tests that acls are deduplicated across different resource types - final Set actions = Stream.of(S3Action.LIST_ALL_MY_BUCKETS, S3Action.LIST_BUCKET, S3Action.GET_OBJECT) + final Set actions = Stream.of(LIST_ALL_MY_BUCKETS, LIST_BUCKET, GET_OBJECT) .collect(Collectors.toSet()); // actions at volume, bucket and key levels final Set resourceSpecs = Collections.singleton( new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.ANY, "*", null, null)); expectIllegalArgumentException( - () -> createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, new LinkedHashMap<>()), + () -> createPathsAndPermissions( + VOLUME, NATIVE, actions, resourceSpecs, null, new LinkedHashMap<>(), new LinkedHashMap<>()), "ResourceSpec type ANY not supported for OzoneNativeAuthorizer"); final Map> objToAclsMapRanger = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger); - final Set resultRanger = groupObjectsByAcls(objToAclsMapRanger); - final Set readAndListObjects = objSet(volume(), bucket("*")); // volume, bucket level have READ, LIST - final Set readObject = objSet(key("*", "*")); // key level has READ + final Map> objToActionsMapRanger = new LinkedHashMap<>(); + createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger, objToActionsMapRanger); + final Set resultRanger = groupObjectsByAclsAndActions(objToAclsMapRanger, objToActionsMapRanger); + // volume and bucket have READ, LIST; key has READ; result is now grouped by (ACLs, S3 actions) assertThat(resultRanger).containsExactlyInAnyOrder( - new OzoneGrant(readAndListObjects, acls(READ, LIST)), - new OzoneGrant(readObject, acls(READ))); + new OzoneGrant(objSet(volume()), acls(READ, LIST), strSet("ListAllMyBuckets", "ListBucket", "GetObject")), + new OzoneGrant(objSet(bucket("*")), acls(READ, LIST), strSet("ListBucket", "GetObject")), + new OzoneGrant(objSet(key("*", "*")), acls(READ), strSet("ListBucket", "GetObject"))); } @Test public void testCreatePathsAndPermissionsWithBucketResourceThatIsListBucket() { - final Set actions = Collections.singleton(IamSessionPolicyResolver.S3Action.LIST_BUCKET); + final Set actions = Collections.singleton(LIST_BUCKET); final Set resourceSpecs = Collections.singleton( new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null)); final Set readAndListObject = objSet(bucket("bucket1")); final Map> objToAclsMapNative = new LinkedHashMap<>(); + final Map> objToActionsMapNative = new LinkedHashMap<>(); final Set nativeReadObjects = objSet(volume(), prefix("bucket1", "")); - createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative); - final Set resultNative = groupObjectsByAcls(objToAclsMapNative); + createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative, objToActionsMapNative); + final Set resultNative = groupObjectsByAclsAndActions(objToAclsMapNative, objToActionsMapNative); assertThat(resultNative).containsExactlyInAnyOrder( - new OzoneGrant(readAndListObject, acls(READ, LIST)), new OzoneGrant(nativeReadObjects, acls(READ))); + new OzoneGrant(readAndListObject, acls(READ, LIST), strSet("ListBucket")), + new OzoneGrant(nativeReadObjects, acls(READ), strSet("ListBucket"))); final Map> objToAclsMapRanger = new LinkedHashMap<>(); + final Map> objToActionsMapRanger = new LinkedHashMap<>(); final Set rangerReadObjects = objSet(volume(), key("bucket1", "*")); - createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger); - final Set resultRanger = groupObjectsByAcls(objToAclsMapRanger); + createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger, objToActionsMapRanger); + final Set resultRanger = groupObjectsByAclsAndActions(objToAclsMapRanger, objToActionsMapRanger); assertThat(resultRanger).containsExactlyInAnyOrder( - new OzoneGrant(readAndListObject, acls(READ, LIST)), new OzoneGrant(rangerReadObjects, acls(READ))); + new OzoneGrant(readAndListObject, acls(READ, LIST), strSet("ListBucket")), + new OzoneGrant(rangerReadObjects, acls(READ), strSet("ListBucket"))); } @Test public void testCreatePathsAndPermissionsWithBucketResourceThatIsNotListBucket() { - final Set actions = Collections.singleton(S3Action.CREATE_BUCKET); + final Set actions = Collections.singleton(CREATE_BUCKET); final Set resourceSpecs = Collections.singleton( new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null)); final Set createObject = objSet(bucket("bucket1")); final Set readObject = objSet(volume()); final Map> objToAclsMapNative = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative); - final Set resultNative = groupObjectsByAcls(objToAclsMapNative); + final Map> objToActionsMapNative = new LinkedHashMap<>(); + createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative, objToActionsMapNative); + final Set resultNative = groupObjectsByAclsAndActions(objToAclsMapNative, objToActionsMapNative); assertThat(resultNative).containsExactlyInAnyOrder( - new OzoneGrant(createObject, acls(CREATE)), new OzoneGrant(readObject, acls(READ))); + new OzoneGrant(createObject, acls(CREATE), strSet("CreateBucket")), + new OzoneGrant(readObject, acls(READ), strSet("CreateBucket"))); final Map> objToAclsMapRanger = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger); - final Set resultRanger = groupObjectsByAcls(objToAclsMapRanger); + final Map> objToActionsMapRanger = new LinkedHashMap<>(); + createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger, objToActionsMapRanger); + final Set resultRanger = groupObjectsByAclsAndActions(objToAclsMapRanger, objToActionsMapRanger); assertThat(resultRanger).containsExactlyInAnyOrder( - new OzoneGrant(createObject, acls(CREATE)), new OzoneGrant(readObject, acls(READ))); + new OzoneGrant(createObject, acls(CREATE), strSet("CreateBucket")), + new OzoneGrant(readObject, acls(READ), strSet("CreateBucket"))); } @Test public void testCreatePathsAndPermissionsWithBucketWildcardResource() { - final Set actions = Collections.singleton(IamSessionPolicyResolver.S3Action.PUT_BUCKET_ACL); + final Set actions = Collections.singleton(PUT_BUCKET_ACL); final Set resourceSpecs = Collections.singleton( new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET_WILDCARD, "bucket1*", null, null)); final Set readReadAclAndWriteAclObject = objSet(bucket("bucket1*")); final Set readVolume = objSet(volume()); expectIllegalArgumentException( - () -> createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, new LinkedHashMap<>()), + () -> createPathsAndPermissions( + VOLUME, NATIVE, actions, resourceSpecs, null, new LinkedHashMap<>(), new LinkedHashMap<>()), "ResourceSpec type BUCKET_WILDCARD not supported for OzoneNativeAuthorizer"); final Map> objToAclsMapRanger = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger); - final Set resultRanger = groupObjectsByAcls(objToAclsMapRanger); + final Map> objToActionsMapRanger = new LinkedHashMap<>(); + createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger, objToActionsMapRanger); + final Set resultRanger = groupObjectsByAclsAndActions(objToAclsMapRanger, objToActionsMapRanger); assertThat(resultRanger).containsExactlyInAnyOrder( - new OzoneGrant(readReadAclAndWriteAclObject, acls(READ, READ_ACL, WRITE_ACL)), - new OzoneGrant(readVolume, acls(READ))); + new OzoneGrant(readReadAclAndWriteAclObject, acls(READ, READ_ACL, WRITE_ACL), strSet("PutBucketAcl")), + new OzoneGrant(readVolume, acls(READ), strSet("PutBucketAcl"))); } @Test @@ -787,124 +845,148 @@ public void testCreatePathsAndPermissionsWithBucketsWildcardResourceAll() { // Resource values. The "*" case is covered by testCreatePathsAndPermissionsWithResourceAny. // This test ensures that "arn:aws:s3:::*" (parsed as BUCKET_WILDCARD with bucket="*") // also grants the expected volume-level permissions for ListAllMyBuckets. - final Set actions = Stream.of(S3Action.LIST_ALL_MY_BUCKETS, S3Action.LIST_BUCKET) + final Set actions = Stream.of(LIST_ALL_MY_BUCKETS, LIST_BUCKET) .collect(Collectors.toSet()); final Set resourceSpecs = Collections.singleton( new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET_WILDCARD, "*", null, null)); expectIllegalArgumentException( - () -> createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, new LinkedHashMap<>()), + () -> createPathsAndPermissions( + VOLUME, NATIVE, actions, resourceSpecs, null, new LinkedHashMap<>(), new LinkedHashMap<>()), "ResourceSpec type BUCKET_WILDCARD not supported for OzoneNativeAuthorizer"); final Map> objToAclsMapRanger = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger); + final Map> objToActionsMapRanger = new LinkedHashMap<>(); + createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger, objToActionsMapRanger); // Both the volume and the wildcard bucket should end up with READ + LIST permissions. // We also need READ access on the keys - final Set resultRanger = groupObjectsByAcls(objToAclsMapRanger); - final Set readAndListObjects = objSet(volume(), bucket("*")); + final Set resultRanger = groupObjectsByAclsAndActions(objToAclsMapRanger, objToActionsMapRanger); + final Set volumeObj = objSet(volume()); + final Set bucketObj = objSet(bucket("*")); final Set readObjects = objSet(key("*", "*")); assertThat(resultRanger).containsExactlyInAnyOrder( - new OzoneGrant(readAndListObjects, acls(READ, LIST)), new OzoneGrant(readObjects, acls(READ))); + new OzoneGrant(volumeObj, acls(READ, LIST), strSet("ListAllMyBuckets", "ListBucket")), + new OzoneGrant(bucketObj, acls(READ, LIST), strSet("ListBucket")), + new OzoneGrant(readObjects, acls(READ), strSet("ListBucket"))); } @Test public void testCreatePathsAndPermissionsWithObjectExactResource() { - final Set actions = Collections.singleton(S3Action.GET_OBJECT); + final Set actions = Collections.singleton(GET_OBJECT); final Set resourceSpecs = Collections.singleton( new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_EXACT, "bucket1", null, "key.txt")); - final Set readObjects = objSet(key("bucket1", "key.txt"), bucket("bucket1"), volume()); + final Set readVolumeBucketAndKey = objSet(volume(), bucket("bucket1"), key("bucket1", "key.txt")); final Map> objToAclsMapNative = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative); - final Set resultNative = groupObjectsByAcls(objToAclsMapNative); - assertThat(resultNative).containsExactly(new OzoneGrant(readObjects, acls(READ))); + final Map> objToActionsMapNative = new LinkedHashMap<>(); + createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative, objToActionsMapNative); + final Set resultNative = groupObjectsByAclsAndActions(objToAclsMapNative, objToActionsMapNative); + assertThat(resultNative).containsExactlyInAnyOrder( + new OzoneGrant(readVolumeBucketAndKey, acls(READ), strSet("GetObject"))); final Map> objToAclsMapRanger = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger); - final Set resultRanger = groupObjectsByAcls(objToAclsMapRanger); - assertThat(resultRanger).containsExactly(new OzoneGrant(readObjects, acls(READ))); + final Map> objToActionsMapRanger = new LinkedHashMap<>(); + createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger, objToActionsMapRanger); + final Set resultRanger = groupObjectsByAclsAndActions(objToAclsMapRanger, objToActionsMapRanger); + assertThat(resultRanger).containsExactlyInAnyOrder( + new OzoneGrant(readVolumeBucketAndKey, acls(READ), strSet("GetObject"))); } @Test public void testCreatePathsAndPermissionsWithDeleteObjectGrantsDeleteOnKey() { - final Set actions = Collections.singleton(S3Action.DELETE_OBJECT); + final Set actions = Collections.singleton(DELETE_OBJECT); final Set resourceSpecs = Collections.singleton( new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_EXACT, "bucket1", null, "key.txt")); final Set readVolumeAndBucket = objSet(volume(), bucket("bucket1")); final Set deleteKey = objSet(key("bucket1", "key.txt")); final Map> objToAclsMapNative = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative); - final Set resultNative = groupObjectsByAcls(objToAclsMapNative); + final Map> objToActionsMapNative = new LinkedHashMap<>(); + createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative, objToActionsMapNative); + final Set resultNative = groupObjectsByAclsAndActions(objToAclsMapNative, objToActionsMapNative); assertThat(resultNative).containsExactlyInAnyOrder( - new OzoneGrant(readVolumeAndBucket, acls(READ)), new OzoneGrant(deleteKey, acls(DELETE))); + new OzoneGrant(readVolumeAndBucket, acls(READ), strSet("DeleteObject")), + new OzoneGrant(deleteKey, acls(DELETE), strSet("DeleteObject"))); final Map> objToAclsMapRanger = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger); - final Set resultRanger = groupObjectsByAcls(objToAclsMapRanger); + final Map> objToActionsMapRanger = new LinkedHashMap<>(); + createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger, objToActionsMapRanger); + final Set resultRanger = groupObjectsByAclsAndActions(objToAclsMapRanger, objToActionsMapRanger); assertThat(resultRanger).containsExactlyInAnyOrder( - new OzoneGrant(readVolumeAndBucket, acls(READ)), new OzoneGrant(deleteKey, acls(DELETE))); + new OzoneGrant(readVolumeAndBucket, acls(READ), strSet("DeleteObject")), + new OzoneGrant(deleteKey, acls(DELETE), strSet("DeleteObject"))); } @Test public void testCreatePathsAndPermissionsWithAbortMultipartUploadGrantsWriteOnKey() { - final Set actions = Collections.singleton(S3Action.ABORT_MULTIPART_UPLOAD); + final Set actions = Collections.singleton(ABORT_MULTIPART_UPLOAD); final Set resourceSpecs = Collections.singleton( new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_EXACT, "bucket1", null, "key.txt")); final Set readVolumeAndBucket = objSet(volume(), bucket("bucket1")); final Set writeKey = objSet(key("bucket1", "key.txt")); final Map> objToAclsMapNative = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative); - final Set resultNative = groupObjectsByAcls(objToAclsMapNative); + final Map> objToActionsMapNative = new LinkedHashMap<>(); + createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative, objToActionsMapNative); + final Set resultNative = groupObjectsByAclsAndActions(objToAclsMapNative, objToActionsMapNative); assertThat(resultNative).containsExactlyInAnyOrder( - new OzoneGrant(readVolumeAndBucket, acls(READ)), new OzoneGrant(writeKey, acls(WRITE))); + new OzoneGrant(readVolumeAndBucket, acls(READ), strSet("AbortMultipartUpload")), + new OzoneGrant(writeKey, acls(WRITE), strSet("AbortMultipartUpload"))); final Map> objToAclsMapRanger = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger); - final Set resultRanger = groupObjectsByAcls(objToAclsMapRanger); + final Map> objToActionsMapRanger = new LinkedHashMap<>(); + createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger, objToActionsMapRanger); + final Set resultRanger = groupObjectsByAclsAndActions(objToAclsMapRanger, objToActionsMapRanger); assertThat(resultRanger).containsExactlyInAnyOrder( - new OzoneGrant(readVolumeAndBucket, acls(READ)), new OzoneGrant(writeKey, acls(WRITE))); + new OzoneGrant(readVolumeAndBucket, acls(READ), strSet("AbortMultipartUpload")), + new OzoneGrant(writeKey, acls(WRITE), strSet("AbortMultipartUpload"))); } @Test public void testCreatePathsAndPermissionsWithObjectPrefixResource() { - final Set actions = Collections.singleton(S3Action.GET_OBJECT); + final Set actions = Collections.singleton(GET_OBJECT); final Set resourceSpecs = Collections.singleton( new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX, "bucket1", "prefix/", null)); - final Set nativeReadObjects = objSet(prefix("bucket1", "prefix/"), bucket("bucket1"), volume()); + final Set nativeReadVolumeBucketAndPrefix = objSet( + bucket("bucket1"), volume(), prefix("bucket1", "prefix/")); final Map> objToAclsMapNative = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative); - final Set resultNative = groupObjectsByAcls(objToAclsMapNative); - assertThat(resultNative).containsExactly(new OzoneGrant(nativeReadObjects, acls(READ))); + final Map> objToActionsMapNative = new LinkedHashMap<>(); + createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative, objToActionsMapNative); + final Set resultNative = groupObjectsByAclsAndActions(objToAclsMapNative, objToActionsMapNative); + assertThat(resultNative).containsExactlyInAnyOrder( + new OzoneGrant(nativeReadVolumeBucketAndPrefix, acls(READ), strSet("GetObject"))); expectIllegalArgumentException( - () -> createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, new LinkedHashMap<>()), + () -> createPathsAndPermissions( + VOLUME, RANGER, actions, resourceSpecs, null, new LinkedHashMap<>(), new LinkedHashMap<>()), "ResourceSpec type OBJECT_PREFIX not supported for RangerOzoneAuthorizer"); } @Test public void testCreatePathsAndPermissionsWithObjectPrefixWildcardResource() { - final Set actions = Collections.singleton(S3Action.GET_OBJECT); + final Set actions = Collections.singleton(GET_OBJECT); final Set resourceSpecs = Collections.singleton( new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket1", "prefix/*", null)); expectIllegalArgumentException( - () -> createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, new LinkedHashMap<>()), + () -> createPathsAndPermissions( + VOLUME, NATIVE, actions, resourceSpecs, null, new LinkedHashMap<>(), new LinkedHashMap<>()), "ResourceSpec type OBJECT_PREFIX_WILDCARD not supported for OzoneNativeAuthorizer"); - final Set rangerReadObjects = objSet(key("bucket1", "prefix/*"), bucket("bucket1"), volume()); final Map> objToAclsMapRanger = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger); - final Set resultRanger = groupObjectsByAcls(objToAclsMapRanger); - assertThat(resultRanger).containsExactly(new OzoneGrant(rangerReadObjects, acls(READ))); + final Map> objToActionsMapRanger = new LinkedHashMap<>(); + createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger, objToActionsMapRanger); + final Set resultRanger = groupObjectsByAclsAndActions(objToAclsMapRanger, objToActionsMapRanger); + assertThat(resultRanger).containsExactlyInAnyOrder( + new OzoneGrant( + objSet(bucket("bucket1"), volume(), key("bucket1", "prefix/*")), acls(READ), strSet("GetObject"))); } @Test public void testCreatePathsAndPermissionsWithConditionPrefixesForObjectActionMustIgnoreConditionPrefixes() { - final Set actions = Collections.singleton(S3Action.GET_OBJECT); + final Set actions = Collections.singleton(GET_OBJECT); final Set prefixes = strSet("folder1/", "folder2/"); final IamSessionPolicyResolver.Condition condition = new IamSessionPolicyResolver.Condition( "StringEquals", prefixes); @@ -912,23 +994,29 @@ public void testCreatePathsAndPermissionsWithConditionPrefixesForObjectActionMus final Set nativeResourceSpecs = Collections.singleton( new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX, "bucket1", "", null)); final Map> objToAclsMapNative = new LinkedHashMap<>(); - final Set nativeReadObjects = objSet(prefix("bucket1", ""), bucket("bucket1"), volume()); - createPathsAndPermissions(VOLUME, NATIVE, actions, nativeResourceSpecs, condition, objToAclsMapNative); - final Set resultNative = groupObjectsByAcls(objToAclsMapNative); - assertThat(resultNative).containsExactly(new OzoneGrant(nativeReadObjects, acls(READ))); + final Map> objToActionsMapNative = new LinkedHashMap<>(); + final Set nativeReadVolumeBucketAndPrefix = objSet(bucket("bucket1"), volume(), prefix("bucket1", "")); + createPathsAndPermissions( + VOLUME, NATIVE, actions, nativeResourceSpecs, condition, objToAclsMapNative, objToActionsMapNative); + final Set resultNative = groupObjectsByAclsAndActions(objToAclsMapNative, objToActionsMapNative); + assertThat(resultNative).containsExactlyInAnyOrder( + new OzoneGrant(nativeReadVolumeBucketAndPrefix, acls(READ), strSet("GetObject"))); final Set rangerResourceSpecs = Collections.singleton( new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket1", "*", null)); final Map> objToAclsMapRanger = new LinkedHashMap<>(); - final Set rangerReadObjects = objSet(key("bucket1", "*"), bucket("bucket1"), volume()); - createPathsAndPermissions(VOLUME, RANGER, actions, rangerResourceSpecs, condition, objToAclsMapRanger); - final Set resultRanger = groupObjectsByAcls(objToAclsMapRanger); - assertThat(resultRanger).containsExactly(new OzoneGrant(rangerReadObjects, acls(READ))); + final Map> objToActionsMapRanger = new LinkedHashMap<>(); + final Set rangerReadVolumeBucketAndKey = objSet(bucket("bucket1"), volume(), key("bucket1", "*")); + createPathsAndPermissions( + VOLUME, RANGER, actions, rangerResourceSpecs, condition, objToAclsMapRanger, objToActionsMapRanger); + final Set resultRanger = groupObjectsByAclsAndActions(objToAclsMapRanger, objToActionsMapRanger); + assertThat(resultRanger).containsExactlyInAnyOrder( + new OzoneGrant(rangerReadVolumeBucketAndKey, acls(READ), strSet("GetObject"))); } @Test public void testCreatePathsAndPermissionsWithConditionPrefixesForBucketActionWhenActionIsListBucket() { - final Set actions = Collections.singleton(S3Action.LIST_BUCKET); + final Set actions = Collections.singleton(LIST_BUCKET); final Set prefixes = strSet("folder1/", "folder2/"); final IamSessionPolicyResolver.Condition condition = new IamSessionPolicyResolver.Condition( "StringEquals", prefixes); @@ -939,10 +1027,13 @@ public void testCreatePathsAndPermissionsWithConditionPrefixesForBucketActionWhe prefix("bucket1", "folder1/"), prefix("bucket1", "folder2/"), volume()); final Set nativeReadAndListObject = objSet(bucket("bucket1")); final Map> objToAclsMapNative = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, NATIVE, actions, nativeResourceSpecs, condition, objToAclsMapNative); - final Set resultNative = groupObjectsByAcls(objToAclsMapNative); + final Map> objToActionsMapNative = new LinkedHashMap<>(); + createPathsAndPermissions( + VOLUME, NATIVE, actions, nativeResourceSpecs, condition, objToAclsMapNative, objToActionsMapNative); + final Set resultNative = groupObjectsByAclsAndActions(objToAclsMapNative, objToActionsMapNative); assertThat(resultNative).containsExactlyInAnyOrder( - new OzoneGrant(nativeReadObjects, acls(READ)), new OzoneGrant(nativeReadAndListObject, acls(READ, LIST))); + new OzoneGrant(nativeReadAndListObject, acls(READ, LIST), strSet("ListBucket")), + new OzoneGrant(nativeReadObjects, acls(READ), strSet("ListBucket"))); final Set rangerResourceSpecs = Collections.singleton( new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null)); @@ -950,15 +1041,18 @@ public void testCreatePathsAndPermissionsWithConditionPrefixesForBucketActionWhe key("bucket1", "folder1/"), key("bucket1", "folder2/"), volume()); final Set rangerReadAndListObject = objSet(bucket("bucket1")); final Map> objToAclsMapRanger = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, RANGER, actions, rangerResourceSpecs, condition, objToAclsMapRanger); - final Set resultRanger = groupObjectsByAcls(objToAclsMapRanger); + final Map> objToActionsMapRanger = new LinkedHashMap<>(); + createPathsAndPermissions( + VOLUME, RANGER, actions, rangerResourceSpecs, condition, objToAclsMapRanger, objToActionsMapRanger); + final Set resultRanger = groupObjectsByAclsAndActions(objToAclsMapRanger, objToActionsMapRanger); assertThat(resultRanger).containsExactlyInAnyOrder( - new OzoneGrant(rangerReadObjects, acls(READ)), new OzoneGrant(rangerReadAndListObject, acls(READ, LIST))); + new OzoneGrant(rangerReadAndListObject, acls(READ, LIST), strSet("ListBucket")), + new OzoneGrant(rangerReadObjects, acls(READ), strSet("ListBucket"))); } @Test public void testCreatePathsAndPermissionsWithConditionPrefixesForBucketActionWhenActionIsNotListBucket() { - final Set actions = Collections.singleton(S3Action.GET_BUCKET_ACL); + final Set actions = Collections.singleton(GET_BUCKET_ACL); final Set prefixes = strSet("folder1/", "folder2/"); final IamSessionPolicyResolver.Condition condition = new IamSessionPolicyResolver.Condition( "StringEquals", prefixes); @@ -968,18 +1062,24 @@ public void testCreatePathsAndPermissionsWithConditionPrefixesForBucketActionWhe final Set nativeResourceSpecs = Collections.singleton( new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null)); final Map> objToAclsMapNative = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, NATIVE, actions, nativeResourceSpecs, condition, objToAclsMapNative); - final Set resultNative = groupObjectsByAcls(objToAclsMapNative); + final Map> objToActionsMapNative = new LinkedHashMap<>(); + createPathsAndPermissions( + VOLUME, NATIVE, actions, nativeResourceSpecs, condition, objToAclsMapNative, objToActionsMapNative); + final Set resultNative = groupObjectsByAclsAndActions(objToAclsMapNative, objToActionsMapNative); assertThat(resultNative).containsExactlyInAnyOrder( - new OzoneGrant(readObject, acls(READ)), new OzoneGrant(readAndReadAclObject, acls(READ, READ_ACL))); + new OzoneGrant(readObject, acls(READ), strSet("GetBucketAcl")), + new OzoneGrant(readAndReadAclObject, acls(READ, READ_ACL), strSet("GetBucketAcl"))); final Set rangerResourceSpecs = Collections.singleton( new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null)); final Map> objToAclsMapRanger = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, RANGER, actions, rangerResourceSpecs, condition, objToAclsMapRanger); - final Set resultRanger = groupObjectsByAcls(objToAclsMapRanger); + final Map> objToActionsMapRanger = new LinkedHashMap<>(); + createPathsAndPermissions( + VOLUME, RANGER, actions, rangerResourceSpecs, condition, objToAclsMapRanger, objToActionsMapRanger); + final Set resultRanger = groupObjectsByAclsAndActions(objToAclsMapRanger, objToActionsMapRanger); assertThat(resultRanger).containsExactlyInAnyOrder( - new OzoneGrant(readObject, acls(READ)), new OzoneGrant(readAndReadAclObject, acls(READ, READ_ACL))); + new OzoneGrant(readObject, acls(READ), strSet("GetBucketAcl")), + new OzoneGrant(readAndReadAclObject, acls(READ, READ_ACL), strSet("GetBucketAcl"))); } @Test @@ -989,38 +1089,45 @@ public void testCreatePathsAndPermissionsWithNoMappedActions() { final Set nativeResourceSpecs = Collections.singleton( new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX, "bucket1", null, null)); final Map> objToAclsMapNative = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, NATIVE, actions, nativeResourceSpecs, null, objToAclsMapNative); - final Set resultNative = groupObjectsByAcls(objToAclsMapNative); + final Map> objToActionsMapNative = new LinkedHashMap<>(); + createPathsAndPermissions( + VOLUME, NATIVE, actions, nativeResourceSpecs, null, objToAclsMapNative, objToActionsMapNative); + final Set resultNative = groupObjectsByAclsAndActions(objToAclsMapNative, objToActionsMapNative); assertThat(resultNative).isEmpty(); final Set rangerResourceSpecs = Collections.singleton( new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket1", null, null)); final Map> objToAclsMapRanger = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, RANGER, actions, rangerResourceSpecs, null, objToAclsMapRanger); - final Set resultRanger = groupObjectsByAcls(objToAclsMapRanger); + final Map> objToActionsMapRanger = new LinkedHashMap<>(); + createPathsAndPermissions( + VOLUME, RANGER, actions, rangerResourceSpecs, null, objToAclsMapRanger, objToActionsMapRanger); + final Set resultRanger = groupObjectsByAclsAndActions(objToAclsMapRanger, objToActionsMapRanger); assertThat(resultRanger).isEmpty(); } @Test public void testCreatePathsAndPermissionsWithNoMappedResources() { - final Set actions = Collections.singleton(S3Action.GET_OBJECT); + final Set actions = Collections.singleton(GET_OBJECT); final Set resourceSpecs = emptySet(); final Map> objToAclsMapNative = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative); - final Set resultNative = groupObjectsByAcls(objToAclsMapNative); + final Map> objToActionsMapNative = new LinkedHashMap<>(); + createPathsAndPermissions( + VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative, objToActionsMapNative); + final Set resultNative = groupObjectsByAclsAndActions(objToAclsMapNative, objToActionsMapNative); assertThat(resultNative).isEmpty(); final Map> objToAclsMapRanger = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger); - final Set resultRanger = groupObjectsByAcls(objToAclsMapRanger); + final Map> objToActionsMapRanger = new LinkedHashMap<>(); + createPathsAndPermissions( + VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger, objToActionsMapRanger); + final Set resultRanger = groupObjectsByAclsAndActions(objToAclsMapRanger, objToActionsMapRanger); assertThat(resultRanger).isEmpty(); } @Test public void testCreatePathsAndPermissionsDeduplicatesAcrossSameResourceTypes() { - final Set actions = Stream.of( - S3Action.GET_OBJECT, S3Action.GET_OBJECT_TAGGING, S3Action.DELETE_OBJECT, S3Action.DELETE_OBJECT_TAGGING) + final Set actions = Stream.of(GET_OBJECT, GET_OBJECT_TAGGING, DELETE_OBJECT, DELETE_OBJECT_TAGGING) .collect(Collectors.toSet()); final Set resourceSpecs = Collections.singleton( new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_EXACT, "bucket1", null, "key.txt")); @@ -1028,44 +1135,60 @@ public void testCreatePathsAndPermissionsDeduplicatesAcrossSameResourceTypes() { final Set readObjects = objSet(bucket("bucket1"), volume()); final Map> objToAclsMapNative = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative); - final Set resultNative = groupObjectsByAcls(objToAclsMapNative); + final Map> objToActionsMapNative = new LinkedHashMap<>(); + createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative, objToActionsMapNative); + final Set resultNative = groupObjectsByAclsAndActions(objToAclsMapNative, objToActionsMapNative); assertThat(resultNative).containsExactlyInAnyOrder( - new OzoneGrant(readAndDeleteAndWriteObject, acls(READ, DELETE, WRITE)), - new OzoneGrant(readObjects, acls(READ))); + new OzoneGrant( + readAndDeleteAndWriteObject, acls(READ, DELETE, WRITE), + strSet("GetObject", "GetObjectTagging", "DeleteObject", "DeleteObjectTagging")), + new OzoneGrant(readObjects, acls(READ), + strSet("GetObject", "GetObjectTagging", "DeleteObject", "DeleteObjectTagging"))); final Map> objToAclsMapRanger = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger); - final Set resultRanger = groupObjectsByAcls(objToAclsMapRanger); + final Map> objToActionsMapRanger = new LinkedHashMap<>(); + createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger, objToActionsMapRanger); + final Set resultRanger = groupObjectsByAclsAndActions(objToAclsMapRanger, objToActionsMapRanger); assertThat(resultRanger).containsExactlyInAnyOrder( - new OzoneGrant(readAndDeleteAndWriteObject, acls(READ, DELETE, WRITE)), - new OzoneGrant(readObjects, acls(READ))); + new OzoneGrant( + readAndDeleteAndWriteObject, acls(READ, DELETE, WRITE), + strSet("GetObject", "GetObjectTagging", "DeleteObject", "DeleteObjectTagging")), + new OzoneGrant(readObjects, acls(READ), + strSet("GetObject", "GetObjectTagging", "DeleteObject", "DeleteObjectTagging"))); } @Test - public void testCreatePathsAndPermissionsWithAllS3ActionsOverridesAnyOtherAction() { - final Set actions = Stream.of( - S3Action.ALL_S3, S3Action.GET_OBJECT, S3Action.DELETE_OBJECT, S3Action.LIST_BUCKET) + public void testCreatePathsAndPermissionsWithAllConcreteS3Actions() { + final Set actions = Stream.of(S3Action.values()) .collect(Collectors.toSet()); final Set resourceSpecs = Stream.of( new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_EXACT, "bucket1", null, "key.txt"), new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket2", null, null)) .collect(Collectors.toSet()); - final Set allObjects = objSet(key("bucket1", "key.txt"), bucket("bucket2")); - - final Set nativeReadObjects = objSet(volume(), bucket("bucket1"), prefix("bucket2", "")); final Map> objToAclsMapNative = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative); - final Set resultNative = groupObjectsByAcls(objToAclsMapNative); + final Map> objToActionsMapNative = new LinkedHashMap<>(); + createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative, objToActionsMapNative); + final Set resultNative = groupObjectsByAclsAndActions(objToAclsMapNative, objToActionsMapNative); assertThat(resultNative).containsExactlyInAnyOrder( - new OzoneGrant(allObjects, acls(ALL)), new OzoneGrant(nativeReadObjects, acls(READ))); + new OzoneGrant(objSet(key("bucket1", "key.txt")), acls(READ, CREATE, WRITE, DELETE), ALL_OBJECT_ACTIONS), + new OzoneGrant(objSet(bucket("bucket1")), acls(READ), ALL_OBJECT_ACTIONS), + new OzoneGrant(objSet(volume()), acls(READ), ALL_BUCKET_AND_OBJECT_ACTIONS), + new OzoneGrant( + objSet(bucket("bucket2")), acls(READ, LIST, CREATE, DELETE, READ_ACL, WRITE_ACL), ALL_BUCKET_ACTIONS), + new OzoneGrant(objSet(prefix("bucket2", "")), acls(READ), strSet("ListBucket"))); - final Set rangerReadObjects = objSet(volume(), bucket("bucket1"), key("bucket2", "*")); final Map> objToAclsMapRanger = new LinkedHashMap<>(); - createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger); - final Set resultRanger = groupObjectsByAcls(objToAclsMapRanger); + final Map> objToActionsMapRanger = new LinkedHashMap<>(); + createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger, objToActionsMapRanger); + final Set resultRanger = groupObjectsByAclsAndActions(objToAclsMapRanger, objToActionsMapRanger); assertThat(resultRanger).containsExactlyInAnyOrder( - new OzoneGrant(allObjects, acls(ALL)), new OzoneGrant(rangerReadObjects, acls(READ))); + new OzoneGrant(objSet(key("bucket1", "key.txt")), acls(READ, CREATE, WRITE, DELETE), ALL_OBJECT_ACTIONS), + new OzoneGrant(objSet(bucket("bucket1")), acls(READ), ALL_OBJECT_ACTIONS), + new OzoneGrant(objSet(volume()), acls(READ), ALL_BUCKET_AND_OBJECT_ACTIONS), + new OzoneGrant( + objSet(bucket("bucket2")), acls(READ, LIST, CREATE, DELETE, READ_ACL, WRITE_ACL), ALL_BUCKET_ACTIONS), + new OzoneGrant(objSet(key("bucket2", "*")), acls(READ), strSet("ListBucket"))); + } @Test @@ -1102,14 +1225,21 @@ public void testDeduplicatesAcrossMultipleStatementsWhenSameStatementsArePresent // Expected for native: bucket READ, LIST, READ_ACL, WRITE_ACL; volume and prefix "" READ final Set bucketSet = objSet(bucket("my-bucket")); final Set bucketAcls = acls(READ, LIST, READ_ACL, WRITE_ACL); - expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls)); - expectedResolvedNative.add(new OzoneGrant(objSet(volume(), prefix("my-bucket", "")), acls(READ))); + expectedResolvedNative.add( + new OzoneGrant(bucketSet, bucketAcls, strSet("GetBucketAcl", "PutBucketAcl", "ListBucket"))); + expectedResolvedNative.add(new OzoneGrant(objSet(prefix("my-bucket", "")), acls(READ), strSet("ListBucket"))); + expectedResolvedNative.add( + new OzoneGrant(objSet(volume()), acls(READ), strSet("GetBucketAcl", "PutBucketAcl", "ListBucket"))); assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative); final Set expectedResolvedRanger = new LinkedHashSet<>(); - // Expected for Ranger: bucket READ, LIST, READ_ACL, WRITE_ACL; volume and key "*" READ - expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls)); - expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("my-bucket", "*")), acls(READ))); + // Expected for Ranger: bucket READ, LIST, READ_ACL, WRITE_ACL; volume and key "*" READ + expectedResolvedRanger.add( + new OzoneGrant(bucketSet, bucketAcls, strSet("GetBucketAcl", "PutBucketAcl", "ListBucket"))); + expectedResolvedRanger.add( + new OzoneGrant( + objSet(volume()), acls(READ), strSet("GetBucketAcl", "PutBucketAcl", "ListBucket"))); + expectedResolvedRanger.add(new OzoneGrant(objSet(key("my-bucket", "*")), acls(READ), strSet("ListBucket"))); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1147,16 +1277,22 @@ public void testDeduplicatesAcrossMultipleStatementsForSameActionsButDifferentRe // Expected for native: bucket READ, LIST, READ_ACL, WRITE_ACL; volume and prefix "" READ final Set bucketSet = objSet(bucket("my-bucket"), bucket("my-bucket2")); final Set bucketAcls = acls(READ, LIST, READ_ACL, WRITE_ACL); - expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls)); + expectedResolvedNative.add( + new OzoneGrant(bucketSet, bucketAcls, strSet("GetBucketAcl", "PutBucketAcl", "ListBucket"))); expectedResolvedNative.add(new OzoneGrant( - objSet(volume(), prefix("my-bucket2", ""), prefix("my-bucket", "")), acls(READ))); + objSet(prefix("my-bucket2", ""), prefix("my-bucket", "")), acls(READ), strSet("ListBucket"))); + expectedResolvedNative.add( + new OzoneGrant(objSet(volume()), acls(READ), strSet("GetBucketAcl", "PutBucketAcl", "ListBucket"))); assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative); final Set expectedResolvedRanger = new LinkedHashSet<>(); // Expected for Ranger: bucket READ, LIST, READ_ACL, WRITE_ACL; volume and key "*" READ - expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls)); - expectedResolvedRanger.add(new OzoneGrant( - objSet(volume(), key("my-bucket2", "*"), key("my-bucket", "*")), acls(READ))); + expectedResolvedRanger.add( + new OzoneGrant(bucketSet, bucketAcls, strSet("GetBucketAcl", "PutBucketAcl", "ListBucket"))); + expectedResolvedRanger.add( + new OzoneGrant(objSet(key("my-bucket2", "*"), key("my-bucket", "*")), acls(READ), strSet("ListBucket"))); + expectedResolvedRanger.add( + new OzoneGrant(objSet(volume()), acls(READ), strSet("GetBucketAcl", "PutBucketAcl", "ListBucket"))); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1193,14 +1329,22 @@ public void testDeduplicatesAcrossMultipleStatementsForDifferentActionsButSameRe // Expected for native: bucket READ, LIST, READ_ACL, WRITE_ACL, CREATE; volume, prefix "" READ final Set bucketSet = objSet(bucket("my-bucket")); final Set bucketAcls = acls(READ, LIST, READ_ACL, WRITE_ACL, CREATE); - expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls)); - expectedResolvedNative.add(new OzoneGrant(objSet(volume(), prefix("my-bucket", "")), acls(READ))); + expectedResolvedNative.add( + new OzoneGrant(bucketSet, bucketAcls, strSet("GetBucketAcl", "PutBucketAcl", "ListBucket", "CreateBucket"))); + expectedResolvedNative.add(new OzoneGrant(objSet(prefix("my-bucket", "")), acls(READ), strSet("ListBucket"))); + expectedResolvedNative.add( + new OzoneGrant( + objSet(volume()), acls(READ), strSet("GetBucketAcl", "PutBucketAcl", "ListBucket", "CreateBucket"))); assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative); final Set expectedResolvedRanger = new LinkedHashSet<>(); // Expected for Ranger: bucket READ, LIST, READ_ACL, WRITE_ACL, CREATE; volume, key "*" READ - expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls)); - expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("my-bucket", "*")), acls(READ))); + expectedResolvedRanger.add( + new OzoneGrant(bucketSet, bucketAcls, strSet("GetBucketAcl", "PutBucketAcl", "ListBucket", "CreateBucket"))); + expectedResolvedRanger.add(new OzoneGrant(objSet(key("my-bucket", "*")), acls(READ), strSet("ListBucket"))); + expectedResolvedRanger.add( + new OzoneGrant( + objSet(volume()), acls(READ), strSet("GetBucketAcl", "PutBucketAcl", "ListBucket", "CreateBucket"))); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1231,17 +1375,19 @@ public void testDeduplicatesAcrossMultipleStatementsWhenAllActionPresent() throw // Ensure what we got is what we expected final Set expectedResolvedNative = new LinkedHashSet<>(); - // Expected for native: bucket ALL (instead of individual actions); volume and prefix "" READ + // Expected for native: bucket union of supported bucket ACLs; volume READ; prefix "" READ (from ListBucket) final Set bucketSet = objSet(bucket("my-bucket")); - final Set bucketAcls = acls(ALL); - expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls)); - expectedResolvedNative.add(new OzoneGrant(objSet(volume(), prefix("my-bucket", "")), acls(READ))); + final Set bucketAcls = acls(READ, LIST, CREATE, DELETE, READ_ACL, WRITE_ACL); + expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls, ALL_BUCKET_ACTIONS)); + expectedResolvedNative.add(new OzoneGrant(objSet(prefix("my-bucket", "")), acls(READ), strSet("ListBucket"))); + expectedResolvedNative.add(new OzoneGrant(objSet(volume()), acls(READ), ALL_BUCKET_ACTIONS)); assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative); final Set expectedResolvedRanger = new LinkedHashSet<>(); - // Expected for Ranger: bucket ALL (instead of individual actions); volume and key "*" READ - expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls)); - expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("my-bucket", "*")), acls(READ))); + // Expected for Ranger: bucket union of supported bucket ACLs; volume READ; key "*" READ (from ListBucket) + expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls, ALL_BUCKET_ACTIONS)); + expectedResolvedRanger.add(new OzoneGrant(objSet(key("my-bucket", "*")), acls(READ), strSet("ListBucket"))); + expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ), ALL_BUCKET_ACTIONS)); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1264,8 +1410,10 @@ public void testAllowGetPutOnKey() throws OMException { // Expected: READ, CREATE, WRITE on key; bucket READ; volume READ final Set keySet = objSet(key("my-bucket", "folder/file.txt")); final Set keyAcls = acls(READ, CREATE, WRITE); - expectedResolvedFromBothAuthorizers.add(new OzoneGrant(objSet(volume(), bucket("my-bucket")), acls(READ))); - expectedResolvedFromBothAuthorizers.add(new OzoneGrant(keySet, keyAcls)); + expectedResolvedFromBothAuthorizers.add( + new OzoneGrant(objSet(volume(), bucket("my-bucket")), acls(READ), strSet("GetObject", "PutObject"))); + expectedResolvedFromBothAuthorizers.add( + new OzoneGrant(keySet, keyAcls, strSet("GetObject", "PutObject"))); assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedFromBothAuthorizers); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedFromBothAuthorizers); @@ -1286,18 +1434,18 @@ public void testAllActionsForKey() throws OMException { // Ensure what we got is what we expected final Set expectedResolvedNative = new LinkedHashSet<>(); - // Expected for native: all key ACLs on prefix "" under bucket; bucket READ, volume READ + // Expected for native: all supported object ACLs on prefix "" under bucket; bucket READ, volume READ final Set keyPrefixSet = objSet(prefix("my-bucket", "")); - final Set allKeyAcls = acls(ALL); - expectedResolvedNative.add(new OzoneGrant(keyPrefixSet, allKeyAcls)); - expectedResolvedNative.add(new OzoneGrant(objSet(volume(), bucket("my-bucket")), acls(READ))); + final Set allKeyAcls = acls(READ, CREATE, WRITE, DELETE); + expectedResolvedNative.add(new OzoneGrant(keyPrefixSet, allKeyAcls, ALL_OBJECT_ACTIONS)); + expectedResolvedNative.add(new OzoneGrant(objSet(volume(), bucket("my-bucket")), acls(READ), ALL_OBJECT_ACTIONS)); assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative); - // Expected for Ranger: all key acls for resource type KEY with key name "*" + // Expected for Ranger: all supported object ACLs for resource type KEY with key name "*" final Set expectedResolvedRanger = new LinkedHashSet<>(); final Set rangerKeySet = objSet(key("my-bucket", "*")); - expectedResolvedRanger.add(new OzoneGrant(rangerKeySet, allKeyAcls)); - expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), bucket("my-bucket")), acls(READ))); + expectedResolvedRanger.add(new OzoneGrant(rangerKeySet, allKeyAcls, ALL_OBJECT_ACTIONS)); + expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), bucket("my-bucket")), acls(READ), ALL_OBJECT_ACTIONS)); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1340,17 +1488,19 @@ public void testAllActionsForBucket() throws OMException { // Ensure what we got is what we expected final Set expectedResolvedNative = new LinkedHashSet<>(); - // Expected for native: all Bucket ACLs for bucket; volume, prefix "" READ + // Expected for native: union of supported bucket ACLs for bucket; volume READ; prefix "" READ (from ListBucket) final Set bucketSet = objSet(bucket("my-bucket")); - final Set allBucketAcls = acls(ALL); - expectedResolvedNative.add(new OzoneGrant(objSet(volume(), prefix("my-bucket", "")), acls(READ))); - expectedResolvedNative.add(new OzoneGrant(bucketSet, allBucketAcls)); + final Set allBucketAcls = acls(READ, LIST, CREATE, DELETE, READ_ACL, WRITE_ACL); + expectedResolvedNative.add(new OzoneGrant(bucketSet, allBucketAcls, ALL_BUCKET_ACTIONS)); + expectedResolvedNative.add(new OzoneGrant(objSet(prefix("my-bucket", "")), acls(READ), strSet("ListBucket"))); + expectedResolvedNative.add(new OzoneGrant(objSet(volume()), acls(READ), ALL_BUCKET_ACTIONS)); assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative); - // Expected for Ranger: all Bucket ACLs for bucket; volume, key "*" READ + // Expected for Ranger: union of supported bucket ACLs for bucket; volume READ; key "*" READ (from ListBucket) final Set expectedResolvedRanger = new LinkedHashSet<>(); - expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("my-bucket", "*")), acls(READ))); - expectedResolvedRanger.add(new OzoneGrant(bucketSet, allBucketAcls)); + expectedResolvedRanger.add(new OzoneGrant(bucketSet, allBucketAcls, ALL_BUCKET_ACTIONS)); + expectedResolvedRanger.add(new OzoneGrant(objSet(key("my-bucket", "*")), acls(READ), strSet("ListBucket"))); + expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ), ALL_BUCKET_ACTIONS)); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1379,18 +1529,20 @@ public void testAllActionsForBucketWithPrefixCondition() throws OMException { final Set bucketSet = objSet(bucket("my-bucket")); final Set bucketAcls = acls(READ, LIST); expectedResolvedNative.add( - new OzoneGrant(objSet(volume(), prefix("my-bucket", "team/folder"), prefix("my-bucket", "team/folder/")), - acls(READ))); - expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls)); + new OzoneGrant( + objSet(volume(), prefix("my-bucket", "team/folder"), + prefix("my-bucket", "team/folder/")), acls(READ), strSet("ListBucket"))); + expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls, strSet("ListBucket"))); assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative); // Expected for Ranger: READ, LIST ACLs for bucket (only ListBucket supports s3:prefix); volume READ, // key "team/folder", "team/folder/*" READ final Set expectedResolvedRanger = new LinkedHashSet<>(); expectedResolvedRanger.add( - new OzoneGrant(objSet(volume(), key("my-bucket", "team/folder"), key("my-bucket", "team/folder/*")), - acls(READ))); - expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls)); + new OzoneGrant( + objSet(volume(), key("my-bucket", "team/folder"), key("my-bucket", "team/folder/*")), + acls(READ), strSet("ListBucket"))); + expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls, strSet("ListBucket"))); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1421,24 +1573,27 @@ public void testMultipleResourcesInSeparateStatements() throws OMException { // Ensure what we got is what we expected final Set expectedResolvedNative = new LinkedHashSet<>(); - // Expected for native: bucket READ, LIST, READ_ACL, WRITE_ACL; volume READ + // Expected for native: bucket READ, LIST, READ_ACL, WRITE_ACL; volume READ; prefix "" has all supported object + // ACLs and is further restricted to the object actions + ListBucket. final Set bucketSet = objSet(bucket("my-bucket")); final Set bucketAcls = acls(READ, LIST, READ_ACL, WRITE_ACL); - expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls)); - expectedResolvedNative.add(new OzoneGrant(objSet(volume()), acls(READ))); - // Expected for native: all key ACLs on prefix "" under bucket + final Set bucketAndObjectActions = new LinkedHashSet<>( + strSet("GetBucketAcl", "PutBucketAcl", "ListBucket")); + bucketAndObjectActions.addAll(ALL_OBJECT_ACTIONS); + expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls, bucketAndObjectActions)); + expectedResolvedNative.add(new OzoneGrant(objSet(volume()), acls(READ), bucketAndObjectActions)); final Set keyPrefixSet = objSet(prefix("my-bucket", "")); - final Set keyAllAcls = acls(ALL); - expectedResolvedNative.add(new OzoneGrant(keyPrefixSet, keyAllAcls)); + final Set allKeyAcls = acls(READ, CREATE, WRITE, DELETE); + expectedResolvedNative.add(new OzoneGrant(keyPrefixSet, allKeyAcls, ALL_OBJECT_ACTIONS_WITH_LIST_BUCKET)); assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative); final Set expectedResolvedRanger = new LinkedHashSet<>(); - // Expected for Ranger: bucket READ, LIST, READ_ACL, WRITE_ACL; volume READ - expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls)); - expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ))); - // Expected for Ranger: all key acls for resource type KEY with key name "*" + // Expected for Ranger: bucket READ, LIST, READ_ACL, WRITE_ACL; volume READ; key "*" has all supported object ACLs + // and is further restricted to the object actions + ListBucket. + expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls, bucketAndObjectActions)); + expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ), bucketAndObjectActions)); final Set rangerKeySet = objSet(key("my-bucket", "*")); - expectedResolvedRanger.add(new OzoneGrant(rangerKeySet, keyAllAcls)); + expectedResolvedRanger.add(new OzoneGrant(rangerKeySet, allKeyAcls, ALL_OBJECT_ACTIONS_WITH_LIST_BUCKET)); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1465,17 +1620,26 @@ public void testMultipleResourcesInOneStatement() throws OMException { // Ensure what we got is what we expected final Set expectedResolvedNative = new LinkedHashSet<>(); - // Expected for native: all for bucket and key acls; volume READ - final Set resourceSetNative = objSet(bucket("my-bucket"), prefix("my-bucket", "")); - expectedResolvedNative.add(new OzoneGrant(resourceSetNative, acls(ALL))); - expectedResolvedNative.add(new OzoneGrant(objSet(volume()), acls(READ))); + // Expected for native: bucket union of supported bucket ACLs; prefix "" union of supported object ACLs; volume READ + final Set bucketSet = objSet(bucket("my-bucket")); + expectedResolvedNative.add( + new OzoneGrant( + bucketSet, acls(READ, LIST, CREATE, DELETE, READ_ACL, WRITE_ACL), ALL_BUCKET_AND_OBJECT_ACTIONS)); + expectedResolvedNative.add( + new OzoneGrant( + objSet(prefix("my-bucket", "")), acls(READ, CREATE, WRITE, DELETE), ALL_OBJECT_ACTIONS_WITH_LIST_BUCKET)); + expectedResolvedNative.add(new OzoneGrant(objSet(volume()), acls(READ), ALL_BUCKET_AND_OBJECT_ACTIONS)); assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative); final Set expectedResolvedRanger = new LinkedHashSet<>(); - // Expected for Ranger: all for bucket and key acls; volume READ - final Set resourceSetRanger = objSet(bucket("my-bucket"), key("my-bucket", "*")); - expectedResolvedRanger.add(new OzoneGrant(resourceSetRanger, acls(ALL))); - expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ))); + // Expected for Ranger: bucket union of supported bucket ACLs; key "*" union of supported object ACLs; volume READ + expectedResolvedRanger.add( + new OzoneGrant( + bucketSet, acls(READ, LIST, CREATE, DELETE, READ_ACL, WRITE_ACL), ALL_BUCKET_AND_OBJECT_ACTIONS)); + expectedResolvedRanger.add( + new OzoneGrant( + objSet(key("my-bucket", "*")), acls(READ, CREATE, WRITE, DELETE), ALL_OBJECT_ACTIONS_WITH_LIST_BUCKET)); + expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ), ALL_BUCKET_AND_OBJECT_ACTIONS)); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1502,22 +1666,22 @@ public void testMultipleResourcesWithDifferentBucketsAndDeepPathsInOneStatement( // Ensure what we got is what we expected final Set expectedResolvedNative = new LinkedHashSet<>(); - // Expected for native: all key ACLs on prefix "team/folder1/security/" under - // my-bucket and all key ACLs on prefix "team/folder2/misc/" under my-bucket2; bucket READ; volume READ + // Expected for native: all supported object ACLs on both prefixes; bucket READ; volume READ final Set keyPrefixSet = objSet( prefix("my-bucket", "team/folder1/security/"), prefix("my-bucket2", "team/folder2/misc/")); - final Set keyAllAcls = acls(ALL); - expectedResolvedNative.add(new OzoneGrant(keyPrefixSet, keyAllAcls)); - expectedResolvedNative.add(new OzoneGrant(objSet(volume(), bucket("my-bucket"), bucket("my-bucket2")), acls(READ))); + expectedResolvedNative.add(new OzoneGrant(keyPrefixSet, acls(READ, CREATE, WRITE, DELETE), ALL_OBJECT_ACTIONS)); + expectedResolvedNative.add( + new OzoneGrant(objSet(volume(), bucket("my-bucket"), bucket("my-bucket2")), acls(READ), ALL_OBJECT_ACTIONS)); assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative); final Set expectedResolvedRanger = new LinkedHashSet<>(); - // Expected for Ranger: all key acls for resource type KEY with key name + // Expected for Ranger: all supported object ACLs for resource type KEY with key name // "team/folder1/security/*" under my-bucket and "team/folder2/misc/*" under my-bucket2; bucket READ; volume READ final Set rangerKeySet = objSet( key("my-bucket", "team/folder1/security/*"), key("my-bucket2", "team/folder2/misc/*")); - expectedResolvedRanger.add(new OzoneGrant(rangerKeySet, keyAllAcls)); - expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), bucket("my-bucket"), bucket("my-bucket2")), acls(READ))); + expectedResolvedRanger.add(new OzoneGrant(rangerKeySet, acls(READ, CREATE, WRITE, DELETE), ALL_OBJECT_ACTIONS)); + expectedResolvedRanger.add( + new OzoneGrant(objSet(volume(), bucket("my-bucket"), bucket("my-bucket2")), acls(READ), ALL_OBJECT_ACTIONS)); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1573,8 +1737,8 @@ public void testListBucketWithWildcard() throws OMException { // Expected for Ranger: bucket READ and LIST on wildcard pattern; volume and key "*" READ final Set bucketSet = objSet(bucket("proj-*")); final Set bucketAcls = acls(READ, LIST); - expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls)); - expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("proj-*", "*")), acls(READ))); + expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls, strSet("ListBucket"))); + expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("proj-*", "*")), acls(READ), strSet("ListBucket"))); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1601,16 +1765,22 @@ public void testListBucketOperationsWithNoPrefixes() throws OMException { // Expected for native: bucket READ and LIST; volume, prefix "" READ final Set bucketSet = objSet(bucket("proj")); final Set bucketAcls = acls(READ, LIST); - final Set nativeReadObjects = objSet(volume(), prefix("proj", "")); - expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls)); - expectedResolvedNative.add(new OzoneGrant(nativeReadObjects, acls(READ))); + expectedResolvedNative.add( + new OzoneGrant(bucketSet, bucketAcls, strSet("ListBucket", "ListBucketMultipartUploads"))); + expectedResolvedNative.add( + new OzoneGrant(objSet(volume()), acls(READ), strSet("ListBucket", "ListBucketMultipartUploads"))); + expectedResolvedNative.add( + new OzoneGrant(objSet(prefix("proj", "")), acls(READ), strSet("ListBucket"))); assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative); // Expected for Ranger: bucket READ and LIST; volume, key "*" READ - final Set rangerReadObjects = objSet(volume(), key("proj", "*")); final Set expectedResolvedRanger = new LinkedHashSet<>(); - expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls)); - expectedResolvedRanger.add(new OzoneGrant(rangerReadObjects, acls(READ))); + expectedResolvedRanger.add( + new OzoneGrant(bucketSet, bucketAcls, strSet("ListBucket", "ListBucketMultipartUploads"))); + expectedResolvedRanger.add( + new OzoneGrant(objSet(volume()), acls(READ), strSet("ListBucket", "ListBucketMultipartUploads"))); + expectedResolvedRanger.add( + new OzoneGrant(objSet(key("proj", "*")), acls(READ), strSet("ListBucket"))); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1649,16 +1819,20 @@ public void testIgnoresUnsupportedActionsWhenSupportedActionsAreIncluded() throw // Expected for native: READ, LIST bucket acls; volume and prefixes "team/folder", "team/folder/" READ final Set bucketSet = objSet(bucket("bucket1")); final Set bucketAcls = acls(READ, LIST); - expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls)); - expectedResolvedNative.add(new OzoneGrant( - objSet(volume(), prefix("bucket1", "team/folder"), prefix("bucket1", "team/folder/")), acls(READ))); + expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls, strSet("ListBucket"))); + expectedResolvedNative.add( + new OzoneGrant( + objSet(volume(), prefix("bucket1", "team/folder"), prefix("bucket1", "team/folder/")), + acls(READ), strSet("ListBucket"))); assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative); final Set expectedResolvedRanger = new LinkedHashSet<>(); // Expected for Ranger: READ, LIST bucket acls; volume and keys "team/folder" and "team/folder/*" READ - expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls)); - expectedResolvedRanger.add(new OzoneGrant( - objSet(volume(), key("bucket1", "team/folder"), key("bucket1", "team/folder/*")), acls(READ))); + expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls, strSet("ListBucket"))); + expectedResolvedRanger.add( + new OzoneGrant( + objSet(volume(), key("bucket1", "team/folder"), key("bucket1", "team/folder/*")), + acls(READ), strSet("ListBucket"))); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1698,15 +1872,17 @@ public void testListAndGetWithPrefixConditionSkipsObjectAction() throws OMExcept // Expected for native (GetObject is ignored because s3:prefix is present): READ, LIST bucket acls; volume READ; // prefix "log/team" READ final Set expectedResolvedNative = new LinkedHashSet<>(); - expectedResolvedNative.add(new OzoneGrant(objSet(bucket("logs")), acls(READ, LIST))); - expectedResolvedNative.add(new OzoneGrant(objSet(volume(), prefix("logs", "team/")), acls(READ))); + expectedResolvedNative.add(new OzoneGrant(objSet(bucket("logs")), acls(READ, LIST), strSet("ListBucket"))); + expectedResolvedNative.add( + new OzoneGrant(objSet(volume(), prefix("logs", "team/")), acls(READ), strSet("ListBucket"))); assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative); // Expected for Ranger (GetObject is ignored because s3:prefix is present): READ, LIST bucket acls; volume READ; // key "log/team/*" READ final Set expectedResolvedRanger = new LinkedHashSet<>(); - expectedResolvedRanger.add(new OzoneGrant(objSet(bucket("logs")), acls(READ, LIST))); - expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("logs", "team/*")), acls(READ))); + expectedResolvedRanger.add(new OzoneGrant(objSet(bucket("logs")), acls(READ, LIST), strSet("ListBucket"))); + expectedResolvedRanger.add( + new OzoneGrant(objSet(volume(), key("logs", "team/*")), acls(READ), strSet("ListBucket"))); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1746,8 +1922,8 @@ public void testObjectResourceWithWildcardInMiddle() throws OMException { // Ensure what we got is what we expected final Set expectedResolvedRanger = new LinkedHashSet<>(); // Expected for Ranger: READ acl on key "file*.log", bucket READ, volume READ - final Set readObjectsRanger = objSet(key("logs", "file*.log"), bucket("logs"), volume()); - expectedResolvedRanger.add(new OzoneGrant(readObjectsRanger, acls(READ))); + expectedResolvedRanger.add( + new OzoneGrant(objSet(key("logs", "file*.log"), bucket("logs"), volume()), acls(READ), strSet("GetObject"))); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1767,14 +1943,16 @@ public void testObjectResourceWithPrefixWildcard() throws OMException { // Ensure what we got is what we expected final Set expectedResolvedNative = new LinkedHashSet<>(); // Expected for native: READ acl on prefix "file" under bucket, bucket READ, volume READ - final Set readObjectsNative = objSet(prefix("myBucket", "file"), bucket("myBucket"), volume()); - expectedResolvedNative.add(new OzoneGrant(readObjectsNative, acls(READ))); + expectedResolvedNative.add( + new OzoneGrant( + objSet(prefix("myBucket", "file"), bucket("myBucket"), volume()), acls(READ), strSet("GetObject"))); assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative); final Set expectedResolvedRanger = new LinkedHashSet<>(); // Expected for Ranger: READ acl on key "file*", bucket READ, volume READ - final Set readObjectsRanger = objSet(key("myBucket", "file*"), bucket("myBucket"), volume()); - expectedResolvedRanger.add(new OzoneGrant(readObjectsRanger, acls(READ))); + expectedResolvedRanger.add( + new OzoneGrant( + objSet(key("myBucket", "file*"), bucket("myBucket"), volume()), acls(READ), strSet("GetObject"))); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1798,9 +1976,10 @@ public void testBucketActionOnAllResources() throws OMException { // Ensure what we got is what we expected final Set expectedResolvedRanger = new LinkedHashSet<>(); // Expected for Ranger: READ and LIST on volume and bucket (wildcard), READ on key "*" - final Set resourceSet = objSet(volume(), bucket("*")); - expectedResolvedRanger.add(new OzoneGrant(resourceSet, acls(READ, LIST))); - expectedResolvedRanger.add(new OzoneGrant(objSet(key("*", "*")), acls(READ))); + expectedResolvedRanger.add( + new OzoneGrant(objSet(volume()), acls(READ, LIST), strSet("ListAllMyBuckets", "ListBucket"))); + expectedResolvedRanger.add(new OzoneGrant(objSet(bucket("*")), acls(READ, LIST), strSet("ListBucket"))); + expectedResolvedRanger.add(new OzoneGrant(objSet(key("*", "*")), acls(READ), strSet("ListBucket"))); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1823,8 +2002,8 @@ public void testObjectActionOnAllResources() throws OMException { // Expected for Ranger: CREATE and WRITE key acls on wildcard pattern, bucket READ, volume READ final Set keySet = objSet(key("*", "*")); final Set keyAcls = acls(CREATE, WRITE); - expectedResolvedRanger.add(new OzoneGrant(keySet, keyAcls)); - expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), bucket("*")), acls(READ))); + expectedResolvedRanger.add(new OzoneGrant(keySet, keyAcls, strSet("PutObject"))); + expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), bucket("*")), acls(READ), strSet("PutObject"))); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1851,9 +2030,10 @@ public void testAllActionsOnAllResourcesWithPrefixCondition() throws OMException final Set expectedResolvedRanger = new LinkedHashSet<>(); // Expected for Ranger: (only ListBucket supports s3:prefix) READ volume; READ, LIST acl on bucket; // READ on key "team/folder", "team/folder/*" - expectedResolvedRanger.add(new OzoneGrant(objSet(bucket("*")), acls(READ, LIST))); + expectedResolvedRanger.add(new OzoneGrant(objSet(bucket("*")), acls(READ, LIST), strSet("ListBucket"))); expectedResolvedRanger.add( - new OzoneGrant(objSet(volume(), key("*", "team/folder"), key("*", "team/folder/*")), acls(READ))); + new OzoneGrant( + objSet(volume(), key("*", "team/folder"), key("*", "team/folder/*")), acls(READ), strSet("ListBucket"))); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1873,10 +2053,16 @@ public void testAllActionsOnAllResources() throws OMException { final Set resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER); // Ensure what we got is what we expected final Set expectedResolvedRanger = new LinkedHashSet<>(); - // Expected for Ranger: READ, LIST acl on volume, ALL acl bucket (wildcard) and key (wildcard) - final Set resourceSet = objSet(bucket("*"), key("*", "*")); - expectedResolvedRanger.add(new OzoneGrant(resourceSet, acls(ALL))); - expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ, LIST))); + // Expected for Ranger: + // - volume READ, LIST (ListAllMyBuckets applies at volume scope; other actions imply navigation READ) + // - bucket union of supported bucket ACLs + // - key union of supported object ACLs (and is only action-restricted to object actions + ListBucket) + expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ, LIST), emptySet())); + expectedResolvedRanger.add( + new OzoneGrant( + objSet(bucket("*")), acls(READ, LIST, CREATE, DELETE, READ_ACL, WRITE_ACL), ALL_BUCKET_AND_OBJECT_ACTIONS)); + expectedResolvedRanger.add( + new OzoneGrant(objSet(key("*", "*")), acls(READ, CREATE, WRITE, DELETE), ALL_OBJECT_ACTIONS_WITH_LIST_BUCKET)); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1896,12 +2082,13 @@ public void testAllActionsOnAllBucketResources() throws OMException { final Set resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER); // Ensure what we got is what we expected final Set expectedResolvedRanger = new LinkedHashSet<>(); - // Expected for Ranger: ALL bucket acls on wildcard pattern, volume READ, key "*" READ + // Expected for Ranger: union of supported bucket ACLs on wildcard bucket; volume READ, LIST; key "*" READ final Set bucketSet = objSet(bucket("*")); - final Set bucketAcls = acls(ALL); - expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls)); - expectedResolvedRanger.add(new OzoneGrant(objSet(key("*", "*")), acls(READ))); - expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ, LIST))); + final Set bucketAcls = acls(READ, LIST, CREATE, DELETE, READ_ACL, WRITE_ACL); + expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls, ALL_BUCKET_ACTIONS)); + expectedResolvedRanger.add(new OzoneGrant(objSet(key("*", "*")), acls(READ), strSet("ListBucket"))); + expectedResolvedRanger.add( + new OzoneGrant(objSet(volume()), acls(READ, LIST), ALL_BUCKET_ACTIONS_WITH_LIST_ALL_MY_BUCKETS)); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1921,11 +2108,11 @@ public void testAllActionsOnAllObjectResources() throws OMException { final Set resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER); // Ensure what we got is what we expected final Set expectedResolvedRanger = new LinkedHashSet<>(); - // Expected for Ranger: ALL key acls on wildcard pattern; bucket READ; volume READ + // Expected for Ranger: union of supported object ACLs on wildcard key; bucket READ; volume READ final Set keySet = objSet(key("*", "*")); - final Set keyAcls = acls(ALL); - expectedResolvedRanger.add(new OzoneGrant(keySet, keyAcls)); - expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), bucket("*")), acls(READ))); + final Set keyAcls = acls(READ, CREATE, WRITE, DELETE); + expectedResolvedRanger.add(new OzoneGrant(keySet, keyAcls, ALL_OBJECT_ACTIONS)); + expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), bucket("*")), acls(READ), ALL_OBJECT_ACTIONS)); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1950,18 +2137,24 @@ public void testWildcardActionGroupGetStar() throws OMException { // Expected for native: bucket READ, READ_ACL acls final Set bucketSet = objSet(bucket("my-bucket")); final Set bucketAcls = acls(READ, READ_ACL); - expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls)); + expectedResolvedNative.add( + new OzoneGrant(bucketSet, bucketAcls, strSet("GetBucketAcl", "GetObject", "GetObjectTagging"))); // Expected for native: READ acl on prefix "" under bucket; volume READ - final Set readObjectsNative = objSet(prefix("my-bucket", ""), volume()); - expectedResolvedNative.add(new OzoneGrant(readObjectsNative, acls(READ))); + expectedResolvedNative.add( + new OzoneGrant(objSet(prefix("my-bucket", "")), acls(READ), strSet("GetObject", "GetObjectTagging"))); + expectedResolvedNative.add( + new OzoneGrant(objSet(volume()), acls(READ), strSet("GetBucketAcl", "GetObject", "GetObjectTagging"))); assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative); final Set expectedResolvedRanger = new LinkedHashSet<>(); // Expected for Ranger: bucket READ, READ_ACL acls - expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls)); + expectedResolvedRanger.add( + new OzoneGrant(bucketSet, bucketAcls, strSet("GetBucketAcl", "GetObject", "GetObjectTagging"))); // Expected for Ranger: READ key acl for resource type KEY with key name "*"; volume READ - final Set readObjectsRanger = objSet(key("my-bucket", "*"), volume()); - expectedResolvedRanger.add(new OzoneGrant(readObjectsRanger, acls(READ))); + expectedResolvedRanger.add( + new OzoneGrant(objSet(key("my-bucket", "*")), acls(READ), strSet("GetObject", "GetObjectTagging"))); + expectedResolvedRanger.add( + new OzoneGrant(objSet(volume()), acls(READ), strSet("GetBucketAcl", "GetObject", "GetObjectTagging"))); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -1983,21 +2176,33 @@ public void testWildcardActionGroupListStar() throws OMException { // Ensure what we got is what we expected final Set expectedResolvedNative = new LinkedHashSet<>(); - // Expected for native: READ, LIST bucket acls - final Set bucketSet = objSet(bucket("my-bucket")); - final Set bucketAcls = acls(READ, LIST); - expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls)); - // Expected for native: READ acl on prefix "" under bucket; volume READ - final Set readObjectsNative = objSet(prefix("my-bucket", ""), volume()); - expectedResolvedNative.add(new OzoneGrant(readObjectsNative, acls(READ))); + // Expected for native: READ, LIST bucket acls, READ acl on prefix "" under bucket; volume READ + final Set readAndListObjectsNative = objSet(bucket("my-bucket")); + expectedResolvedNative.add( + new OzoneGrant( + readAndListObjectsNative, acls(READ, LIST), + strSet("ListBucket", "ListBucketMultipartUploads", "ListMultipartUploadParts"))); + expectedResolvedNative.add( + new OzoneGrant(objSet(volume()), acls(READ), + strSet("ListBucket", "ListBucketMultipartUploads", "ListMultipartUploadParts"))); + expectedResolvedNative.add( + new OzoneGrant(objSet(prefix("my-bucket", "")), acls(READ), strSet("ListBucket", "ListMultipartUploadParts"))); assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative); final Set expectedResolvedRanger = new LinkedHashSet<>(); - // Expected for Ranger: READ, LIST bucket acls - expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls)); - // Expected for Ranger: READ key acl for resource type KEY with key name "*"; volume READ - final Set readObjectsRanger = objSet(key("my-bucket", "*"), volume()); - expectedResolvedRanger.add(new OzoneGrant(readObjectsRanger, acls(READ))); + // Expected for Ranger: READ, LIST bucket acls; READ key acl for resource type KEY with key name "*"; + // volume READ + final Set readAndListObjectRanger = objSet(bucket("my-bucket")); + expectedResolvedRanger.add( + new OzoneGrant( + readAndListObjectRanger, acls(READ, LIST), + strSet("ListBucket", "ListBucketMultipartUploads", "ListMultipartUploadParts"))); + expectedResolvedRanger.add( + new OzoneGrant( + objSet(volume()), acls(READ), + strSet("ListBucket", "ListBucketMultipartUploads", "ListMultipartUploadParts"))); + expectedResolvedRanger.add( + new OzoneGrant(objSet(key("my-bucket", "*")), acls(READ), strSet("ListBucket", "ListMultipartUploadParts"))); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -2022,23 +2227,27 @@ public void testWildcardActionGroupPutStar() throws OMException { // Expected for native: bucket READ, READ_ACL, WRITE_ACL acl final Set bucketSet = objSet(bucket("my-bucket")); final Set bucketAcl = acls(READ, READ_ACL, WRITE_ACL); - expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcl)); + expectedResolvedNative.add( + new OzoneGrant(bucketSet, bucketAcl, strSet("PutBucketAcl", "PutObject", "PutObjectTagging"))); // Expected for native: CREATE, WRITE acls on prefix "" under bucket final Set keyPrefixSet = objSet(prefix("my-bucket", "")); final Set keyAcls = acls(CREATE, WRITE); - expectedResolvedNative.add(new OzoneGrant(keyPrefixSet, keyAcls)); + expectedResolvedNative.add(new OzoneGrant(keyPrefixSet, keyAcls, strSet("PutObject", "PutObjectTagging"))); // Expected for native: volume READ - expectedResolvedNative.add(new OzoneGrant(objSet(volume()), acls(READ))); + expectedResolvedNative.add( + new OzoneGrant(objSet(volume()), acls(READ), strSet("PutBucketAcl", "PutObject", "PutObjectTagging"))); assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative); final Set expectedResolvedRanger = new LinkedHashSet<>(); // Expected for Ranger: bucket READ, READ_ACL, WRITE_ACL acl - expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcl)); + expectedResolvedRanger.add( + new OzoneGrant(bucketSet, bucketAcl, strSet("PutBucketAcl", "PutObject", "PutObjectTagging"))); // Expected for Ranger: CREATE, WRITE key acls for resource type KEY with key name "*" final Set rangerKeySet = objSet(key("my-bucket", "*")); - expectedResolvedRanger.add(new OzoneGrant(rangerKeySet, keyAcls)); + expectedResolvedRanger.add(new OzoneGrant(rangerKeySet, keyAcls, strSet("PutObject", "PutObjectTagging"))); // Expected for Ranger: volume READ - expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ))); + expectedResolvedRanger.add( + new OzoneGrant(objSet(volume()), acls(READ), strSet("PutBucketAcl", "PutObject", "PutObjectTagging"))); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); } @@ -2062,17 +2271,27 @@ public void testWildcardActionGroupDeleteStar() throws OMException { final Set expectedResolvedNative = new LinkedHashSet<>(); // Expected for native: DELETE and WRITE on prefix "" under bucket; bucket READ, DELETE; volume READ final Set resourceSetNative = objSet(prefix("my-bucket", "")); - expectedResolvedNative.add(new OzoneGrant(resourceSetNative, acls(DELETE, WRITE))); - expectedResolvedNative.add(new OzoneGrant(objSet(bucket("my-bucket")), acls(READ, DELETE))); - expectedResolvedNative.add(new OzoneGrant(objSet(volume()), acls(READ))); + expectedResolvedNative.add( + new OzoneGrant(resourceSetNative, acls(DELETE, WRITE), strSet("DeleteObject", "DeleteObjectTagging"))); + expectedResolvedNative.add( + new OzoneGrant( + objSet(bucket("my-bucket")), acls(READ, DELETE), + strSet("DeleteBucket", "DeleteObject", "DeleteObjectTagging"))); + expectedResolvedNative.add( + new OzoneGrant(objSet(volume()), acls(READ), strSet("DeleteBucket", "DeleteObject", "DeleteObjectTagging"))); assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative); final Set expectedResolvedRanger = new LinkedHashSet<>(); // Expected for Ranger: DELETE and WRITE on resource type KEY with key name "*"; bucket READ, DELETE; volume READ final Set resourceSetRanger = objSet(key("my-bucket", "*")); - expectedResolvedRanger.add(new OzoneGrant(resourceSetRanger, acls(DELETE, WRITE))); - expectedResolvedRanger.add(new OzoneGrant(objSet(bucket("my-bucket")), acls(READ, DELETE))); - expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ))); + expectedResolvedRanger.add( + new OzoneGrant(resourceSetRanger, acls(DELETE, WRITE), strSet("DeleteObject", "DeleteObjectTagging"))); + expectedResolvedRanger.add( + new OzoneGrant( + objSet(bucket("my-bucket")), acls(READ, DELETE), + strSet("DeleteBucket", "DeleteObject", "DeleteObjectTagging"))); + expectedResolvedRanger.add( + new OzoneGrant(objSet(volume()), acls(READ), strSet("DeleteBucket", "DeleteObject", "DeleteObjectTagging"))); assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger); }