Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
HDDS-7035. Generate strToSign before applying virtual host style filt…
…er. (#5123)
- Loading branch information
1 parent
850bc96
commit 1c48935
Showing
24 changed files
with
741 additions
and
246 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
47 changes: 47 additions & 0 deletions
47
hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3g-virtual-host.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| # Licensed to the Apache Software Foundation (ASF) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The ASF licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| version: "3.8" | ||
|
|
||
| x-s3g-virtual-host-config: | ||
| &s3g-virtual-host-config | ||
| environment: | ||
| - OZONE-SITE.XML_ozone.s3g.domain.name=s3g.internal | ||
| services: | ||
| datanode1: | ||
| <<: *s3g-virtual-host-config | ||
| datanode2: | ||
| <<: *s3g-virtual-host-config | ||
| datanode3: | ||
| <<: *s3g-virtual-host-config | ||
| om1: | ||
| <<: *s3g-virtual-host-config | ||
| om2: | ||
| <<: *s3g-virtual-host-config | ||
| om3: | ||
| <<: *s3g-virtual-host-config | ||
| scm1.org: | ||
| <<: *s3g-virtual-host-config | ||
| scm2.org: | ||
| <<: *s3g-virtual-host-config | ||
| scm3.org: | ||
| <<: *s3g-virtual-host-config | ||
| s3g: | ||
| <<: *s3g-virtual-host-config | ||
| extra_hosts: | ||
| - "bucket1.s3g.internal: 172.25.0.114" | ||
| recon: | ||
| <<: *s3g-virtual-host-config |
34 changes: 34 additions & 0 deletions
34
hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-s3g-virtual-host.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| #!/usr/bin/env bash | ||
| # Licensed to the Apache Software Foundation (ASF) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The ASF licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| #suite:HA-secure | ||
|
|
||
| COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" | ||
| export COMPOSE_DIR | ||
|
|
||
| export SECURITY_ENABLED=true | ||
| export OM_SERVICE_ID="omservice" | ||
| export SCM=scm1.org | ||
| export COMPOSE_FILE=docker-compose.yaml:s3g-virtual-host.yaml | ||
|
|
||
| # shellcheck source=/dev/null | ||
| source "$COMPOSE_DIR/../testlib.sh" | ||
|
|
||
| start_docker_env | ||
|
|
||
| ## Run virtual host test cases | ||
| execute_robot_test s3g -N s3-virtual-host s3/awss3virtualhost.robot |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
56 changes: 56 additions & 0 deletions
56
hadoop-ozone/dist/src/main/smoketest/s3/awss3virtualhost.robot
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| # Licensed to the Apache Software Foundation (ASF) under one or more | ||
| # contributor license agreements. See the NOTICE file distributed with | ||
| # this work for additional information regarding copyright ownership. | ||
| # The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| # (the "License"); you may not use this file except in compliance with | ||
| # the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| *** Settings *** | ||
| Documentation S3 gateway test with aws cli using virtual host style address | ||
| Library OperatingSystem | ||
| Library String | ||
| Resource ../commonlib.robot | ||
| Resource ./commonawslib.robot | ||
| Test Timeout 5 minutes | ||
| Suite Setup Setup s3 tests | ||
| Default Tags virtual-host | ||
|
|
||
| *** Variables *** | ||
| ${ENDPOINT_URL} http://s3g.internal:9878 | ||
| ${BUCKET} bucket1 | ||
| ${OZONE_S3_ADDRESS_STYLE} virtual | ||
|
|
||
| *** Test Cases *** | ||
|
|
||
| File upload and directory list with virtual style addressing | ||
| Create bucket with name ${BUCKET} | ||
| Execute date > /tmp/testfile | ||
| ${result} = Execute AWSS3Cli cp /tmp/testfile s3://${BUCKET} --debug | ||
| Should contain ${result} url=http://bucket1.s3g.internal:9878/ | ||
| Should contain ${result} upload | ||
| ${result} = Execute AWSS3Cli cp /tmp/testfile s3://${BUCKET}/dir1/dir2/file --debug | ||
| Should contain ${result} url=http://bucket1.s3g.internal:9878/dir1/dir2/file | ||
| Should contain ${result} upload | ||
| ${result} = Execute AWSS3Cli ls s3://${BUCKET} --debug | ||
| Should contain ${result} url=http://bucket1.s3g.internal:9878/ | ||
| Should contain ${result} testfile | ||
| Should contain ${result} dir1 | ||
| Should not contain ${result} dir2 | ||
| ${result} = Execute AWSS3Cli ls s3://${BUCKET}/dir1/ --debug | ||
| Should contain ${result} url=http://bucket1.s3g.internal:9878/ | ||
| Should contain ${result} prefix=dir1 | ||
| Should not contain ${result} testfile | ||
| Should contain ${result} dir2/ | ||
| ${result} = Execute AWSS3Cli ls s3://${BUCKET}/dir1/dir2/file | ||
| Should not contain ${result} testfile | ||
| Should not contain ${result} dir1 | ||
| Should not contain ${result} dir2 | ||
| Should contain ${result} file |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
125 changes: 125 additions & 0 deletions
125
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/AuthorizationFilter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,125 @@ | ||
| /** | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * <p> | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * <p> | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.hadoop.ozone.s3; | ||
|
|
||
| import javax.annotation.Priority; | ||
| import javax.inject.Inject; | ||
| import javax.ws.rs.WebApplicationException; | ||
| import javax.ws.rs.container.ContainerRequestContext; | ||
| import javax.ws.rs.container.ContainerRequestFilter; | ||
| import javax.ws.rs.container.PreMatching; | ||
| import javax.ws.rs.core.Response; | ||
| import javax.ws.rs.ext.Provider; | ||
|
|
||
| import com.google.common.annotations.VisibleForTesting; | ||
| import org.apache.hadoop.ozone.s3.exception.OS3Exception; | ||
| import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; | ||
| import org.apache.hadoop.ozone.s3.signature.SignatureInfo; | ||
| import org.apache.hadoop.ozone.s3.signature.SignatureInfo.Version; | ||
| import org.apache.hadoop.ozone.s3.signature.SignatureProcessor; | ||
| import org.apache.hadoop.ozone.s3.signature.StringToSignProducer; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| import java.io.IOException; | ||
|
|
||
| import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.ACCESS_DENIED; | ||
| import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INTERNAL_ERROR; | ||
| import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.S3_AUTHINFO_CREATION_ERROR; | ||
|
|
||
| /** | ||
| * Filter used to construct string to sign from unfiltered request. | ||
| * It should be executed before all other filters as the original | ||
| * could be enriched later. | ||
| */ | ||
|
|
||
| @Provider | ||
| @PreMatching | ||
| @Priority(AuthorizationFilter.PRIORITY) | ||
| public class AuthorizationFilter implements ContainerRequestFilter { | ||
| public static final int PRIORITY = 50; | ||
|
|
||
| private static final Logger LOG = LoggerFactory.getLogger( | ||
| AuthorizationFilter.class); | ||
|
|
||
| @Inject | ||
| private SignatureProcessor signatureProcessor; | ||
|
|
||
| @Inject | ||
| private SignatureInfo signatureInfo; | ||
|
|
||
| @Override | ||
| public void filter(ContainerRequestContext context) throws | ||
| IOException { | ||
| // Skip authentication if the uri is hitting S3Secret generation or | ||
| // revocation endpoint. | ||
| if (context.getUriInfo().getRequestUri().getPath().startsWith("/secret")) { | ||
| return; | ||
| } | ||
|
|
||
| try { | ||
| signatureInfo.initialize(signatureProcessor.parseSignature()); | ||
| if (signatureInfo.getVersion() == Version.V4) { | ||
| signatureInfo.setStrToSign( | ||
| StringToSignProducer.createSignatureBase(signatureInfo, context)); | ||
| } else { | ||
| LOG.debug("Unsupported AWS signature version: {}", | ||
| signatureInfo.getVersion()); | ||
| throw S3_AUTHINFO_CREATION_ERROR; | ||
| } | ||
|
|
||
| String awsAccessId = signatureInfo.getAwsAccessId(); | ||
| // ONLY validate aws access id when needed. | ||
| if (awsAccessId == null || awsAccessId.equals("")) { | ||
| LOG.debug("Malformed s3 header. awsAccessID: {}", awsAccessId); | ||
| throw ACCESS_DENIED; | ||
| } | ||
| } catch (OS3Exception ex) { | ||
| LOG.debug("Error during Client Creation: ", ex); | ||
| throw wrapOS3Exception(ex); | ||
| } catch (Exception e) { | ||
| // For any other critical errors during object creation throw Internal | ||
| // error. | ||
| LOG.debug("Error during Client Creation: ", e); | ||
| throw wrapOS3Exception( | ||
| S3ErrorTable.newError(INTERNAL_ERROR, null, e)); | ||
| } | ||
| } | ||
|
|
||
| @VisibleForTesting | ||
| public void setSignatureParser(SignatureProcessor awsSignatureProcessor) { | ||
| this.signatureProcessor = awsSignatureProcessor; | ||
| } | ||
|
|
||
| @VisibleForTesting | ||
| public void setSignatureInfo(SignatureInfo signatureInfo) { | ||
| this.signatureInfo = signatureInfo; | ||
| } | ||
|
|
||
| @VisibleForTesting | ||
| public SignatureInfo getSignatureInfo() { | ||
| return signatureInfo; | ||
| } | ||
|
|
||
| private WebApplicationException wrapOS3Exception(OS3Exception os3Exception) { | ||
| return new WebApplicationException(os3Exception.getErrorMessage(), | ||
| os3Exception, | ||
| Response.status(os3Exception.getHttpCode()) | ||
| .entity(os3Exception.toXml()).build()); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.