Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HADOOP-19066. S3A: AWS SDK V2 - Enabling FIPS should be allowed with central endpoint #6539

Merged
merged 1 commit into from
Mar 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ public class DefaultS3ClientFactory extends Configured
*/
@VisibleForTesting
public static final String ERROR_ENDPOINT_WITH_FIPS =
"An endpoint cannot set when " + FIPS_ENDPOINT + " is true";
"Non central endpoint cannot be set when " + FIPS_ENDPOINT + " is true";

@Override
public S3Client createS3Client(
Expand Down Expand Up @@ -290,10 +290,11 @@ private <BuilderT extends S3BaseClientBuilder<BuilderT, ClientT>, ClientT> void
builder.fipsEnabled(fipsEnabled);

if (endpoint != null) {
checkArgument(!fipsEnabled,
"%s : %s", ERROR_ENDPOINT_WITH_FIPS, endpoint);
boolean endpointEndsWithCentral =
endpointStr.endsWith(CENTRAL_ENDPOINT);
checkArgument(!fipsEnabled || endpointEndsWithCentral, "%s : %s",
ERROR_ENDPOINT_WITH_FIPS,
endpoint);

// No region was configured,
// determine the region from the endpoint.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -360,8 +360,18 @@ For a single bucket:
</property>
```

If this option is `true`, the endpoint option `fs.s3a.endpoint` MUST NOT be set:
If `fs.s3a.endpoint.fips` is `true`, the endpoint option `fs.s3a.endpoint` MUST NOT be set to
any non-central endpoint value. If `fs.s3a.endpoint.fips` is `true`, the only *optionally* allowed
value for `fs.s3a.endpoint` is central endpoint `s3.amazonaws.com`.

S3A error message if `s3.eu-west-2.amazonaws.com` endpoint is used with FIPS:
```
Non central endpoint cannot be set when fs.s3a.endpoint.fips is true : https://s3.eu-west-2.amazonaws.com
```

S3A validation is used to fail-fast before the SDK returns error.

AWS SDK error message if S3A does not fail-fast:
```
A custom endpoint cannot be combined with FIPS: https://s3.eu-west-2.amazonaws.com
```
Expand All @@ -379,6 +389,9 @@ Received an UnknownHostException when attempting to interact with a service.

```

For more details on endpoint and region settings, please check
[S3 endpoint and region settings in detail](connecting.html#s3_endpoint_region_details).

*Important* OpenSSL and FIPS endpoints

Linux distributions with an FIPS-compliant SSL library may not be compatible with wildfly.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
import static org.apache.hadoop.fs.s3a.Constants.CENTRAL_ENDPOINT;
import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
import static org.apache.hadoop.fs.s3a.Constants.FIPS_ENDPOINT;
import static org.apache.hadoop.fs.s3a.Constants.PATH_STYLE_ACCESS;
import static org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.ERROR_ENDPOINT_WITH_FIPS;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
Expand Down Expand Up @@ -156,6 +157,11 @@ public void testCentralEndpoint() throws Throwable {
S3Client client = createS3Client(conf, CENTRAL_ENDPOINT, null, US_EAST_2, false);

expectInterceptorException(client);

client = createS3Client(conf, CENTRAL_ENDPOINT, null,
US_EAST_2, true);

expectInterceptorException(client);
}

@Test
Expand All @@ -168,11 +174,21 @@ public void testCentralEndpointWithRegion() throws Throwable {

expectInterceptorException(client);

client = createS3Client(conf, CENTRAL_ENDPOINT, US_WEST_2,
US_WEST_2, true);

expectInterceptorException(client);

client = createS3Client(conf, CENTRAL_ENDPOINT, US_EAST_1,
US_EAST_1, false);

expectInterceptorException(client);

client = createS3Client(conf, CENTRAL_ENDPOINT, US_EAST_1,
US_EAST_1, true);

expectInterceptorException(client);

}

@Test
Expand Down Expand Up @@ -203,7 +219,7 @@ public void testWithFipsAndEndpoint() throws Throwable {
describe("Create a client with fips and an endpoint");

intercept(IllegalArgumentException.class, ERROR_ENDPOINT_WITH_FIPS, () ->
createS3Client(getConfiguration(), CENTRAL_ENDPOINT, null, US_EAST_1, true));
createS3Client(getConfiguration(), US_WEST_2, null, US_EAST_1, true));
}

@Test
Expand Down Expand Up @@ -293,28 +309,23 @@ public void testCentralEndpointAndDifferentRegionThanBucket() throws Throwable {
ENDPOINT,
AWS_REGION,
ALLOW_REQUESTER_PAYS,
KEY_REQUESTER_PAYS_FILE);
KEY_REQUESTER_PAYS_FILE,
FIPS_ENDPOINT);

removeBaseAndBucketOverrides(
DEFAULT_REQUESTER_PAYS_BUCKET_NAME,
newConf,
ENDPOINT,
AWS_REGION,
ALLOW_REQUESTER_PAYS,
KEY_REQUESTER_PAYS_FILE);
KEY_REQUESTER_PAYS_FILE,
FIPS_ENDPOINT);

newConf.set(ENDPOINT, CENTRAL_ENDPOINT);
newConf.set(AWS_REGION, EU_WEST_1);
newConf.setBoolean(ALLOW_REQUESTER_PAYS, true);

Path filePath = new Path(PublicDatasetTestUtils
.getRequesterPaysObject(newConf));
newFS = (S3AFileSystem) filePath.getFileSystem(newConf);

Assertions
.assertThat(newFS.exists(filePath))
.describedAs("Existence of path: " + filePath)
.isTrue();
assertRequesterPaysFileExistence(newConf);
}

@Test
Expand All @@ -329,23 +340,68 @@ public void testCentralEndpointAndSameRegionAsBucket() throws Throwable {
ENDPOINT,
AWS_REGION,
ALLOW_REQUESTER_PAYS,
KEY_REQUESTER_PAYS_FILE);
KEY_REQUESTER_PAYS_FILE,
FIPS_ENDPOINT);

removeBaseAndBucketOverrides(
DEFAULT_REQUESTER_PAYS_BUCKET_NAME,
newConf,
ENDPOINT,
AWS_REGION,
ALLOW_REQUESTER_PAYS,
KEY_REQUESTER_PAYS_FILE);
KEY_REQUESTER_PAYS_FILE,
FIPS_ENDPOINT);

newConf.set(ENDPOINT, CENTRAL_ENDPOINT);
newConf.set(AWS_REGION, US_WEST_2);
newConf.setBoolean(ALLOW_REQUESTER_PAYS, true);

assertRequesterPaysFileExistence(newConf);
}

@Test
public void testCentralEndpointAndFipsForPublicBucket() throws Throwable {
describe("Access public bucket using central endpoint and region "
+ "same as that of the public bucket with fips enabled");
final Configuration conf = getConfiguration();
final Configuration newConf = new Configuration(conf);

removeBaseAndBucketOverrides(
newConf,
ENDPOINT,
AWS_REGION,
ALLOW_REQUESTER_PAYS,
KEY_REQUESTER_PAYS_FILE,
FIPS_ENDPOINT);

removeBaseAndBucketOverrides(
DEFAULT_REQUESTER_PAYS_BUCKET_NAME,
newConf,
ENDPOINT,
AWS_REGION,
ALLOW_REQUESTER_PAYS,
KEY_REQUESTER_PAYS_FILE,
FIPS_ENDPOINT);

newConf.set(ENDPOINT, CENTRAL_ENDPOINT);
newConf.set(AWS_REGION, US_WEST_2);
newConf.setBoolean(ALLOW_REQUESTER_PAYS, true);
newConf.setBoolean(FIPS_ENDPOINT, true);

assertRequesterPaysFileExistence(newConf);
}

/**
* Assert that the file exists on the requester pays public bucket.
*
* @param conf the configuration object.
* @throws IOException if file system operations encounter errors.
*/
private void assertRequesterPaysFileExistence(Configuration conf)
throws IOException {
Path filePath = new Path(PublicDatasetTestUtils
.getRequesterPaysObject(newConf));
newFS = (S3AFileSystem) filePath.getFileSystem(newConf);
.getRequesterPaysObject(conf));
newFS = (S3AFileSystem) filePath.getFileSystem(conf);

Assertions
.assertThat(newFS.exists(filePath))
Expand All @@ -364,9 +420,33 @@ public void testCentralEndpointAndNullRegionWithCRUD() throws Throwable {
removeBaseAndBucketOverrides(
newConf,
ENDPOINT,
AWS_REGION);
AWS_REGION,
FIPS_ENDPOINT);

newConf.set(ENDPOINT, CENTRAL_ENDPOINT);

newFS = new S3AFileSystem();
newFS.initialize(getFileSystem().getUri(), newConf);

assertOpsUsingNewFs();
}

@Test
public void testCentralEndpointAndNullRegionFipsWithCRUD() throws Throwable {
describe("Access the test bucket using central endpoint and"
+ " null region and fips enabled, perform file system CRUD operations");
final Configuration conf = getConfiguration();

final Configuration newConf = new Configuration(conf);

removeBaseAndBucketOverrides(
newConf,
ENDPOINT,
AWS_REGION,
FIPS_ENDPOINT);

newConf.set(ENDPOINT, CENTRAL_ENDPOINT);
newConf.setBoolean(FIPS_ENDPOINT, true);

newFS = new S3AFileSystem();
newFS.initialize(getFileSystem().getUri(), newConf);
Expand Down