diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index cf7bc3ddcf248..eff42c1050f6b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -1362,6 +1362,19 @@ private Constants() {
*/
public static final String XA_HEADER_PREFIX = "header.";
+ /**
+ * S3 cross region access enabled ?
+ * Value: {@value}.
+ */
+
+ public static final String AWS_S3_CROSS_REGION_ACCESS_ENABLED =
+ "fs.s3a.cross.region.access.enabled";
+ /**
+ * Default value for S3 cross region access enabled: {@value}.
+ */
+ public static final boolean AWS_S3_CROSS_REGION_ACCESS_ENABLED_DEFAULT = true;
+
+
/**
* AWS S3 region for the bucket. When set bypasses the construction of
* region through endpoint url.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
index 4b3db99924747..c9c3eee30ea5d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
@@ -55,6 +55,8 @@
import org.apache.hadoop.fs.store.LogExactlyOnce;
import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
+import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_CROSS_REGION_ACCESS_ENABLED;
+import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_CROSS_REGION_ACCESS_ENABLED_DEFAULT;
import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_DEFAULT_REGION;
import static org.apache.hadoop.fs.s3a.Constants.CENTRAL_ENDPOINT;
import static org.apache.hadoop.fs.s3a.Constants.FIPS_ENDPOINT;
@@ -259,8 +261,10 @@ protected ClientOverrideConfiguration.Builder createClientOverrideConfiguration(
*
If endpoint is configured via via fs.s3a.endpoint, set it.
* If no region is configured, try to parse region from endpoint.
* If no region is configured, and it could not be parsed from the endpoint,
- * set the default region as US_EAST_2 and enable cross region access.
+ * set the default region as US_EAST_2
* If configured region is empty, fallback to SDK resolution chain.
+ * S3 cross region is enabled by default irrespective of region or endpoint
+ * is set or not.
*
*
* @param builder S3 client builder.
@@ -320,7 +324,6 @@ private , ClientT> void
builder.endpointOverride(endpoint);
LOG.debug("Setting endpoint to {}", endpoint);
} else {
- builder.crossRegionAccessEnabled(true);
origin = "central endpoint with cross region access";
LOG.debug("Enabling cross region access for endpoint {}",
endpointStr);
@@ -333,7 +336,6 @@ private , ClientT> void
// no region is configured, and none could be determined from the endpoint.
// Use US_EAST_2 as default.
region = Region.of(AWS_S3_DEFAULT_REGION);
- builder.crossRegionAccessEnabled(true);
builder.region(region);
origin = "cross region access fallback";
} else if (configuredRegion.isEmpty()) {
@@ -344,8 +346,14 @@ private , ClientT> void
LOG.debug(SDK_REGION_CHAIN_IN_USE);
origin = "SDK region chain";
}
-
- LOG.debug("Setting region to {} from {}", region, origin);
+ boolean isCrossRegionAccessEnabled = conf.getBoolean(AWS_S3_CROSS_REGION_ACCESS_ENABLED,
+ AWS_S3_CROSS_REGION_ACCESS_ENABLED_DEFAULT);
+ // s3 cross region access
+ if (isCrossRegionAccessEnabled) {
+ builder.crossRegionAccessEnabled(true);
+ }
+ LOG.debug("Setting region to {} from {} with cross region access {}",
+ region, origin, isCrossRegionAccessEnabled);
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/connecting.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/connecting.md
index d39c480b7cc5a..6fa37750ded8c 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/connecting.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/connecting.md
@@ -48,6 +48,16 @@ There are multiple ways to connect to an S3 bucket
The S3A connector supports all these; S3 Endpoints are the primary mechanism used -either explicitly declared or automatically determined from the declared region of the bucket.
+The S3A connector supports S3 cross region access via AWS SDK which is enabled by default. This allows users to access S3 buckets in a different region than the one defined in the S3 endpoint/region configuration, as long as they are within the same AWS partition. However, S3 cross-region access can be disabled by:
+```xml
+
+ fs.s3a.cross.region.access.enabled
+ false
+ S3 cross region access
+
+```
+
+
Not supported:
* AWS [Snowball](https://aws.amazon.com/snowball/).
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
index a3b994054e4d3..967ba885bc90f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -439,6 +439,7 @@ public void testCustomUserAgent() throws Exception {
@Test
public void testRequestTimeout() throws Exception {
conf = new Configuration();
+ skipIfCrossRegionClient(conf);
// remove the safety check on minimum durations.
AWSClientConfig.setMinimumOperationDuration(Duration.ZERO);
try {
@@ -632,8 +633,8 @@ public static boolean isSTSSignerCalled() {
*/
private static void skipIfCrossRegionClient(
Configuration configuration) {
- if (configuration.get(ENDPOINT, null) == null
- && configuration.get(AWS_REGION, null) == null) {
+ if (configuration.getBoolean(AWS_S3_CROSS_REGION_ACCESS_ENABLED,
+ AWS_S3_CROSS_REGION_ACCESS_ENABLED_DEFAULT)) {
skip("Skipping test as cross region client is in use ");
}
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
index d06224df5b355..80b061de03183 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
@@ -44,8 +44,10 @@
import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext;
import org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
import static org.apache.hadoop.fs.s3a.Constants.ALLOW_REQUESTER_PAYS;
import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
+import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_CROSS_REGION_ACCESS_ENABLED;
import static org.apache.hadoop.fs.s3a.Constants.CENTRAL_ENDPOINT;
import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
import static org.apache.hadoop.fs.s3a.Constants.FIPS_ENDPOINT;
@@ -71,6 +73,8 @@ public class ITestS3AEndpointRegion extends AbstractS3ATestBase {
private static final String US_WEST_2 = "us-west-2";
+ private static final String SA_EAST_1 = "sa-east-1";
+
private static final String EU_WEST_2 = "eu-west-2";
private static final String CN_NORTHWEST_1 = "cn-northwest-1";
@@ -346,6 +350,46 @@ public void testCentralEndpointAndDifferentRegionThanBucket() throws Throwable {
assertRequesterPaysFileExistence(newConf);
}
+ @Test
+ public void testWithOutCrossRegionAccess() throws Exception {
+ describe("Verify cross region access fails when disabled");
+ // skip the test if the region is sa-east-1
+ skipCrossRegionTest();
+ final Configuration newConf = new Configuration(getConfiguration());
+ removeBaseAndBucketOverrides(newConf,
+ ENDPOINT,
+ AWS_S3_CROSS_REGION_ACCESS_ENABLED,
+ AWS_REGION);
+ // disable cross region access
+ newConf.setBoolean(AWS_S3_CROSS_REGION_ACCESS_ENABLED, false);
+ newConf.set(AWS_REGION, SA_EAST_1);
+ try (S3AFileSystem fs = new S3AFileSystem()) {
+ fs.initialize(getFileSystem().getUri(), newConf);
+ intercept(AWSRedirectException.class,
+ "does not match the AWS region containing the bucket",
+ () -> fs.exists(getFileSystem().getWorkingDirectory()));
+ }
+ }
+
+ @Test
+ public void testWithCrossRegionAccess() throws Exception {
+ describe("Verify cross region access succeed when enabled");
+ // skip the test if the region is sa-east-1
+ skipCrossRegionTest();
+ final Configuration newConf = new Configuration(getConfiguration());
+ removeBaseAndBucketOverrides(newConf,
+ ENDPOINT,
+ AWS_S3_CROSS_REGION_ACCESS_ENABLED,
+ AWS_REGION);
+ // enable cross region access
+ newConf.setBoolean(AWS_S3_CROSS_REGION_ACCESS_ENABLED, true);
+ newConf.set(AWS_REGION, SA_EAST_1);
+ try (S3AFileSystem fs = new S3AFileSystem()) {
+ fs.initialize(getFileSystem().getUri(), newConf);
+ fs.exists(getFileSystem().getWorkingDirectory());
+ }
+ }
+
@Test
public void testCentralEndpointAndSameRegionAsBucket() throws Throwable {
describe("Access public bucket using central endpoint and region "
@@ -478,6 +522,16 @@ public void testCentralEndpointAndNullRegionFipsWithCRUD() throws Throwable {
assertOpsUsingNewFs();
}
+ /**
+ * Skip the test if the region is null or sa-east-1.
+ */
+ private void skipCrossRegionTest() throws IOException {
+ String region = getFileSystem().getS3AInternals().getBucketMetadata().bucketRegion();
+ if (region == null || SA_EAST_1.equals(region)) {
+ skip("Skipping test since region is null or it is set to sa-east-1");
+ }
+ }
+
private void assertOpsUsingNewFs() throws IOException {
final String file = getMethodName();
final Path basePath = methodPath();