Skip to content

Commit

Permalink
feat(s3): support generating endpoints from multi-region access point (…
Browse files Browse the repository at this point in the history 
…#2742)

feat(s3): support generating endpoints from multi-region access point (#2742)
  • Loading branch information
@AllanZhengYP
AllanZhengYP authored Sep 2, 2021
1 parent afeccd7 commit 49da47b
Show file tree
Hide file tree
Showing 45 changed files with 2,431 additions and 269 deletions.
2 changes: 2 additions & 0 deletions clients/client-s3/runtimeConfig.shared.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { defaultRegionInfoProvider } from "./endpoints";
import { S3SignatureV4 } from "@aws-sdk/middleware-sdk-s3";
import { Logger as __Logger } from "@aws-sdk/types";
import { parseUrl } from "@aws-sdk/url-parser";
import { S3ClientConfig } from "./S3Client";
Expand All @@ -12,6 +13,7 @@ export const getRuntimeConfig = (config: S3ClientConfig) => ({
logger: config?.logger ?? ({} as __Logger),
regionInfoProvider: config?.regionInfoProvider ?? defaultRegionInfoProvider,
serviceId: config?.serviceId ?? "S3",
signerConstructor: config?.signerConstructor ?? S3SignatureV4,
signingEscapePath: config?.signingEscapePath ?? false,
urlParser: config?.urlParser ?? parseUrl,
useArnRegion: config?.useArnRegion ?? false,
Expand Down
4 changes: 4 additions & 0 deletions ...ript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,10 @@ public Map<String, Consumer<TypeScriptWriter>> getRuntimeConfigWriters(TypeScrip
writer.write("false");
}, "useArnRegion", writer -> {
writer.write("false");
}, "signerConstructor", writer -> {
writer.addDependency(AwsDependency.S3_MIDDLEWARE)
.addImport("S3SignatureV4", "S3SignatureV4", AwsDependency.S3_MIDDLEWARE.packageName)
.write("S3SignatureV4");
});
case NODE:
return MapUtils.of("useArnRegion", writer -> {
Expand Down
1 change: 1 addition & 0 deletions packages/middleware-bucket-endpoint/src/bucketEndpointMiddleware.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ describe("bucketEndpointMiddleware", () => {
clientSigningRegion: mockRegion,
useArnRegion: false,
isCustomEndpoint: false,
disableMultiregionAccessPoints: false,
});
expect(previouslyResolvedConfig.region).toBeCalled();
expect(previouslyResolvedConfig.regionInfoProvider).toBeCalled();
Expand Down
131 changes: 63 additions & 68 deletions packages/middleware-bucket-endpoint/src/bucketEndpointMiddleware.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,81 +15,76 @@ import { bucketHostname } from "./bucketHostname";
import { getPseudoRegion } from "./bucketHostnameUtils";
import { BucketEndpointResolvedConfig } from "./configurations";

export const bucketEndpointMiddleware =
(options: BucketEndpointResolvedConfig): BuildMiddleware<any, any> =>
<Output extends MetadataBearer>(
next: BuildHandler<any, Output>,
context: HandlerExecutionContext
): BuildHandler<any, Output> =>
async (args: BuildHandlerArguments<any>): Promise<BuildHandlerOutput<Output>> => {
const { Bucket: bucketName } = args.input as { Bucket: string };
let replaceBucketInPath = options.bucketEndpoint;
const request = args.request;
if (HttpRequest.isInstance(request)) {
if (options.bucketEndpoint) {
request.hostname = bucketName;
} else if (validateArn(bucketName)) {
const bucketArn = parseArn(bucketName);
const clientRegion = getPseudoRegion(await options.region());
const { partition, signingRegion = clientRegion } = (await options.regionInfoProvider(clientRegion)) || {};
const useArnRegion = await options.useArnRegion();
const {
hostname,
bucketEndpoint,
signingRegion: modifiedSigningRegion,
signingService,
} = bucketHostname({
bucketName: bucketArn,
baseHostname: request.hostname,
accelerateEndpoint: options.useAccelerateEndpoint,
dualstackEndpoint: options.useDualstackEndpoint,
pathStyleEndpoint: options.forcePathStyle,
tlsCompatible: request.protocol === "https:",
useArnRegion,
clientPartition: partition,
clientSigningRegion: signingRegion,
clientRegion: clientRegion,
isCustomEndpoint: options.isCustomEndpoint,
});
export const bucketEndpointMiddleware = (options: BucketEndpointResolvedConfig): BuildMiddleware<any, any> => <
Output extends MetadataBearer
>(
next: BuildHandler<any, Output>,
context: HandlerExecutionContext
): BuildHandler<any, Output> => async (args: BuildHandlerArguments<any>): Promise<BuildHandlerOutput<Output>> => {
const { Bucket: bucketName } = args.input as { Bucket: string };
let replaceBucketInPath = options.bucketEndpoint;
const request = args.request;
if (HttpRequest.isInstance(request)) {
if (options.bucketEndpoint) {
request.hostname = bucketName;
} else if (validateArn(bucketName)) {
const bucketArn = parseArn(bucketName);
const clientRegion = getPseudoRegion(await options.region());
const { partition, signingRegion = clientRegion } = (await options.regionInfoProvider(clientRegion)) || {};
const useArnRegion = await options.useArnRegion();
const { hostname, bucketEndpoint, signingRegion: modifiedSigningRegion, signingService } = bucketHostname({
bucketName: bucketArn,
baseHostname: request.hostname,
accelerateEndpoint: options.useAccelerateEndpoint,
dualstackEndpoint: options.useDualstackEndpoint,
pathStyleEndpoint: options.forcePathStyle,
tlsCompatible: request.protocol === "https:",
useArnRegion,
clientPartition: partition,
clientSigningRegion: signingRegion,
clientRegion: clientRegion,
isCustomEndpoint: options.isCustomEndpoint,
disableMultiregionAccessPoints: await options.disableMultiregionAccessPoints(),
});

// If the request needs to use a region or service name inferred from ARN that different from client region, we
// need to set them in the handler context so the signer will use them
if (modifiedSigningRegion && modifiedSigningRegion !== signingRegion) {
context["signing_region"] = modifiedSigningRegion;
}
if (signingService && signingService !== "s3") {
context["signing_service"] = signingService;
}
// If the request needs to use a region or service name inferred from ARN that different from client region, we
// need to set them in the handler context so the signer will use them
if (modifiedSigningRegion && modifiedSigningRegion !== signingRegion) {
context["signing_region"] = modifiedSigningRegion;
}
if (signingService && signingService !== "s3") {
context["signing_service"] = signingService;
}

request.hostname = hostname;
replaceBucketInPath = bucketEndpoint;
} else {
const clientRegion = getPseudoRegion(await options.region());
const { hostname, bucketEndpoint } = bucketHostname({
bucketName,
clientRegion,
baseHostname: request.hostname,
accelerateEndpoint: options.useAccelerateEndpoint,
dualstackEndpoint: options.useDualstackEndpoint,
pathStyleEndpoint: options.forcePathStyle,
tlsCompatible: request.protocol === "https:",
isCustomEndpoint: options.isCustomEndpoint,
});
request.hostname = hostname;
replaceBucketInPath = bucketEndpoint;
} else {
const clientRegion = getPseudoRegion(await options.region());
const { hostname, bucketEndpoint } = bucketHostname({
bucketName,
clientRegion,
baseHostname: request.hostname,
accelerateEndpoint: options.useAccelerateEndpoint,
dualstackEndpoint: options.useDualstackEndpoint,
pathStyleEndpoint: options.forcePathStyle,
tlsCompatible: request.protocol === "https:",
isCustomEndpoint: options.isCustomEndpoint,
});

request.hostname = hostname;
replaceBucketInPath = bucketEndpoint;
}
request.hostname = hostname;
replaceBucketInPath = bucketEndpoint;
}

if (replaceBucketInPath) {
request.path = request.path.replace(/^(\/)?[^\/]+/, "");
if (request.path === "") {
request.path = "/";
}
if (replaceBucketInPath) {
request.path = request.path.replace(/^(\/)?[^\/]+/, "");
if (request.path === "") {
request.path = "/";
}
}
}

return next({ ...args, request });
};
return next({ ...args, request });
};

export const bucketEndpointMiddlewareOptions: RelativeMiddlewareOptions = {
tags: ["BUCKET_ENDPOINT"],
Expand Down
120 changes: 116 additions & 4 deletions packages/middleware-bucket-endpoint/src/bucketHostname.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -424,10 +424,6 @@ describe("bucketHostname", () => {
bucketArn: "arn:aws:s3:us-west-2:123456789012:bucket_name:mybucket",
message: "ARN resource should begin with 'accesspoint:' or 'outpost:'",
},
{
bucketArn: "arn:aws:s3::123456789012:accesspoint:myendpoint",
message: "ARN region is empty",
},
{
bucketArn: "arn:aws:s3:us-west-2::accesspoint:myendpoint",
message: "Access point ARN accountID does not match regex '[0-9]{12}'",
Expand Down Expand Up @@ -481,6 +477,122 @@ describe("bucketHostname", () => {
});
});

describe("from Multi-region Access Point(MRAP) ARN", () => {
["us-east-1", "us-west-2", "aws-global"].forEach((region) => {
it(`should populate endpoint from MRAP ARN in region "${region}"`, () => {
const { bucketEndpoint, hostname, signingRegion } = bucketHostname({
bucketName: parseArn("arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap"),
baseHostname: `s3.${region}.amazonaws.com`,
disableMultiregionAccessPoints: false,
clientRegion: region,
isCustomEndpoint: false,
});
expect(bucketEndpoint).toBe(true);
expect(hostname).toBe("mfzwi23gnjvgw.mrap.accesspoint.s3-global.amazonaws.com");
expect(signingRegion).toBe("*");
});
});

it('should populate endpoint from MRAP ARN in region "cn-north-2"', () => {
const { bucketEndpoint, hostname, signingRegion } = bucketHostname({
bucketName: parseArn("arn:aws-cn:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap"),
clientPartition: "aws-cn",
baseHostname: `s3.${region}.amazonaws.com.cn`,
disableMultiregionAccessPoints: false,
clientRegion: region,
isCustomEndpoint: false,
});
expect(bucketEndpoint).toBe(true);
expect(hostname).toBe("mfzwi23gnjvgw.mrap.accesspoint.s3-global.amazonaws.com.cn");
expect(signingRegion).toBe("*");
});

it("should throw if MRAP ARN is supplied but disabled through options", () => {
expect(() =>
bucketHostname({
bucketName: parseArn("arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap"),
baseHostname: `s3.us-west-2.amazonaws.com`,
disableMultiregionAccessPoints: true,
clientRegion: region,
isCustomEndpoint: false,
})
).toThrow("SDK is attempting to use a MRAP ARN. Please enable to feature.");
});

it("should throw if dualstack option is set", () => {
expect(() =>
bucketHostname({
bucketName: parseArn("arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap"),
baseHostname: `s3.us-west-2.amazonaws.com`,
dualstackEndpoint: true,
clientRegion: region,
isCustomEndpoint: false,
})
).toThrow("Dualstack endpoint is not supported with Outpost or Multi-region Access Point ARN.");
});

it("should throw if accelerate endpoint option is set", () => {
expect(() =>
bucketHostname({
bucketName: parseArn("arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap"),
baseHostname: `s3.us-west-2.amazonaws.com`,
accelerateEndpoint: true,
clientRegion: region,
isCustomEndpoint: false,
})
).toThrow("Accelerate endpoint is not supported when bucket is an ARN");
});

it("should throw if region is empty and disableMultiregionAccessPoints option is set", () => {
expect(() =>
bucketHostname({
bucketName: parseArn("arn:aws:s3::123456789012:accesspoint:myendpoint"),
baseHostname: `s3.us-west-2.amazonaws.com`,
disableMultiregionAccessPoints: true,
clientRegion: region,
isCustomEndpoint: false,
})
).toThrow("");
});

it('should populate endpoint from MRAP ARN with access point name "myendpoint"', () => {
const { bucketEndpoint, hostname } = bucketHostname({
bucketName: parseArn("arn:aws:s3::123456789012:accesspoint:myendpoint"),
baseHostname: `s3.us-west-2.amazonaws.com`,
disableMultiregionAccessPoints: false,
clientRegion: region,
isCustomEndpoint: false,
});
expect(bucketEndpoint).toBe(true);
expect(hostname).toBe("myendpoint.accesspoint.s3-global.amazonaws.com");
});

it('should populate endpoint from MRAP ARN with access point name "my.bucket"', () => {
const { bucketEndpoint, hostname } = bucketHostname({
bucketName: parseArn("arn:aws:s3::123456789012:accesspoint:my.bucket"),
baseHostname: `s3.us-west-2.amazonaws.com`,
disableMultiregionAccessPoints: false,
clientRegion: region,
isCustomEndpoint: false,
});
expect(bucketEndpoint).toBe(true);
expect(hostname).toBe("my.bucket.accesspoint.s3-global.amazonaws.com");
});

it("should populate endpoint from MRAP ARN with custom endpoint", () => {
const { bucketEndpoint, hostname, signingRegion } = bucketHostname({
bucketName: parseArn("arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap"),
baseHostname: "vpce-123-abc.vpce.s3-global.amazonaws.com",
isCustomEndpoint: true,
clientRegion: "us-west-2",
disableMultiregionAccessPoints: false,
});
expect(bucketEndpoint).toBe(true);
expect(hostname).toBe("mfzwi23gnjvgw.mrap.vpce-123-abc.vpce.s3-global.amazonaws.com");
expect(signingRegion).toBe("*");
});
});

describe("from Outpost ARN", () => {
describe("populates access point endpoint from ARN", () => {
const s3Hostname = "s3.us-west-2.amazonaws.com";
Expand Down
Loading

0 comments on commit 49da47b

Please sign in to comment.