Skip to content

Commit 63cb2da

Browse files
AlexCheemamergify[bot]
authored andcommitted
feat(aws-s3-deployment): support specifying objects metadata (#4288)
* feat(aws-s3-deployment): support specifying objects metadata objects metadata can now be given as part of a deployment which can be useful for example in setting content-type to text/html for static web files * docs(aws-s3-deployment): section in README for objects metadata * chore(aws-s3-deployment): typo in docs * chore(aws-s3-deployment): full stop in docs * feat(aws-s3-deployment): simplify objects metadata api metadata is now defined per deployment, system-defined metadata helpers * chore(aws-s3-deployment): use date for expires system metadata value * chore(aws-s3-deployment): update metadata docs * chore(aws-s3-deployment): use interface instead of types * chore(aws-s3-deployment): add missing dependency fs * chore(aws-s3-deployment): make objects metadata properties readonly * feat(aws-s3-deployment): flatten metadata types, provide access to all available system metadata * chore(aws-s3-deployment): fix builds errors * chore(aws-s3-deployment): public and private are reserved words so use setPublic and setPrivate instead of CacheControl * chore(aws-s3-deployment): add s-max-age to cache-control * chore(aws-s3-deployment): fix style issues * chore(aws-s3-deployment): fix style issues * chore(aws-s3-deployment): fix style issues * chore(aws-s3-deployment): fix whitespace * chore(aws-s3-deployment): better docs for optional metadata properties * chore(aws-s3-deployment): fix whitespace issue * chore(aws-s3-deployment): stricter test for metadata * chore(aws-s3-deployment): import order in test * chore(aws-s3-deployment): import order in test * chore(aws-s3-deployment): update docs, increase test coverage * chore(aws-s3-deployment): add all system-defined metadata keys to README * fix(aws-s3-deployment): handle expires system metadata key * chore(aws-s3-deployment): style fixes * chore(aws-s3-deployment): add keys to metadata test * chore(aws-s3-deployment): docs for metadata classes/enums, change userMetadata to metadata for consistency with aws cli * chore(aws-s3-deployment): re-run integration tests * chore(aws-s3-deployment): fix name of metadata key in test * chore(aws-s3-deployment): shorten comment * chore(aws-s3-deployment): update output from integration tests * chore(aws-s3-deployment): remove trailing whitespace
1 parent be0d2c6 commit 63cb2da

File tree

8 files changed

+352
-37
lines changed

8 files changed

+352
-37
lines changed

packages/@aws-cdk/aws-s3-deployment/README.md

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,48 @@ By default, the contents of the destination bucket will be deleted when the
6565
changed. You can use the option `retainOnDelete: true` to disable this behavior,
6666
in which case the contents will be retained.
6767

68+
## Objects metadata
69+
70+
You can specify metadata to be set on all the objects in your deployment.
71+
There are 2 types of metadata in S3: system-defined metadata and user-defined metadata.
72+
System-defined metadata have a special purpose, for example cache-control defines how long to keep an object cached.
73+
User-defined metadata are not used by S3 and keys always begin with `x-amzn-meta-` (if this is not provided, it is added automatically).
74+
75+
System defined metadata keys include the following:
76+
77+
- cache-control
78+
- content-disposition
79+
- content-encoding
80+
- content-language
81+
- content-type
82+
- expires
83+
- server-side-encryption
84+
- storage-class
85+
- website-redirect-location
86+
- ssekms-key-id
87+
- sse-customer-algorithm
88+
89+
```ts
90+
const websiteBucket = new s3.Bucket(this, 'WebsiteBucket', {
91+
websiteIndexDocument: 'index.html',
92+
publicReadAccess: true
93+
});
94+
95+
new s3deploy.BucketDeployment(this, 'DeployWebsite', {
96+
sources: [s3deploy.Source.asset('./website-dist')],
97+
destinationBucket: websiteBucket,
98+
destinationKeyPrefix: 'web/static', // optional prefix in destination bucket
99+
userMetadata: { "A": "1", "b": "2" }, // user-defined metadata
100+
101+
// system-defined metadata
102+
contentType: "text/html",
103+
contentLanguage: "en",
104+
storageClass: StorageClass.INTELLIGENT_TIERING,
105+
serverSideEncryption: ServerSideEncryption.AES_256,
106+
cacheControl: [CacheControl.setPublic(), CacheControl.maxAge(cdk.Duration.hours(1))],
107+
});
108+
```
109+
68110
## CloudFront Invalidation
69111

70112
You can provide a CloudFront distribution and optional paths to invalidate after the bucket deployment finishes.

packages/@aws-cdk/aws-s3-deployment/lambda/src/index.py

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,12 @@ def cfn_error(message=None):
4141
try:
4242
source_bucket_names = props['SourceBucketNames']
4343
source_object_keys = props['SourceObjectKeys']
44-
dest_bucket_name = props['DestinationBucketName']
45-
dest_bucket_prefix = props.get('DestinationBucketKeyPrefix', '')
46-
retain_on_delete = props.get('RetainOnDelete', "true") == "true"
47-
distribution_id = props.get('DistributionId', '')
44+
dest_bucket_name = props['DestinationBucketName']
45+
dest_bucket_prefix = props.get('DestinationBucketKeyPrefix', '')
46+
retain_on_delete = props.get('RetainOnDelete', "true") == "true"
47+
distribution_id = props.get('DistributionId', '')
48+
user_metadata = props.get('UserMetadata', {})
49+
system_metadata = props.get('SystemMetadata', {})
4850

4951
default_distribution_path = dest_bucket_prefix
5052
if not default_distribution_path.endswith("/"):
@@ -96,7 +98,7 @@ def cfn_error(message=None):
9698
aws_command("s3", "rm", old_s3_dest, "--recursive")
9799

98100
if request_type == "Update" or request_type == "Create":
99-
s3_deploy(s3_source_zips, s3_dest)
101+
s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata)
100102

101103
if distribution_id:
102104
cloudfront_invalidate(distribution_id, distribution_paths)
@@ -110,7 +112,7 @@ def cfn_error(message=None):
110112

111113
#---------------------------------------------------------------------------------------------------
112114
# populate all files from s3_source_zips to a destination bucket
113-
def s3_deploy(s3_source_zips, s3_dest):
115+
def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata):
114116
# create a temporary working directory
115117
workdir=tempfile.mkdtemp()
116118
logger.info("| workdir: %s" % workdir)
@@ -129,7 +131,7 @@ def s3_deploy(s3_source_zips, s3_dest):
129131
zip.extractall(contents_dir)
130132

131133
# sync from "contents" to destination
132-
aws_command("s3", "sync", "--delete", contents_dir, s3_dest)
134+
aws_command("s3", "sync", "--delete", contents_dir, s3_dest, *create_metadata_args(user_metadata, system_metadata))
133135
shutil.rmtree(workdir)
134136

135137
#---------------------------------------------------------------------------------------------------
@@ -149,6 +151,23 @@ def cloudfront_invalidate(distribution_id, distribution_paths):
149151
DistributionId=distribution_id,
150152
Id=invalidation_resp['Invalidation']['Id'])
151153

154+
#---------------------------------------------------------------------------------------------------
155+
# set metadata
156+
def create_metadata_args(raw_user_metadata, raw_system_metadata):
157+
if len(raw_user_metadata) == 0 and len(raw_system_metadata) == 0:
158+
return []
159+
160+
format_system_metadata_key = lambda k: k.lower()
161+
format_user_metadata_key = lambda k: k.lower() if k.lower().startswith("x-amzn-meta-") else f"x-amzn-meta-{k.lower()}"
162+
163+
system_metadata = { format_system_metadata_key(k): v for k, v in raw_system_metadata.items() }
164+
user_metadata = { format_user_metadata_key(k): v for k, v in raw_user_metadata.items() }
165+
166+
system_args = [f"--{k} '{v}'" for k, v in system_metadata.items()]
167+
user_args = ["--metadata", f"'{json.dumps(user_metadata)}'"] if len(user_metadata) > 0 else []
168+
169+
return system_args + user_args + ["--metadata-directive", "REPLACE"]
170+
152171
#---------------------------------------------------------------------------------------------------
153172
# executes an "aws" cli command
154173
def aws_command(*args):

packages/@aws-cdk/aws-s3-deployment/lambda/test/aws

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@ import shutil
1515

1616
scriptdir=os.path.dirname(os.path.realpath(__file__))
1717

18-
# if "cp" is called, copy a test zip file to the destination
19-
if sys.argv[2] == "cp":
18+
# if "cp" is called with a local destination, copy a test zip file to the destination or
19+
if sys.argv[2] == "cp" and not sys.argv[4].startswith("s3://"):
2020
shutil.copyfile(os.path.join(scriptdir, 'test.zip'), sys.argv[4])
2121
sys.argv[4] = "archive.zip"
2222

packages/@aws-cdk/aws-s3-deployment/lambda/test/test.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,21 @@ def test_create_update_with_dest_key(self):
7979
"s3 sync --delete contents.zip s3://<dest-bucket-name>/<dest-key-prefix>"
8080
)
8181

82+
def test_create_update_with_metadata(self):
83+
invoke_handler("Create", {
84+
"SourceBucketNames": ["<source-bucket>"],
85+
"SourceObjectKeys": ["<source-object-key>"],
86+
"DestinationBucketName": "<dest-bucket-name>",
87+
"DestinationBucketKeyPrefix": "<dest-key-prefix>",
88+
"UserMetadata": { "best": "game" },
89+
"SystemMetadata": { "content-type": "text/html", "content-language": "en" }
90+
})
91+
92+
self.assertAwsCommands(
93+
"s3 cp s3://<source-bucket>/<source-object-key> archive.zip",
94+
"s3 sync --delete contents.zip s3://<dest-bucket-name>/<dest-key-prefix> --content-type 'text/html' --content-language 'en' --metadata '{\"x-amzn-meta-best\": \"game\"}' --metadata-directive REPLACE"
95+
)
96+
8297
def test_delete_no_retain(self):
8398
invoke_handler("Delete", {
8499
"SourceBucketNames": ["<source-bucket>"],

packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts

Lines changed: 212 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
1-
import cloudformation = require('@aws-cdk/aws-cloudformation');
2-
import cloudfront = require('@aws-cdk/aws-cloudfront');
3-
import iam = require('@aws-cdk/aws-iam');
4-
import lambda = require('@aws-cdk/aws-lambda');
5-
import s3 = require('@aws-cdk/aws-s3');
6-
import cdk = require('@aws-cdk/core');
7-
import { Token } from '@aws-cdk/core';
1+
import cloudformation = require("@aws-cdk/aws-cloudformation");
2+
import cloudfront = require("@aws-cdk/aws-cloudfront");
3+
import iam = require("@aws-cdk/aws-iam");
4+
import lambda = require("@aws-cdk/aws-lambda");
5+
import s3 = require("@aws-cdk/aws-s3");
6+
import cdk = require("@aws-cdk/core");
7+
import { Token } from "@aws-cdk/core";
88
import crypto = require('crypto');
99
import fs = require('fs');
10-
import path = require('path');
11-
import { ISource, SourceConfig } from './source';
10+
import path = require("path");
11+
import { ISource, SourceConfig } from "./source";
1212

13-
const handlerCodeBundle = path.join(__dirname, '..', 'lambda', 'bundle.zip');
13+
const handlerCodeBundle = path.join(__dirname, "..", "lambda", "bundle.zip");
1414
const handlerSourceDirectory = path.join(__dirname, '..', 'lambda', 'src');
1515

1616
export interface BucketDeploymentProps {
@@ -77,6 +77,80 @@ export interface BucketDeploymentProps {
7777
* @default - A role is automatically created
7878
*/
7979
readonly role?: iam.IRole;
80+
81+
/**
82+
* User-defined object metadata to be set on all objects in the deployment
83+
* @default - No user metadata is set
84+
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#UserMetadata
85+
*/
86+
readonly metadata?: UserDefinedObjectMetadata;
87+
88+
/**
89+
* System-defined cache-control metadata to be set on all objects in the deployment.
90+
* @default - Not set.
91+
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
92+
*/
93+
readonly cacheControl?: CacheControl[];
94+
/**
95+
* System-defined cache-disposition metadata to be set on all objects in the deployment.
96+
* @default - Not set.
97+
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
98+
*/
99+
readonly contentDisposition?: string;
100+
/**
101+
* System-defined content-encoding metadata to be set on all objects in the deployment.
102+
* @default - Not set.
103+
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
104+
*/
105+
readonly contentEncoding?: string;
106+
/**
107+
* System-defined content-language metadata to be set on all objects in the deployment.
108+
* @default - Not set.
109+
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
110+
*/
111+
readonly contentLanguage?: string;
112+
/**
113+
* System-defined content-type metadata to be set on all objects in the deployment.
114+
* @default - Not set.
115+
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
116+
*/
117+
readonly contentType?: string;
118+
/**
119+
* System-defined expires metadata to be set on all objects in the deployment.
120+
* @default - The objects in the distribution will not expire.
121+
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
122+
*/
123+
readonly expires?: Expires;
124+
/**
125+
* System-defined x-amz-server-side-encryption metadata to be set on all objects in the deployment.
126+
* @default - Server side encryption is not used.
127+
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
128+
*/
129+
readonly serverSideEncryption?: ServerSideEncryption;
130+
/**
131+
* System-defined x-amz-storage-class metadata to be set on all objects in the deployment.
132+
* @default - Default storage-class for the bucket is used.
133+
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
134+
*/
135+
readonly storageClass?: StorageClass;
136+
/**
137+
* System-defined x-amz-website-redirect-location metadata to be set on all objects in the deployment.
138+
* @default - No website redirection.
139+
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
140+
*/
141+
readonly websiteRedirectLocation?: string;
142+
/**
143+
* System-defined x-amz-server-side-encryption-aws-kms-key-id metadata to be set on all objects in the deployment.
144+
* @default - Not set.
145+
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
146+
*/
147+
readonly serverSideEncryptionAwsKmsKeyId?: string;
148+
/**
149+
* System-defined x-amz-server-side-encryption-customer-algorithm metadata to be set on all objects in the deployment.
150+
* @default - Not set.
151+
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
152+
*/
153+
readonly serverSideEncryptionCustomerAlgorithm?: string;
80154
}
81155

82156
export class BucketDeployment extends cdk.Construct {
@@ -123,6 +197,8 @@ export class BucketDeployment extends cdk.Construct {
123197
DestinationBucketName: props.destinationBucket.bucketName,
124198
DestinationBucketKeyPrefix: props.destinationKeyPrefix,
125199
RetainOnDelete: props.retainOnDelete,
200+
UserMetadata: props.metadata ? mapUserMetadata(props.metadata) : undefined,
201+
SystemMetadata: mapSystemMetadata(props),
126202
DistributionId: props.distribution ? props.distribution.distributionId : undefined,
127203
DistributionPaths: props.distributionPaths
128204
}
@@ -165,3 +241,129 @@ function calcSourceHash(srcDir: string): string {
165241

166242
return sha.digest('hex');
167243
}
244+
245+
/**
246+
* Metadata
247+
*/
248+
249+
function mapUserMetadata(metadata: UserDefinedObjectMetadata) {
250+
const mapKey = (key: string) =>
251+
key.toLowerCase().startsWith("x-amzn-meta-")
252+
? key.toLowerCase()
253+
: `x-amzn-meta-${key.toLowerCase()}`;
254+
255+
return Object.keys(metadata).reduce((o, key) => ({ ...o, [mapKey(key)]: metadata[key] }), {});
256+
}
257+
258+
function mapSystemMetadata(metadata: BucketDeploymentProps) {
259+
function mapCacheControlDirective(cacheControl: CacheControl) {
260+
const { value } = cacheControl;
261+
262+
if (typeof value === "string") { return value; }
263+
if ("max-age" in value) { return `max-age=${value["max-age"].toSeconds()}`; }
264+
if ("s-max-age" in value) { return `s-max-age=${value["s-max-age"].toSeconds()}`; }
265+
266+
throw new Error(`Unsupported cache-control directive ${value}`);
267+
}
268+
function mapExpires(expires: Expires) {
269+
const { value } = expires;
270+
271+
if (typeof value === "string") { return value; }
272+
if (value instanceof Date) { return value.toUTCString(); }
273+
if (value instanceof cdk.Duration) { return new Date(Date.now() + value.toMilliseconds()).toUTCString(); }
274+
275+
throw new Error(`Unsupported expires ${expires}`);
276+
}
277+
278+
const res: { [key: string]: string } = {};
279+
280+
if (metadata.cacheControl) { res["cache-control"] = metadata.cacheControl.map(mapCacheControlDirective).join(", "); }
281+
if (metadata.expires) { res.expires = mapExpires(metadata.expires); }
282+
if (metadata.contentDisposition) { res["content-disposition"] = metadata.contentDisposition; }
283+
if (metadata.contentEncoding) { res["content-encoding"] = metadata.contentEncoding; }
284+
if (metadata.contentLanguage) { res["content-language"] = metadata.contentLanguage; }
285+
if (metadata.contentType) { res["content-type"] = metadata.contentType; }
286+
if (metadata.serverSideEncryption) { res["server-side-encryption"] = metadata.serverSideEncryption; }
287+
if (metadata.storageClass) { res["storage-class"] = metadata.storageClass; }
288+
if (metadata.websiteRedirectLocation) { res["website-redirect-location"] = metadata.websiteRedirectLocation; }
289+
if (metadata.serverSideEncryptionAwsKmsKeyId) { res["ssekms-key-id"] = metadata.serverSideEncryptionAwsKmsKeyId; }
290+
if (metadata.serverSideEncryptionCustomerAlgorithm) { res["sse-customer-algorithm"] = metadata.serverSideEncryptionCustomerAlgorithm; }
291+
292+
return Object.keys(res).length === 0 ? undefined : res;
293+
}
294+
295+
/**
296+
* Used for HTTP cache-control header, which influences downstream caches.
297+
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
298+
*/
299+
export class CacheControl {
300+
public static mustRevalidate() { return new CacheControl("must-revalidate"); }
301+
public static noCache() { return new CacheControl("no-cache"); }
302+
public static noTransform() { return new CacheControl("no-transform"); }
303+
public static setPublic() { return new CacheControl("public"); }
304+
public static setPrivate() { return new CacheControl("private"); }
305+
public static proxyRevalidate() { return new CacheControl("proxy-revalidate"); }
306+
public static maxAge(t: cdk.Duration) { return new CacheControl({ "max-age": t }); }
307+
public static sMaxAge(t: cdk.Duration) { return new CacheControl({ "s-max-age": t }); }
308+
public static fromString(s: string) { return new CacheControl(s); }
309+
310+
private constructor(public value: any) {}
311+
}
312+
313+
/**
314+
* Indicates whether server-side encryption is enabled for the object, and whether that encryption is
315+
* from the AWS Key Management Service (AWS KMS) or from Amazon S3 managed encryption (SSE-S3).
316+
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
317+
*/
318+
export enum ServerSideEncryption {
319+
AES_256 = 'AES256',
320+
AWS_KMS = 'aws:kms'
321+
}
322+
323+
/**
324+
* Storage class used for storing the object.
325+
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
326+
*/
327+
export enum StorageClass {
328+
STANDARD = 'STANDARD',
329+
REDUCED_REDUNDANCY = 'REDUCED_REDUNDANCY',
330+
STANDARD_IA = 'STANDARD_IA',
331+
ONEZONE_IA = 'ONEZONE_IA',
332+
INTELLIGENT_TIERING = 'INTELLIGENT_TIERING',
333+
GLACIER = 'GLACIER',
334+
DEEP_ARCHIVE = 'DEEP_ARCHIVE'
335+
}
336+
337+
/**
338+
* Used for HTTP expires header, which influences downstream caches. Does NOT influence deletion of the object.
339+
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
340+
*/
341+
export class Expires {
342+
/**
343+
* Expire at the specified date
344+
* @param d date to expire at
345+
*/
346+
public static atDate(d: Date) { return new Expires(d); }
347+
/**
348+
* Expire at the specified timestamp
349+
* @param t timestamp in unix milliseconds
350+
*/
351+
public static atTimestamp(t: number) { return new Expires(t); }
352+
/**
353+
* Expire once the specified duration has passed since deployment time
354+
* @param t the duration to wait before expiring
355+
*/
356+
public static after(t: cdk.Duration) { return new Expires(t); }
357+
public static fromString(s: string) { return new Expires(s); }
358+
359+
private constructor(public value: any) {}
360+
}
361+
362+
export interface UserDefinedObjectMetadata {
363+
/**
364+
* Arbitrary metadata key-values
365+
* Keys must begin with `x-amzn-meta-` (will be added automatically if not provided)
366+
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#UserMetadata
367+
*/
368+
readonly [key: string]: string;
369+
}

0 commit comments

Comments
 (0)