Skip to content

Commit

Permalink
feat(s3): date-based partitioning for log objects (#28790)
Browse files Browse the repository at this point in the history
S3 server access logging newly supports date-based partitioning.
This feature allows key formats of log objects to be partitioned by date, which is useful when querying from Athena.

In CloudFormation, `TargetObjectKeyFormat` was added.
This object has the `PartitionedPrefix` and the `SimplePrefix` property, only one of which is allowed.
The `SimplePrefix` is the key format that originally existed.
The `PartitionedPrefix` is the new key format and allows the user to select either the event time or delivery time as the date source.

```yaml
Resources:
  S3BucketServerAccessLogEnabledTest:
    Type: AWS::S3::Bucket
    Properties:
      BucketName: access-log-test
      LoggingConfiguration:
        DestinationBucketName: myDestinationBucket
        LogFilePrefix: 'hello'
        TargetObjectKeyFormat:
          # You can deploy in either of the following formats
          SimplePrefix: {} # 1
          PartitionedPrefix: {} # 2
          PartitionedPrefix: # 3
            PartitionDateSource: EventTime # | DeliveryTime
```

whats-new
https://aws.amazon.com/about-aws/whats-new/2023/11/amazon-s3-server-access-logging-date-partitioning

CloudFormation User Guides
https://docs.aws.amazon.com/ja_jp/AWSCloudFormation/latest/UserGuide/aws-properties-s3-bucket-loggingconfiguration.html#cfn-s3-bucket-loggingconfiguration-targetobjectkeyformat

S3 User Guides
https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerLogs.html#server-access-logging-overview

Closes #28141

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
  • Loading branch information
sakurai-ryo committed Jan 25, 2024
1 parent 7a30f5d commit 2952408
Show file tree
Hide file tree
Showing 11 changed files with 746 additions and 26 deletions.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,150 @@
]
]
}
},
{
"Action": "s3:PutObject",
"Condition": {
"ArnLike": {
"aws:SourceArn": {
"Fn::GetAtt": [
"MyBucket26E0C3623",
"Arn"
]
}
},
"StringEquals": {
"aws:SourceAccount": {
"Ref": "AWS::AccountId"
}
}
},
"Effect": "Allow",
"Principal": {
"Service": "logging.s3.amazonaws.com"
},
"Resource": {
"Fn::Join": [
"",
[
{
"Fn::GetAtt": [
"MyAccessLogsBucketF7FE6635",
"Arn"
]
},
"/example2*"
]
]
}
},
{
"Action": "s3:PutObject",
"Condition": {
"ArnLike": {
"aws:SourceArn": {
"Fn::GetAtt": [
"MyBucket3CC4F8735",
"Arn"
]
}
},
"StringEquals": {
"aws:SourceAccount": {
"Ref": "AWS::AccountId"
}
}
},
"Effect": "Allow",
"Principal": {
"Service": "logging.s3.amazonaws.com"
},
"Resource": {
"Fn::Join": [
"",
[
{
"Fn::GetAtt": [
"MyAccessLogsBucketF7FE6635",
"Arn"
]
},
"/example3*"
]
]
}
},
{
"Action": "s3:PutObject",
"Condition": {
"ArnLike": {
"aws:SourceArn": {
"Fn::GetAtt": [
"MyBucket43E0A113B",
"Arn"
]
}
},
"StringEquals": {
"aws:SourceAccount": {
"Ref": "AWS::AccountId"
}
}
},
"Effect": "Allow",
"Principal": {
"Service": "logging.s3.amazonaws.com"
},
"Resource": {
"Fn::Join": [
"",
[
{
"Fn::GetAtt": [
"MyAccessLogsBucketF7FE6635",
"Arn"
]
},
"/example4*"
]
]
}
},
{
"Action": "s3:PutObject",
"Condition": {
"ArnLike": {
"aws:SourceArn": {
"Fn::GetAtt": [
"MyBucket53983D51A",
"Arn"
]
}
},
"StringEquals": {
"aws:SourceAccount": {
"Ref": "AWS::AccountId"
}
}
},
"Effect": "Allow",
"Principal": {
"Service": "logging.s3.amazonaws.com"
},
"Resource": {
"Fn::Join": [
"",
[
{
"Fn::GetAtt": [
"MyAccessLogsBucketF7FE6635",
"Arn"
]
},
"/example5*"
]
]
}
}
],
"Version": "2012-10-17"
Expand All @@ -66,6 +210,74 @@
},
"UpdateReplacePolicy": "Delete",
"DeletionPolicy": "Delete"
},
"MyBucket26E0C3623": {
"Type": "AWS::S3::Bucket",
"Properties": {
"LoggingConfiguration": {
"DestinationBucketName": {
"Ref": "MyAccessLogsBucketF7FE6635"
},
"LogFilePrefix": "example2",
"TargetObjectKeyFormat": {
"SimplePrefix": {}
}
}
},
"UpdateReplacePolicy": "Delete",
"DeletionPolicy": "Delete"
},
"MyBucket3CC4F8735": {
"Type": "AWS::S3::Bucket",
"Properties": {
"LoggingConfiguration": {
"DestinationBucketName": {
"Ref": "MyAccessLogsBucketF7FE6635"
},
"LogFilePrefix": "example3",
"TargetObjectKeyFormat": {
"PartitionedPrefix": {
"PartitionDateSource": "EventTime"
}
}
}
},
"UpdateReplacePolicy": "Delete",
"DeletionPolicy": "Delete"
},
"MyBucket43E0A113B": {
"Type": "AWS::S3::Bucket",
"Properties": {
"LoggingConfiguration": {
"DestinationBucketName": {
"Ref": "MyAccessLogsBucketF7FE6635"
},
"LogFilePrefix": "example4",
"TargetObjectKeyFormat": {
"PartitionedPrefix": {
"PartitionDateSource": "DeliveryTime"
}
}
}
},
"UpdateReplacePolicy": "Delete",
"DeletionPolicy": "Delete"
},
"MyBucket53983D51A": {
"Type": "AWS::S3::Bucket",
"Properties": {
"LoggingConfiguration": {
"DestinationBucketName": {
"Ref": "MyAccessLogsBucketF7FE6635"
},
"LogFilePrefix": "example5",
"TargetObjectKeyFormat": {
"PartitionedPrefix": {}
}
}
},
"UpdateReplacePolicy": "Delete",
"DeletionPolicy": "Delete"
}
},
"Parameters": {
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 2952408

Please sign in to comment.