Add configuration options for AWS S3 server access logging

docs/_docs/04_reference/config-blocks-and-attributes.md

@@ -398,7 +398,12 @@ For the `s3` backend, the following additional properties are supported in the `
   such as the CRC32 check for DynamoDB. This can be used to workaround
   https://github.com/gruntwork-io/terragrunt/issues/1059.
 - `accesslogging_bucket_name`: (Optional) When provided as a valid `string`, create an S3 bucket with this name to store the access logs for the S3 bucket used to store Terraform state. If not provided, or string is empty or invalid S3 bucket name, then server access logging for the S3 bucket storing the terraform state will be disabled. **Note:** When access logging is enabled supported encryption for state bucket is only `AES256`. Reference: [S3 server access logging](https://docs.aws.amazon.com/AmazonS3/latest/userguide/enable-server-access-logging.html)
+- `accesslogging_target_object_partition_date_source`: (Optional) When provided as a valid `string`, it configures the `PartitionDateSource` option. This option is part of the `TargetObjectKeyFormat` and `PartitionedPrefix` AWS configurations, allowing you to configure the log object key format for the access log files.
 - `accesslogging_target_prefix`: (Optional) When provided as a valid `string`, set the `TargetPrefix` for the access log objects in the S3 bucket used to store Terraform state. If set to **empty**`string`, then `TargetPrefix` will be set to **empty** `string`. If attribute is not provided at all, then `TargetPrefix` will be set to **default** value `TFStateLogs/`. This attribute won't take effect if the `accesslogging_bucket_name` attribute is not present.
+- `skip_accesslogging_bucket_acl`: When set to `true`, the S3 bucket where access logs are stored will not be configured with bucket ACL.
+- `skip_accesslogging_bucket_enforced_tls`: When set to `true`, the S3 bucket where access logs are stored will not be configured with a bucket policy that enforces access to the bucket via a TLS connection.
+- `skip_accesslogging_bucket_public_access_blocking`: When set to `true`, the S3 bucket where access logs are stored will not have public access blocking enabled.
+- `skip_accesslogging_bucket_ssencryption`: When set to `true`, the S3 bucket where access logs are stored will not be configured with server-side encryption.
 - `bucket_sse_algorithm`: (Optional) The algorithm to use for server side encryption of the state bucket. Defaults to `aws:kms`.
 - `bucket_sse_kms_key_id`: (Optional) The KMS Key to use when the encryption algorithm is `aws:kms`. Defaults to the AWS Managed `aws/s3` key.
 - `assume_role`: (Optional) A configuration `map` to use when assuming a role (starting with Terraform 1.6). Override top level arguments

remote/remote_state_s3.go

@@ -39,23 +39,28 @@ const (
 type ExtendedRemoteStateConfigS3 struct {
 	remoteStateConfigS3 RemoteStateConfigS3
 
-	S3BucketTags                   map[string]string `mapstructure:"s3_bucket_tags"`
-	DynamotableTags                map[string]string `mapstructure:"dynamodb_table_tags"`
-	AccessLoggingBucketTags        map[string]string `mapstructure:"accesslogging_bucket_tags"`
-	SkipCredentialsValidation      bool              `mapstructure:"skip_credentials_validation"`
-	SkipBucketVersioning           bool              `mapstructure:"skip_bucket_versioning"`
-	SkipBucketSSEncryption         bool              `mapstructure:"skip_bucket_ssencryption"`
-	SkipBucketAccessLogging        bool              `mapstructure:"skip_bucket_accesslogging"`
-	SkipBucketRootAccess           bool              `mapstructure:"skip_bucket_root_access"`
-	SkipBucketEnforcedTLS          bool              `mapstructure:"skip_bucket_enforced_tls"`
-	SkipBucketPublicAccessBlocking bool              `mapstructure:"skip_bucket_public_access_blocking"`
-	DisableBucketUpdate            bool              `mapstructure:"disable_bucket_update"`
-	EnableLockTableSSEncryption    bool              `mapstructure:"enable_lock_table_ssencryption"`
-	DisableAWSClientChecksums      bool              `mapstructure:"disable_aws_client_checksums"`
-	AccessLoggingBucketName        string            `mapstructure:"accesslogging_bucket_name"`
-	AccessLoggingTargetPrefix      string            `mapstructure:"accesslogging_target_prefix"`
-	BucketSSEAlgorithm             string            `mapstructure:"bucket_sse_algorithm"`
-	BucketSSEKMSKeyID              string            `mapstructure:"bucket_sse_kms_key_id"`
+	S3BucketTags                                 map[string]string `mapstructure:"s3_bucket_tags"`
+	DynamotableTags                              map[string]string `mapstructure:"dynamodb_table_tags"`
+	AccessLoggingBucketTags                      map[string]string `mapstructure:"accesslogging_bucket_tags"`
+	SkipCredentialsValidation                    bool              `mapstructure:"skip_credentials_validation"`
+	SkipBucketVersioning                         bool              `mapstructure:"skip_bucket_versioning"`
+	SkipBucketSSEncryption                       bool              `mapstructure:"skip_bucket_ssencryption"`
+	SkipBucketAccessLogging                      bool              `mapstructure:"skip_bucket_accesslogging"`
+	SkipBucketRootAccess                         bool              `mapstructure:"skip_bucket_root_access"`
+	SkipBucketEnforcedTLS                        bool              `mapstructure:"skip_bucket_enforced_tls"`
+	SkipBucketPublicAccessBlocking               bool              `mapstructure:"skip_bucket_public_access_blocking"`
+	DisableBucketUpdate                          bool              `mapstructure:"disable_bucket_update"`
+	EnableLockTableSSEncryption                  bool              `mapstructure:"enable_lock_table_ssencryption"`
+	DisableAWSClientChecksums                    bool              `mapstructure:"disable_aws_client_checksums"`
+	AccessLoggingBucketName                      string            `mapstructure:"accesslogging_bucket_name"`
+	AccessLoggingTargetObjectPartitionDateSource string            `mapstructure:"accesslogging_target_object_partition_date_source"`
+	AccessLoggingTargetPrefix                    string            `mapstructure:"accesslogging_target_prefix"`
+	SkipAccessLoggingBucketAcl                   bool              `mapstructure:"skip_accesslogging_bucket_acl"`
+	SkipAccessLoggingBucketEnforcedTLS           bool              `mapstructure:"skip_accesslogging_bucket_enforced_tls"`
+	SkipAccessLoggingBucketPublicAccessBlocking  bool              `mapstructure:"skip_accesslogging_bucket_public_access_blocking"`
+	SkipAccessLoggingBucketSSEncryption          bool              `mapstructure:"skip_accesslogging_bucket_ssencryption"`
+	BucketSSEAlgorithm                           string            `mapstructure:"bucket_sse_algorithm"`
+	BucketSSEKMSKeyID                            string            `mapstructure:"bucket_sse_kms_key_id"`
 }
 
 // These are settings that can appear in the remote_state config that are ONLY used by Terragrunt and NOT forwarded
@@ -74,7 +79,12 @@ var terragruntOnlyConfigs = []string{
 	"enable_lock_table_ssencryption",
 	"disable_aws_client_checksums",
 	"accesslogging_bucket_name",
+	"accesslogging_target_object_partition_date_source",
 	"accesslogging_target_prefix",
+	"skip_accesslogging_bucket_acl",
+	"skip_accesslogging_bucket_enforced_tls",
+	"skip_accesslogging_bucket_public_access_blocking",
+	"skip_accesslogging_bucket_ssencryption",
 	"bucket_sse_algorithm",
 	"bucket_sse_kms_key_id",
 }
@@ -120,6 +130,29 @@ func (c *ExtendedRemoteStateConfigS3) GetAwsSessionConfig() *aws_helper.AwsSessi
 	}
 }
 
+// Builds AWS S3 logging input struct from the configuration.
+func (c *ExtendedRemoteStateConfigS3) createS3LoggingInput() s3.PutBucketLoggingInput {
+	loggingInput := s3.PutBucketLoggingInput{
+		Bucket: aws.String(c.remoteStateConfigS3.Bucket),
+		BucketLoggingStatus: &s3.BucketLoggingStatus{
+			LoggingEnabled: &s3.LoggingEnabled{
+				TargetBucket: aws.String(c.AccessLoggingBucketName),
+				TargetPrefix: aws.String(c.AccessLoggingTargetPrefix),
+			},
+		},
+	}
+
+	if c.AccessLoggingTargetObjectPartitionDateSource != "" {
+		loggingInput.BucketLoggingStatus.LoggingEnabled.TargetObjectKeyFormat = &s3.TargetObjectKeyFormat{
+			PartitionedPrefix: &s3.PartitionedPrefix{
+				PartitionDateSource: aws.String(c.AccessLoggingTargetObjectPartitionDateSource),
+			},
+		}
+	}
+
+	return loggingInput
+}
+
 // The DynamoDB lock table attribute used to be called "lock_table", but has since been renamed to "dynamodb_table", and
 // the old attribute name deprecated. The old attribute name has been eventually removed from Terraform starting with
 // release 0.13. To maintain backwards compatibility, we support both names.
@@ -557,22 +590,30 @@ func configureAccessLogBucket(terragruntOptions *options.TerragruntOptions, s3Cl
 		return errors.WithStackTrace(err)
 	}
 
-	if err := EnablePublicAccessBlockingForS3Bucket(s3Client, config.AccessLoggingBucketName, terragruntOptions); err != nil {
-		return errors.WithStackTrace(err)
+	if !config.SkipAccessLoggingBucketPublicAccessBlocking {
+		if err := EnablePublicAccessBlockingForS3Bucket(s3Client, config.AccessLoggingBucketName, terragruntOptions); err != nil {
+			terragruntOptions.Logger.Errorf("Could not enable public access blocking on %s\n%s", config.AccessLoggingBucketName, err.Error())
+			return errors.WithStackTrace(err)
+		}
 	}
 
-	if err := EnableAccessLoggingForS3BucketWide(s3Client, &config.remoteStateConfigS3, terragruntOptions, config.AccessLoggingBucketName, config.AccessLoggingTargetPrefix); err != nil {
+	if err := EnableAccessLoggingForS3BucketWide(s3Client, config, terragruntOptions); err != nil {
+		terragruntOptions.Logger.Errorf("Could not enable access logging on %s\n%s", config.remoteStateConfigS3.Bucket, err.Error())
 		return errors.WithStackTrace(err)
 	}
 
-	if !config.SkipBucketSSEncryption {
+	if !config.SkipAccessLoggingBucketSSEncryption {
 		if err := EnableSSEForS3BucketWide(s3Client, config.AccessLoggingBucketName, s3.ServerSideEncryptionAes256, config, terragruntOptions); err != nil {
+			terragruntOptions.Logger.Errorf("Could not enable encryption on %s\n%s", config.AccessLoggingBucketName, err.Error())
 			return errors.WithStackTrace(err)
 		}
 	}
 
-	if err := EnableEnforcedTLSAccesstoS3Bucket(s3Client, config.AccessLoggingBucketName, config, terragruntOptions); err != nil {
-		return errors.WithStackTrace(err)
+	if !config.SkipAccessLoggingBucketEnforcedTLS {
+		if err := EnableEnforcedTLSAccesstoS3Bucket(s3Client, config.AccessLoggingBucketName, config, terragruntOptions); err != nil {
+			terragruntOptions.Logger.Errorf("Could not enable TLS access on %s\n%s", config.AccessLoggingBucketName, err.Error())
+			return errors.WithStackTrace(err)
+		}
 	}
 	return nil
 }
@@ -639,7 +680,7 @@ func checkIfS3BucketNeedsUpdate(s3Client *s3.S3, config *ExtendedRemoteStateConf
 	}
 
 	if !config.SkipBucketAccessLogging && config.AccessLoggingBucketName != "" {
-		enabled, err := checkIfAccessLoggingForS3Enabled(s3Client, &config.remoteStateConfigS3, terragruntOptions)
+		enabled, err := checkS3AccessLoggingConfiguration(s3Client, config, terragruntOptions)
 		if err != nil {
 			return false, configBucket, err
 		}
@@ -1228,45 +1269,48 @@ func checkIfSSEForS3Enabled(s3Client *s3.S3, config *ExtendedRemoteStateConfigS3
 }
 
 // Enable bucket-wide Access Logging for the AWS S3 bucket specified in the given config
-func EnableAccessLoggingForS3BucketWide(s3Client *s3.S3, config *RemoteStateConfigS3, terragruntOptions *options.TerragruntOptions, logsBucket string, logsBucketPrefix string) error {
-	if err := configureBucketAccessLoggingAcl(s3Client, aws.String(logsBucket), terragruntOptions); err != nil {
-		return errors.WithStackTrace(err)
-	}
-
-	terragruntOptions.Logger.Debugf("Putting bucket logging on S3 bucket %s with TargetBucket %s and TargetPrefix %s", config.Bucket, logsBucket, logsBucketPrefix)
+func EnableAccessLoggingForS3BucketWide(s3Client *s3.S3, config *ExtendedRemoteStateConfigS3, terragruntOptions *options.TerragruntOptions) error {
+	bucket := config.remoteStateConfigS3.Bucket
+	logsBucket := config.AccessLoggingBucketName
+	logsBucketPrefix := config.AccessLoggingTargetPrefix
 
-	loggingInput := s3.PutBucketLoggingInput{
-		Bucket: aws.String(config.Bucket),
-		BucketLoggingStatus: &s3.BucketLoggingStatus{
-			LoggingEnabled: &s3.LoggingEnabled{
-				TargetBucket: aws.String(logsBucket),
-				TargetPrefix: aws.String(logsBucketPrefix),
-			},
-		},
+	if !config.SkipAccessLoggingBucketAcl {
+		if err := configureBucketAccessLoggingAcl(s3Client, aws.String(logsBucket), terragruntOptions); err != nil {
+			return errors.WithStackTrace(err)
+		}
 	}
 
+	loggingInput := config.createS3LoggingInput()
+	terragruntOptions.Logger.Debugf("Putting bucket logging on S3 bucket %s with TargetBucket %s and TargetPrefix %s\n%s", bucket, logsBucket, logsBucketPrefix, loggingInput)
+
 	if _, err := s3Client.PutBucketLogging(&loggingInput); err != nil {
 		return errors.WithStackTrace(err)
 	}
 
-	terragruntOptions.Logger.Debugf("Enabled bucket-wide Access Logging on AWS S3 bucket %s", config.Bucket)
+	terragruntOptions.Logger.Debugf("Enabled bucket-wide Access Logging on AWS S3 bucket %s", bucket)
 	return nil
 }
 
-func checkIfAccessLoggingForS3Enabled(s3Client *s3.S3, config *RemoteStateConfigS3, terragruntOptions *options.TerragruntOptions) (bool, error) {
-	terragruntOptions.Logger.Debugf("Checking if Access Logging is enabled for AWS S3 bucket %s", config.Bucket)
+func checkS3AccessLoggingConfiguration(s3Client *s3.S3, config *ExtendedRemoteStateConfigS3, terragruntOptions *options.TerragruntOptions) (bool, error) {
+	terragruntOptions.Logger.Debugf("Checking if Access Logging is enabled for AWS S3 bucket %s", config.remoteStateConfigS3.Bucket)
 
-	input := &s3.GetBucketLoggingInput{Bucket: aws.String(config.Bucket)}
+	input := &s3.GetBucketLoggingInput{Bucket: aws.String(config.remoteStateConfigS3.Bucket)}
 	output, err := s3Client.GetBucketLogging(input)
 	if err != nil {
-		terragruntOptions.Logger.Debugf("Error checking if Access Logging is enabled for AWS S3 bucket %s: %s", config.Bucket, err.Error())
+		terragruntOptions.Logger.Debugf("Error checking if Access Logging is enabled for AWS S3 bucket %s: %s", config.remoteStateConfigS3.Bucket, err.Error())
 		return false, err
 	}
 
 	if output.LoggingEnabled == nil {
 		return false, nil
 	}
 
+	loggingInput := config.createS3LoggingInput()
+
+	if !reflect.DeepEqual(output.LoggingEnabled, loggingInput.BucketLoggingStatus.LoggingEnabled) {
+		return false, nil
+	}
+
 	return true, nil
 }