extend backup lifecycle policies to support frequency tag

splunk · Dec 13, 2023 · cc38cc5 · cc38cc5
1 parent 0d01765
commit cc38cc5
Show file tree

Hide file tree

Showing 4 changed files with 159 additions and 7 deletions.
diff --git a/terraform/provision.tf b/terraform/provision.tf
@@ -49,7 +49,7 @@ resource "local_file" "ansible_vars_tf" {
     smartstore_site_number: ${var.splunksmartstoresitenumber}
     dns_zone_name: ${var.dns-zone-name}
     splunk_ssh_key_arn: ${module.ssh.splunk_ssh_key_arn}
-    backupretention: ${var.backup-retention}
+    backupretention: ${var.backup-retention-days}
     deleteddataretention: ${var.deleteddata-retention}
   tasks:
   - name: create directories for target jinja

diff --git a/terraform/s3buckets-secondary.tf b/terraform/s3buckets-secondary.tf
@@ -126,7 +126,7 @@ resource "aws_s3_bucket_lifecycle_configuration" "s3_backup_secondary_lifecycle"
       prefix = "splunkconf-backup/"
     }
     noncurrent_version_expiration {
-      noncurrent_days = var.backup-retention
+      noncurrent_days = var.backup-retention-days
     }
     abort_incomplete_multipart_upload {
       days_after_initiation = 1

diff --git a/terraform/s3buckets.tf b/terraform/s3buckets.tf
@@ -102,12 +102,14 @@ resource "aws_s3_bucket_lifecycle_configuration" "s3_backup_lifecycle" {
   bucket   = aws_s3_bucket.s3_backup.id
 
   rule {
-    id = "purge-old-noncurrent-versionned-backup"
+    id = "purge-old-noncurrent-versionned-backup-global"
+    # this will catch up all objects so we purge those without tags but it will be added to the other by tag so make sure this is consistent
     filter {
       prefix = "splunkconf-backup/"
     }
     noncurrent_version_expiration {
-      noncurrent_days = var.backup-retention
+      newer_noncurrent_versions = var.backup-min-versions 
+      noncurrent_days = var.backup-retention-days
     }
     abort_incomplete_multipart_upload {
       days_after_initiation = 1
@@ -117,6 +119,64 @@ resource "aws_s3_bucket_lifecycle_configuration" "s3_backup_lifecycle" {
     }
     status = "Enabled"
   }
+  rule {
+    id = "purge-hourly-backup-bytag"
+    filter {
+      and {
+        prefix = "splunkconf-backup/"
+        tags = { frequency = "hourly" }
+      }
+    }
+    noncurrent_version_expiration {
+      newer_noncurrent_versions = var.backup-min-versions-hourly 
+      noncurrent_days = var.backup-retention-days-hourly
+    }
+    status = "Enabled"
+  }
+  rule {
+    id = "purge-daily-backup-bytag"
+    filter {
+      and {
+        prefix = "splunkconf-backup/"
+        tags = { frequency = "daily" }
+      }
+    }
+    noncurrent_version_expiration {
+      newer_noncurrent_versions = var.backup-min-versions-daily 
+      noncurrent_days = var.backup-retention-days-daily
+    }
+    status = "Enabled"
+  }
+  rule {
+    id = "purge-weekly-backup-bytag"
+    filter {
+      and {
+        prefix = "splunkconf-backup/"
+        tags = { frequency = "weekly" }
+      }
+    }
+    noncurrent_version_expiration {
+      newer_noncurrent_versions = var.backup-min-versions-weekly 
+      noncurrent_days = var.backup-retention-days-weekly
+    }
+    status = "Enabled"
+  }
+  rule {
+    id = "purge-monthly-backup-bytag"
+    filter {
+      and {
+        prefix = "splunkconf-backup/"
+        tags = { frequency = "monthly" }
+      }
+    }
+    noncurrent_version_expiration {
+      newer_noncurrent_versions = var.backup-min-versions-monthly 
+      noncurrent_days = var.backup-retention-days-monthly
+    }
+    status = "Enabled"
+  }
+
+
 }
 
 resource "aws_s3_bucket" "s3_data" {

diff --git a/terraform/variables.tf b/terraform/variables.tf
@@ -432,13 +432,105 @@ variable "associate_public_ip" {
   #default = "false"
 }
 
-variable "backup-retention" {
+variable "backup-retention-days" {
   description= "Number of days before removing old backups from S3 (only for versions)"
   type    = number
   default = 31
   validation {
-    condition     = var.backup-retention >= 31
-    error_message = "backup-retention need to be at least 31 days (to match S3-IA pricing structure)"
+    condition     = var.backup-retention-days >= 31
+    error_message = "backup-retention-days need to be at least 31 days (to match S3-IA pricing structure)"
+  }
+}
+
+variable backup-min-versions {
+  description = "Minimum number of non current versions backup to keep"
+  type = number
+  default = 10
+  validation {    
+    condition     = var.backup-min-versions >= 1
+    error_message = "backup-min-versions need to be at least 1"
+  }
+}
+
+variable "backup-retention-days-hourly" {
+  description= "Number of days before removing old backups from S3 tagged with frequency=hourly (only for versions)"
+  type    = number
+  default = 1
+  validation {    
+    condition     = var.backup-retention-days-hourly >= 1
+    #condition     = var.backup-retention-days-hourly >= 31
+    error_message = "backup-retention-days-hourly need to be at least 1 days (to match S3-IA pricing structure)"
+    #error_message = "backup-retention-days-hourly need to be at least 31 days (to match S3-IA pricing structure)"
+  }
+}
+
+variable backup-min-versions-hourly {
+  description = "Minimum number of non current versions backup tagged with frequency=hourly to keep"
+  type = number
+  default = 10
+  validation {    
+    condition     = var.backup-min-versions-hourly >= 1
+    error_message = "backup-min-versions-hourly need to be at least 1"
+  }
+}
+
+variable "backup-retention-days-daily" {
+  description= "Number of days before removing old backups from S3 tagged with frequency=daily (only for versions)"
+  type    = number
+  default = 60
+  validation {    
+    condition     = var.backup-retention-days-daily >= 31
+    error_message = "backup-retention-days-daily need to be at least 31 days (to match S3-IA pricing structure)"
+  }
+}
+
+variable backup-min-versions-daily {
+  description = "Minimum number of non current versions backup tagged with frequency=daily to keep"
+  type = number
+  default = 10
+  validation {    
+    condition     = var.backup-min-versions-daily >= 1
+    error_message = "backup-min-versions-daily need to be at least 1"
+  }
+}
+
+variable "backup-retention-days-weekly" {
+  description= "Number of days before removing old backups from S3 tagged with frequency=weekly (only for versions)"
+  type    = number
+  default = 120 
+  validation {
+    condition     = var.backup-retention-days-weekly >= 31
+    error_message = "backup-retention-days-weekly need to be at least 31 days (to match S3-IA pricing structure)"
+  }
+}
+
+variable backup-min-versions-weekly {
+  description = "Minimum number of non current versions backup tagged with frequency=weekly to keep"
+  type = number
+  default = 10
+  validation {    
+    condition     = var.backup-min-versions-weekly >= 1
+    error_message = "backup-min-versions-weekly need to be at least 1"
+  }
+}
+
+variable "backup-retention-days-monthly" {
+  description= "Number of days before removing old backups from S3 tagged with frequency=monthly (only for versions)"
+  type    = number
+  default = 365 
+  validation {
+    condition     = var.backup-retention-days-monthly >= 31
+    error_message = "backup-retention-days-monthly need to be at least 31 days (to match S3-IA pricing structure)"
+  }
+}
+
+variable backup-min-versions-monthly {
+  description = "Minimum number of non current versions backup tagged with frequency=monthly to keep"
+  type = number
+  default = 10
+  validation {    
+    condition     = var.backup-min-versions-monthly >= 1
+    error_message = "backup-min-versions-monthly need to be at least 1"
   }
 }