Hackna/s3 backup (#3125)

* Enable versioning for S3 media bucket
* Allow app server role to list object versions in media bucket
* Implement expiration of non-current object versions and delete markers

deployment/terraform/modules/ec2/main.tf

@@ -84,6 +84,11 @@ resource "aws_iam_role_policy" "app_server_s3_access" {
         "arn:aws:s3:::${var.static_bucket_name}",
         "arn:aws:s3:::${var.upload_bucket_name}"
       ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["s3:ListBucketVersions"],
+      "Resource": "arn:aws:s3:::${var.media_bucket_name}"
     }
   ]
 }

deployment/terraform/modules/s3/main.tf

@@ -70,6 +70,21 @@ resource "aws_s3_bucket" "media_files" {
   ]
 }
 EOF
+
+  versioning {
+    enabled = true
+  }
+
+  lifecycle_rule {
+    id      = "Delete non-current object versions and expired delete markers"
+    enabled = true
+    expiration {
+      expired_object_delete_marker = true
+    }
+    noncurrent_version_expiration {
+      days = 14
+    }
+  }
 }
 
 resource "aws_s3_bucket" "log_files" {

refinery/file_store/utils.py

@@ -24,12 +24,20 @@
 
 
 class S3MediaStorage(S3Boto3Storage):
-    """Django media (user data) files storage"""
+    """Django media files (user data) storage"""
     bucket_name = settings.MEDIA_BUCKET
     custom_domain = settings.MEDIA_BUCKET + '.s3.amazonaws.com'
     file_overwrite = False
 
+    def exists(self, name):
+        # returns False only if no object versions or delete markers are
+        # present to prevent overwrites
+        s3 = boto3.client('s3')
+        result = s3.list_object_versions(Bucket=self.bucket_name, Prefix=name)
+        return bool(result.get('Versions') or result.get('DeleteMarkers'))
+
     def get_available_name(self, name, max_length=None):
+        name = self._clean_name(name)
         while True:
             # remove leading '-' characters to make file management easier
             # limit file name length to 255 to make "fully portable" in POSIX