Add options to manage file in subfolder (#23)

We need to be able to upload files to S3 in a "subfolder" (or at least with a prefix attached so it looks like this).

This is so we can adapt to the EPR changes and upload files to EPR/export.csv, rather than export.csv.

We shouldn't create the option to add files without also being able to delete them, so this affects other services as well.

lib/defra_ruby/aws/bucket.rb

@@ -26,16 +26,16 @@ def encryption_type
         @_encryption_type ||= @encrypt_with_kms ? "aws:kms" : :AES256
       end
 
-      def load(file)
-        BucketLoaderService.run(self, file)
+      def load(file, options = {})
+        BucketLoaderService.run(self, file, options)
       end
 
-      def presigned_url(file_name)
-        PresignedUrlService.run(self, file_name)
+      def presigned_url(file_name, options = {})
+        PresignedUrlService.run(self, file_name, options)
       end
 
-      def delete(file_name)
-        DeleteFileFromBucketService.run(self, file_name)
+      def delete(file_name, options = {})
+        DeleteFileFromBucketService.run(self, file_name, options)
       end
 
       private

lib/defra_ruby/aws/services/bucket_loader_service.rb

@@ -5,13 +5,14 @@ module Aws
     class BucketLoaderService
       include HasAwsBucketConfiguration
 
-      def self.run(bucket, file)
-        new(bucket, file).run
+      def self.run(bucket, file, options = {})
+        new(bucket, file, options).run
       end
 
-      def initialize(bucket, file)
+      def initialize(bucket, file, options)
         @bucket = bucket
         @file = file
+        @dir = options[:s3_directory]
       end
 
       def run
@@ -20,15 +21,19 @@ def run
 
       private
 
-      attr_reader :bucket, :file
+      attr_reader :bucket, :file, :dir
 
       def response_exe
         lambda do
           s3_bucket
-            .object(File.basename(file.path))
+            .object(destination)
             .upload_file(file.path, server_side_encryption: bucket.encryption_type)
         end
       end
+
+      def destination
+        [*dir, File.basename(file.path)].compact.join("/")
+      end
     end
   end
 end

lib/defra_ruby/aws/services/delete_file_from_bucket_service.rb

@@ -5,13 +5,14 @@ module Aws
     class DeleteFileFromBucketService
       include HasAwsBucketConfiguration
 
-      def self.run(bucket, file_name)
-        new(bucket, file_name).run
+      def self.run(bucket, file_name, options = {})
+        new(bucket, file_name, options).run
       end
 
-      def initialize(bucket, file_name)
+      def initialize(bucket, file_name, options)
         @bucket = bucket
         @file_name = file_name
+        @dir = options[:s3_directory]
       end
 
       def run
@@ -20,15 +21,19 @@ def run
 
       private
 
-      attr_reader :bucket, :file_name
+      attr_reader :bucket, :file_name, :dir
 
       def response_exe
         lambda do
-          delete_object_output = s3_bucket.object(file_name).delete
+          delete_object_output = s3_bucket.object(destination).delete
 
           delete_object_output.request_charged.length.positive?
         end
       end
+
+      def destination
+        [*dir, file_name].compact.join("/")
+      end
     end
   end
 end

lib/defra_ruby/aws/services/presigned_url_service.rb

@@ -5,17 +5,18 @@ module Aws
     class PresignedUrlService
       include HasAwsBucketConfiguration
 
-      def self.run(bucket, file_name)
-        new(bucket, file_name).run
+      def self.run(bucket, file_name, options = {})
+        new(bucket, file_name, options).run
       end
 
-      def initialize(bucket, file_name)
+      def initialize(bucket, file_name, options)
         @bucket = bucket
         @file_name = file_name
+        @dir = options[:s3_directory]
       end
 
       def run
-        s3_bucket.object(file_name).presigned_url(
+        s3_bucket.object(destination).presigned_url(
           :get,
           expires_in: 20 * 60, # 20 minutes in seconds
           secure: true,
@@ -26,7 +27,11 @@ def run
 
       private
 
-      attr_reader :bucket, :file_name
+      attr_reader :bucket, :file_name, :dir
+
+      def destination
+        [*dir, file_name].compact.join("/")
+      end
     end
   end
 end

spec/lib/defra_ruby/aws/bucket_spec.rb

@@ -230,9 +230,10 @@ module Aws
         it "loads the given file to the s3 bucket" do
           result = double(:result)
           file = double(:file)
+          options = double(:options)
 
-          expect(BucketLoaderService).to receive(:run).with(bucket, file).and_return(result)
-          expect(bucket.load(file)).to eq(result)
+          expect(BucketLoaderService).to receive(:run).with(bucket, file, options).and_return(result)
+          expect(bucket.load(file, options)).to eq(result)
         end
       end
     end

spec/lib/defra_ruby/aws/services/bucket_loader_service_spec.rb

@@ -32,6 +32,24 @@ module Aws
 
             expect(described_class.run(bucket, file)).to be_a(Response)
           end
+
+          context "when an s3_directory is provided" do
+            it "loads the given file to the s3 bucket at the correct location using AWS:KMS" do
+              aws_resource = double(:aws_resource)
+              s3_bucket = double(:s3_bulk_bucket)
+              file = double(:file, path: "foo/bar/baz/test.csv")
+              s3_object = double(:s3_object)
+              result = double(:result)
+              options = { s3_directory: "directory" }
+
+              expect(::Aws::S3::Resource).to receive(:new).and_return(aws_resource)
+              expect(aws_resource).to receive(:bucket).with("bulk").and_return(s3_bucket)
+              expect(s3_bucket).to receive(:object).with("directory/test.csv").and_return(s3_object)
+              expect(s3_object).to receive(:upload_file).with("foo/bar/baz/test.csv", server_side_encryption: :AES256).and_return(result)
+
+              expect(described_class.run(bucket, file, options)).to be_a(Response)
+            end
+          end
         end
 
         context "when 'encrypt_with_kms' is set" do
@@ -57,6 +75,24 @@ module Aws
 
             expect(described_class.run(bucket, file)).to be_a(Response)
           end
+
+          context "when an s3_directory is provided" do
+            it "loads the given file to the s3 bucket at the correct location using AWS:KMS" do
+              aws_resource = double(:aws_resource)
+              s3_bucket = double(:s3_bulk_bucket)
+              file = double(:file, path: "foo/bar/baz/test.csv")
+              s3_object = double(:s3_object)
+              result = double(:result)
+              options = { s3_directory: %w[directory second_directory] }
+
+              expect(::Aws::S3::Resource).to receive(:new).and_return(aws_resource)
+              expect(aws_resource).to receive(:bucket).with("bulk").and_return(s3_bucket)
+              expect(s3_bucket).to receive(:object).with("directory/second_directory/test.csv").and_return(s3_object)
+              expect(s3_object).to receive(:upload_file).with("foo/bar/baz/test.csv", server_side_encryption: "aws:kms").and_return(result)
+
+              expect(described_class.run(bucket, file, options)).to be_a(Response)
+            end
+          end
         end
       end
     end

spec/lib/defra_ruby/aws/services/delete_file_from_bucket_service_spec.rb

@@ -33,6 +33,25 @@ module Aws
           expect(response).to be_successful
         end
 
+        context "when an s3_directory is provided" do
+          it "returns a successful response" do
+            aws_resource = double(:aws_resource)
+            s3_bucket = double(:s3_bulk_bucket)
+            file_name = "test.csv"
+            s3_object = double(:s3_object)
+            result = double(:result, request_charged: "present")
+            options = { s3_directory: "directory" }
+
+            expect(::Aws::S3::Resource).to receive(:new).and_return(aws_resource)
+            expect(aws_resource).to receive(:bucket).with("bulk").and_return(s3_bucket)
+            expect(s3_bucket).to receive(:object).with("directory/test.csv").and_return(s3_object)
+            expect(s3_object).to receive(:delete).and_return(result)
+
+            response = described_class.run(bucket, file_name, options)
+            expect(response).to be_successful
+          end
+        end
+
         context "when the response body returns an empty charged requests" do
           it "returns a non successful response" do
             aws_resource = double(:aws_resource)
@@ -49,6 +68,25 @@ module Aws
             response = described_class.run(bucket, file_name)
             expect(response).to_not be_successful
           end
+
+          context "when an s3_directory is provided" do
+            it "returns a non successful response" do
+              aws_resource = double(:aws_resource)
+              s3_bucket = double(:s3_bulk_bucket)
+              file_name = "test.csv"
+              s3_object = double(:s3_object)
+              result = double(:result, request_charged: "")
+              options = { s3_directory: %w[directory second_directory] }
+
+              expect(::Aws::S3::Resource).to receive(:new).and_return(aws_resource)
+              expect(aws_resource).to receive(:bucket).with("bulk").and_return(s3_bucket)
+              expect(s3_bucket).to receive(:object).with("directory/second_directory/test.csv").and_return(s3_object)
+              expect(s3_object).to receive(:delete).and_return(result)
+
+              response = described_class.run(bucket, file_name, options)
+              expect(response).to_not be_successful
+            end
+          end
         end
       end
     end

spec/lib/defra_ruby/aws/services/presigned_url_service_spec.rb

@@ -26,6 +26,19 @@ module Aws
           expect(presigned_url).to include("Amz-Credential")
           expect(presigned_url).to include("Amz-Signature")
         end
+
+        context "when an s3_directory is provided" do
+          let(:presigned_url) { described_class.run(bucket, "testfile.csv", { s3_directory: "directory" }) }
+
+          it "returns a presigned url for a given file in the bucket" do
+            expect(presigned_url).to include("https://test.s3.eu-west-1.amazonaws.com/directory/testfile.csv")
+            expect(presigned_url).to include("response-content-disposition=attachment")
+            expect(presigned_url).to include("response-content-type=text%2Fcsv")
+            expect(presigned_url).to include("Amz-Expires")
+            expect(presigned_url).to include("Amz-Credential")
+            expect(presigned_url).to include("Amz-Signature")
+          end
+        end
       end
     end
   end