Feature/archive files are created at root specified in bucket settings/#7/#8

ditto_web_api/DittoWebApi/example_bucket_settings.ini

@@ -1,7 +1,9 @@
 [ditto-example1]
 groups = group1
-root = /usr/tmp/data1
+data_root = /usr/tmp/data1
+archive_root = /usr/tmp/archive1
 
 [ditto-example2]
 groups = group1, group2
-root = /usr/tmp/data2
+archive_root = /usr/tmp/data2
+archive_root = /usr/tmp/archive2

ditto_web_api/DittoWebApi/src/handlers/create_bucket.py

@@ -13,7 +13,8 @@ class CreateBucketHandler(DittoHandler):
             "properties": {
                 "bucket": "test-bucket-name",
                 "groups": ["group1", "group2"],
-                "root": "/path/to/root"
+                "archive_root": "/path/to/archive_root",
+                "data_root": "/path/to/data_root"
             },
         },
         output_schema=create_object_schema_with_string_properties(["message", "bucket"]),
@@ -27,8 +28,10 @@ def post(self, *args, **kwargs):
         self.check_current_user_is_admin()
         bucket_name = self.get_body_attribute("bucket", required=True)
         groups = self.get_body_attribute("groups", required=True, value_type=list)
-        root = self.get_body_attribute("root", required=True)
-        result = self._data_replication_service.create_bucket(bucket_name, groups, root)
+        data_root = self.get_body_attribute("data_root", required=True)
+        archive_root = self.get_body_attribute("archive_root", required=False)
+        archive_root = data_root if archive_root is None else archive_root
+        result = self._data_replication_service.create_bucket(bucket_name, groups, archive_root, data_root)
         self.check_request_was_completed_successfully(result)
         del result['status']
         return result

ditto_web_api/DittoWebApi/src/handlers/ditto_handler.py

@@ -83,7 +83,7 @@ def _check_attribute_is_not_empty(self, key, default, required, value_type):
     def check_not_trying_to_access_data_outside_root(self, bucket_name, rel_path):
         if rel_path is None:
             return
-        root = self._bucket_settings_service.bucket_root_directory(bucket_name)
+        root = self._bucket_settings_service.bucket_data_root_directory(bucket_name)
         canonical_root_path = self._file_system_helper.canonical_path(root)
         full_path = self._file_system_helper.join_paths(canonical_root_path, rel_path)
         directory_path = self._file_system_helper.file_directory(full_path) \

ditto_web_api/DittoWebApi/src/handlers/schemas/schema_helpers.py

@@ -21,9 +21,10 @@ def create_bucket_input_schema():
                     "items": {"type": "string"}
                 }
             },
-            "root": {"type": "string"}
+            "data_root": {"type": "string"},
+            "archive_root": {"type": "string"}
         },
-        "required": ["bucket", "groups", "root"]
+        "required": ["bucket", "groups", "data_root"]
     }
 
 

ditto_web_api/DittoWebApi/src/services/bucket_settings_service.py

@@ -10,15 +10,28 @@
 class BucketSetting:
     def __init__(self, properties):
         self._groups = str2list(properties['groups']) if isinstance(properties['groups'], str) else properties['groups']
-        self._root_dir = properties['root']
+        self._archive_root_dir = self.get_archive_root(properties)
+        self._data_root_dir = properties['data_root']
 
     @property
     def groups(self):
         return self._groups
 
     @property
-    def root_dir(self):
-        return self._root_dir
+    def data_root_dir(self):
+        return self._data_root_dir
+
+    @property
+    def archive_root_dir(self):
+        return self._archive_root_dir
+
+    @staticmethod
+    def get_archive_root(properties):
+        try:
+            archive_root = properties['archive_root']
+        except KeyError:
+            archive_root = properties['data_root']
+        return archive_root
 
 
 class BucketSettingsService:
@@ -51,19 +64,20 @@ def _write_settings(self):
             settings[bucket_name] = {}
             groups = ','.join(setting.groups)
             settings[bucket_name]['groups'] = groups
-            settings[bucket_name]['root'] = setting.root_dir
+            settings[bucket_name]['archive_root'] = setting.archive_root_dir
+            settings[bucket_name]['data_root'] = setting.data_root_dir
         text = config_to_string(settings)
         self._file_read_write_helper.write_text_to_file_path(self._bucket_settings_path, text)
 
     @property
     def admin_groups(self):
         return self._admin_groups
 
-    def add_bucket(self, bucket_name, groups, root_dir):
+    def add_bucket(self, bucket_name, groups, archive_root_dir, data_root_dir):
         if self.is_bucket_recognised(bucket_name):
             raise exceptions.APIError(404, f'Bucket "{bucket_name}" already exists')
         self._logger.info(f'Adding new bucket "{bucket_name}" to settings')
-        new_setting = BucketSetting({'groups': groups, 'root': root_dir})
+        new_setting = BucketSetting({'groups': groups, 'archive_root': archive_root_dir, 'data_root': data_root_dir})
         self._settings[bucket_name] = new_setting
         self._write_settings()
 
@@ -73,9 +87,15 @@ def bucket_permitted_groups(self, bucket_name):
         self._logger.warning(f'Permitted groups requested for non-existent bucket "{bucket_name}"')
         raise exceptions.APIError(404, f'Bucket "{bucket_name}" does not exist')
 
-    def bucket_root_directory(self, bucket_name):
+    def bucket_data_root_directory(self, bucket_name):
+        if bucket_name in self._settings:
+            return self._settings[bucket_name].data_root_dir
+        self._logger.warning(f'Root directory requested for non-existent bucket "{bucket_name}"')
+        raise exceptions.APIError(404, f'Bucket "{bucket_name}" does not exist')
+
+    def bucket_archive_root_directory(self, bucket_name):
         if bucket_name in self._settings:
-            return self._settings[bucket_name].root_dir
+            return self._settings[bucket_name].archive_root_dir
         self._logger.warning(f'Root directory requested for non-existent bucket "{bucket_name}"')
         raise exceptions.APIError(404, f'Bucket "{bucket_name}" does not exist')
 

ditto_web_api/DittoWebApi/src/services/data_replication/bucket_validator.py

@@ -24,7 +24,7 @@ def check_bucket(self, bucket_name):
             bucket_warning = BucketWarning(messages.bucket_not_exists(bucket_name), StatusCodes.Not_found)
 
         if bucket_warning is not None:
-            self._logger.warning(bucket_warning)
+            self._logger.warning(bucket_warning.message)
             return bucket_warning
 
         self._logger.debug("No bucket related warnings found")

ditto_web_api/DittoWebApi/src/services/data_replication/data_replication_service.py

@@ -48,8 +48,8 @@ def copy_dir(self, bucket_name, dir_path):
         if bucket_warning is not None:
             return return_transfer_summary(message=bucket_warning.message, status=bucket_warning.status)
 
-        root_dir = self._bucket_settings_service.bucket_root_directory(bucket_name)
-        files_in_directory = self._internal_data_service.find_files(root_dir, dir_path)
+        data_root_dir = self._bucket_settings_service.bucket_data_root_directory(bucket_name)
+        files_in_directory = self._internal_data_service.find_files(data_root_dir, dir_path)
 
         directory = dir_path if dir_path else "root"
 
@@ -66,10 +66,11 @@ def copy_dir(self, bucket_name, dir_path):
 
         file_summary = self._storage_difference_processor.return_difference_comparison([], files_in_directory)
         transfer_summary = self._external_data_service.perform_transfer(bucket_name, file_summary)
-        self._internal_data_service.archive_file_transfer(file_summary, root_dir)
+        archive_root_dir = self._bucket_settings_service.bucket_archive_root_directory(bucket_name)
+        self._internal_data_service.archive_file_transfer(file_summary, archive_root_dir)
         return return_transfer_summary(**transfer_summary)
 
-    def create_bucket(self, bucket_name, groups, root_dir):
+    def create_bucket(self, bucket_name, groups, archive_root_dir, data_root_dir):
         self._logger.debug("Called create-bucket handler")
         if not bucket_name:
             warning = messages.no_bucket_name()
@@ -91,7 +92,7 @@ def create_bucket(self, bucket_name, groups, root_dir):
             return return_bucket_message(
                 messages.bucket_breaks_s3_convention(bucket_name), bucket_name, status=StatusCodes.Bad_request)
         self._external_data_service.create_bucket(bucket_name)
-        self._bucket_settings_service.add_bucket(bucket_name, groups, root_dir)
+        self._bucket_settings_service.add_bucket(bucket_name, groups, archive_root_dir, data_root_dir)
         return return_bucket_message(messages.bucket_created(bucket_name), bucket_name)
 
     def try_delete_file(self, bucket_name, file_rel_path):
@@ -121,8 +122,8 @@ def copy_new(self, bucket_name, dir_path):
         if bucket_warning is not None:
             return return_transfer_summary(message=bucket_warning.message, status=bucket_warning.status)
         directory = dir_path if dir_path else "root"
-        root_dir = self._bucket_settings_service.bucket_root_directory(bucket_name)
-        files_in_directory = self._internal_data_service.find_files(root_dir, dir_path)
+        data_root_dir = self._bucket_settings_service.bucket_data_root_directory(bucket_name)
+        files_in_directory = self._internal_data_service.find_files(data_root_dir, dir_path)
 
         if not files_in_directory:
             warning = messages.no_files_found(directory)
@@ -139,7 +140,8 @@ def copy_new(self, bucket_name, dir_path):
             return return_transfer_summary(message=message,
                                            files_skipped=len(files_in_directory))
         transfer_summary = self._external_data_service.perform_transfer(bucket_name, files_summary)
-        self._internal_data_service.archive_file_transfer(files_summary, root_dir)
+        archive_root_dir = self._bucket_settings_service.bucket_archive_root_directory(bucket_name)
+        self._internal_data_service.archive_file_transfer(files_summary, archive_root_dir)
         return return_transfer_summary(**transfer_summary)
 
     def copy_new_and_update(self, bucket_name, dir_path):
@@ -149,8 +151,8 @@ def copy_new_and_update(self, bucket_name, dir_path):
             return return_transfer_summary(message=bucket_warning.message, status=bucket_warning.status)
 
         directory = dir_path if dir_path else "root"
-        root_dir = self._bucket_settings_service.bucket_root_directory(bucket_name)
-        files_in_directory = self._internal_data_service.find_files(root_dir, dir_path)
+        data_root_dir = self._bucket_settings_service.bucket_data_root_directory(bucket_name)
+        files_in_directory = self._internal_data_service.find_files(data_root_dir, dir_path)
 
         if not files_in_directory:
             warning = messages.no_files_found(directory)
@@ -167,7 +169,8 @@ def copy_new_and_update(self, bucket_name, dir_path):
             return return_transfer_summary(message=message,
                                            files_skipped=len(files_summary.files_to_be_skipped()))
         transfer_summary = self._external_data_service.perform_transfer(bucket_name, files_summary)
-        self._internal_data_service.archive_file_transfer(files_summary, root_dir)
+        archive_root_dir = self._bucket_settings_service.bucket_archive_root_directory(bucket_name)
+        self._internal_data_service.archive_file_transfer(files_summary, archive_root_dir)
         return return_transfer_summary(**transfer_summary)
 
 

ditto_web_api/DittoWebApi/src/services/internal/archiver.py

@@ -9,6 +9,7 @@ def __init__(self, file_read_write_helper, file_system_helper, logger):
 
     def write_archive(self, archive_file_path, file_summary):
         try:
+            self._create_directory_for_archive(archive_file_path)
             content = {}
             new_archive_file = self._file_system_helper.create_and_open_file_for_writing(archive_file_path)
 
@@ -53,3 +54,14 @@ def _archive_file_summary(self, file_summary, content):
 
         for file in file_summary.updated_files:
             content[file.file_name] = self._archive_file(file, time_of_transfer, "file update")
+
+    def _create_directory_for_archive(self, archive_file_path):
+        archive_file_directory_path = self._file_system_helper.file_directory(archive_file_path)
+        if self._file_system_helper.does_path_exist(archive_file_directory_path) is True:
+            return
+        else:
+            try:
+                self._file_system_helper.make_directory(archive_file_directory_path)
+            except OSError as error:
+                self._logger.debug(f"Error caused trying to make directory for archive file: {error}")
+                raise

ditto_web_api/DittoWebApi/src/utils/file_system/files_system_helpers.py

@@ -44,6 +44,9 @@ def join_paths(self, directory, file_name):
     def last_modified(self, file_path):
         return os.path.getmtime(file_path)
 
+    def make_directory(self, directory_path):
+        return os.makedirs(directory_path)
+
     def open_file_for_reading_and_writing(self, file_path):
         return open(file_path, 'r+')
 

ditto_web_api/DittoWebApi/tests/handlers/create_bucket_handler_test.py

@@ -23,7 +23,8 @@ def standard_body(self):
         return {
             'bucket': 'test-bucket',
             'groups': ['testgroup'],
-            'root': '/usr/tmp/data'
+            'data_root': '/usr/tmp/data',
+            'archive_root': '/usr/tmp/archive'
         }
 
     # Security
@@ -95,6 +96,31 @@ def test_post_create_bucket_returns_summary_of_new_bucket_when_successful(self):
         self.mock_data_replication_service.create_bucket.assert_called_once_with(
             "test-bucket",
             ['testgroup'],
+            "/usr/tmp/archive",
+            "/usr/tmp/data"
+        )
+        assert response_code == 200
+        assert response_body['status'] == 'success'
+        assert response_body['data'] == action_summary
+
+    @gen_test
+    def test_post_create_bucket_archive_root_is_taken_as_data_root_when_missing(self):
+        # Arrange
+        action_summary = {"message": "Bucket created", "bucket": "test-bucket", "status": StatusCodes.Okay}
+        self.mock_data_replication_service.create_bucket.return_value = action_summary
+        self._set_admin_user()
+        body = {
+            'bucket': 'test-bucket',
+            'groups': ['testgroup'],
+            'data_root': '/usr/tmp/data',
+        }
+        # Act
+        response_body, response_code = yield self.send_authorised_authenticated_request(body)
+        # Assert
+        self.mock_data_replication_service.create_bucket.assert_called_once_with(
+            "test-bucket",
+            ['testgroup'],
+            "/usr/tmp/data",
             "/usr/tmp/data"
         )
         assert response_code == 200

ditto_web_api/DittoWebApi/tests/services/bucket_settings_service_tests.py → ditto_web_api/DittoWebApi/tests/services/bucket_settings_service_test.py

@@ -1,8 +1,8 @@
 import logging
 import mock
+import unittest
 from mock import call
 import pytest
-import unittest
 
 from DittoWebApi.src.services.bucket_settings_service import BucketSettingsService
 from DittoWebApi.src.utils.configurations import Configuration
@@ -79,53 +79,57 @@ def test_settings_loads_empty_file(self):
     def test_settings_loads_single_bucket_settings_with_single_group(self):
         # Arrange
         self.mock_file_read_write_helper.read_file_path_as_text.return_value = \
-            '[test-bucket]\ngroups = group\nroot = /usr/tmp/ditto\n'
+            '[test-bucket]\ngroups = group\narchive_root = /usr/tmp/archive\ndata_root = /usr/tmp/ditto\n'
         # Act
         self.initialise_service_with_admin_groups(['admin'])
         # Assert
         assert len(self.test_service._settings) == 1
         assert self.test_service.is_bucket_recognised('test-bucket')
         assert self.test_service.bucket_permitted_groups('test-bucket') == ['group']
-        assert self.test_service.bucket_root_directory('test-bucket') == '/usr/tmp/ditto'
+        assert self.test_service.bucket_data_root_directory('test-bucket') == '/usr/tmp/ditto'
+        assert self.test_service.bucket_archive_root_directory('test-bucket') == '/usr/tmp/archive'
 
     def test_settings_loads_single_bucket_settings_with_multiple_groups(self):
         # Arrange
         self.mock_file_read_write_helper.read_file_path_as_text.return_value = \
-            '[test-bucket]\ngroups = group1,group2\nroot = /usr/tmp/ditto\n'
+            '[test-bucket]\ngroups = group1,group2\narchive_root = /usr/tmp/archive\ndata_root = /usr/tmp/ditto\n'
         # Act
         self.initialise_service_with_admin_groups(['admin'])
         # Assert
         assert len(self.test_service._settings) == 1
         assert self.test_service.is_bucket_recognised('test-bucket')
         assert self.test_service.bucket_permitted_groups('test-bucket') == ['group1', 'group2']
-        assert self.test_service.bucket_root_directory('test-bucket') == '/usr/tmp/ditto'
+        assert self.test_service.bucket_data_root_directory('test-bucket') == '/usr/tmp/ditto'
+        assert self.test_service.bucket_archive_root_directory('test-bucket') == '/usr/tmp/archive'
 
     def test_settings_loads_multiple_bucket_settings(self):
         # Arrange
         self.mock_file_read_write_helper.read_file_path_as_text.return_value = \
-            '[test-bucket]\ngroups = group1,group2\nroot = /usr/tmp/ditto\n\n' \
-            '[test-other]\ngroups = admin\nroot = /usr/tmp/other'
+            '[test-bucket]\ngroups = group1,group2\narchive_root = /usr/tmp/archive\ndata_root = /usr/tmp/ditto\n\n' \
+            '[test-other]\ngroups = admin\narchive_root = /usr/tmp/archive2\ndata_root = /usr/tmp/other'
         # Act
         self.initialise_service_with_admin_groups(['admin'])
         # Assert
         assert len(self.test_service._settings) == 2
         assert self.test_service.is_bucket_recognised('test-bucket')
         assert self.test_service.bucket_permitted_groups('test-bucket') == ['group1', 'group2']
-        assert self.test_service.bucket_root_directory('test-bucket') == '/usr/tmp/ditto'
+        assert self.test_service.bucket_data_root_directory('test-bucket') == '/usr/tmp/ditto'
+        assert self.test_service.bucket_archive_root_directory('test-bucket') == '/usr/tmp/archive'
         assert self.test_service.is_bucket_recognised('test-other')
         assert self.test_service.bucket_permitted_groups('test-other') == ['admin']
-        assert self.test_service.bucket_root_directory('test-other') == '/usr/tmp/other'
+        assert self.test_service.bucket_data_root_directory('test-other') == '/usr/tmp/other'
+        assert self.test_service.bucket_archive_root_directory('test-other') == '/usr/tmp/archive2'
 
     def test_add_bucket_appends_new_settings_to_file(self):
         # Arrange
         self.mock_file_read_write_helper.read_file_path_as_text.return_value = \
-            '[test-bucket]\ngroups = group1,group2\nroot = /usr/tmp/ditto\n'
+            '[test-bucket]\ngroups = group1,group2\narchive_root = /usr/tmp/archive\ndata_root = /usr/tmp/ditto\n'
         # Act
         self.initialise_service_with_admin_groups(['admin'])
-        self.test_service.add_bucket('test-other', 'admin', '/usr/tmp/other')
+        self.test_service.add_bucket('test-other', 'admin', '/usr/tmp/archive2', '/usr/tmp/other')
         # Assert
         self.mock_file_read_write_helper.write_text_to_file_path.assert_called_once_with(
-            '[test-bucket]\ngroups = group\nroot = /usr/tmp/ditto\n\n'
-            '[test-other]\ngroups = admin\nroot = /usr/tmp/other\n',
-            self.mock_bucket_settings_path
+            self.mock_bucket_settings_path,
+            '[test-bucket]\ngroups = group1,group2\narchive_root = /usr/tmp/archive\ndata_root = /usr/tmp/ditto\n\n'
+            '[test-other]\ngroups = admin\narchive_root = /usr/tmp/archive2\ndata_root = /usr/tmp/other\n\n'
         )