Merge branch 'feature/create-archive-file-when-performing-a-transfer/#7…

…' into feature/add-metadata-to-archive-files/#8

ditto_web_api/DittoWebApi/example_configuration.ini

@@ -9,3 +9,4 @@ S3SecretKey = SERVICE_KEY
 S3Secure = True
 RootDirectory = PATH_TO_ROOT_DIRECTORY
 BucketStandardisation = BUCKET_STANDARD
+ArchiveFileName = .ditto_archived

ditto_web_api/DittoWebApi/main.py

@@ -53,9 +53,9 @@ def setup_logger(log_file_location, level):
     # Set up services
     S3_ADAPTER = BotoAdapter(CONFIGURATION, LOGGER)
     FILE_SYSTEM_HELPER = FileSystemHelper()
-    ARCHIVER = Archiver(LOGGER, FILE_SYSTEM_HELPER)
-    EXTERNAL_DATA_SERVICE = ExternalDataService(CONFIGURATION, LOGGER, S3_ADAPTER, FILE_SYSTEM_HELPER)
-    INTERNAL_DATA_SERVICE = InternalDataService(CONFIGURATION, FILE_SYSTEM_HELPER, ARCHIVER, LOGGER)
+    ARCHIVER = Archiver(FILE_SYSTEM_HELPER, LOGGER)
+    EXTERNAL_DATA_SERVICE = ExternalDataService(CONFIGURATION, FILE_SYSTEM_HELPER, LOGGER, S3_ADAPTER)
+    INTERNAL_DATA_SERVICE = InternalDataService(ARCHIVER, CONFIGURATION, FILE_SYSTEM_HELPER, LOGGER)
     STORAGE_DIFFERENCE_PROCESSOR = StorageDifferenceProcessor(LOGGER)
     DATA_REPLICATION_SERVICE = DataReplicationService(EXTERNAL_DATA_SERVICE,
                                                       INTERNAL_DATA_SERVICE,

ditto_web_api/DittoWebApi/src/services/data_replication/data_replication_service.py

@@ -56,7 +56,7 @@ def copy_dir(self, bucket_name, dir_path):
 
         file_summary = self._storage_difference_processor.return_difference_comparison([], files_in_directory)
         transfer_summary = self._external_data_service.perform_transfer(bucket_name, file_summary)
-        self._internal_data_service.create_archive_file(dir_path, file_summary=file_summary)
+        self._internal_data_service.archive_file_transfer(dir_path, file_summary=file_summary)
         return return_transfer_summary(**transfer_summary)
 
     def create_bucket(self, bucket_name):
@@ -124,7 +124,7 @@ def copy_new(self, bucket_name, dir_path):
             return return_transfer_summary(message=message,
                                            files_skipped=len(files_in_directory))
         transfer_summary = self._external_data_service.perform_transfer(bucket_name, files_summary)
-        self._internal_data_service.create_archive_file(dir_path, file_summary=files_summary)
+        self._internal_data_service.archive_file_transfer(dir_path, file_summary=files_summary)
         return return_transfer_summary(**transfer_summary)
 
     def copy_new_and_update(self, bucket_name, dir_path):
@@ -152,5 +152,5 @@ def copy_new_and_update(self, bucket_name, dir_path):
             return return_transfer_summary(message=message,
                                            files_skipped=len(files_summary.files_to_be_skipped()))
         transfer_summary = self._external_data_service.perform_transfer(bucket_name, files_summary)
-        self._internal_data_service.create_archive_file(dir_path, file_summary=files_summary)
+        self._internal_data_service.archive_file_transfer(dir_path, file_summary=files_summary)
         return return_transfer_summary(**transfer_summary)

ditto_web_api/DittoWebApi/src/services/external/external_data_service.py

@@ -6,7 +6,7 @@
 
 
 class ExternalDataService:
-    def __init__(self, configuration, logger, s3_adapter, file_system_helper):
+    def __init__(self, configuration, file_system_helper, logger, s3_adapter):
         self._logger = logger
         self._bucket_standard = configuration.bucket_standard
         self._s3_client = s3_adapter

ditto_web_api/DittoWebApi/src/services/internal/archiver.py

@@ -2,18 +2,28 @@
 
 
 class Archiver:
-    def __init__(self, logger, file_system_helper):
+    def __init__(self, file_system_helper, logger):
         self._logger = logger
         self._file_system_helper = file_system_helper
 
-    def archive_transfer(self, file_summary, old_content):
-        content = {} if old_content is None else old_content
-        time_of_transfer = current_time()
-        for file in file_summary.new_files:
-            self._archive_new_file(content, file, time_of_transfer)
-        for file in file_summary.updated_files:
-            self._update_archive(content, file, time_of_transfer)
-        return content
+    def write_archive(self, file_path, file_summary):
+        content = "test"
+        self._file_system_helper.create_file(file_path, content)
+        self._logger.debug(f"Archive file created: {file_path}")
+
+
+    def update_archive(self, file_path, file_summary):
+        content = "test test"
+        try:
+            archived_file = self._file_system_helper.open_file(file_path)
+            archived_file.write(content)
+
+        except Exception as exception:
+            self._logger.error(f"Exception found: {exception}")
+            raise
+        finally:
+            self._file_system_helper.close_file(archived_file)
+        self._logger.debug(f"Archive file updated: {file_path}")
 
     def _archive_new_file(self, content, file, time_of_transfer):
         size = self._file_system_helper.file_size(file.abs_path)
@@ -33,3 +43,12 @@ def _archive_file_deletion(self, content, file, time_of_transfer):
         content[file.rel_path]["latest update"] = time_of_transfer
         content[file.rel_path]["type of transfer"] = "delete"
         content[file.rel_path]["size"] = 0
+
+    def archive_transfer(self, file_summary, old_content):
+        content = {} if old_content is None else old_content
+        time_of_transfer = current_time()
+        for file in file_summary.new_files:
+            self._archive_new_file(content, file, time_of_transfer)
+        for file in file_summary.updated_files:
+            self._update_archive(content, file, time_of_transfer)
+        return content

ditto_web_api/DittoWebApi/src/services/internal/internal_data_service.py

@@ -1,23 +1,21 @@
-import re
-
 from DittoWebApi.src.models.file_information import FileInformation
 
 
 class InternalDataService:
-    def __init__(self, configuration, file_system_helper, archiver, logger):
+    def __init__(self, archiver, configuration, file_system_helper, logger):
         self._root_dir = configuration.root_dir
         self._file_system_helper = file_system_helper
         self._archiver = archiver
         self._logger = logger
+        self._archive_file_name = configuration.archive_file_name
 
     def find_files(self, dir_path):
         dir_to_search = self._file_system_helper.join_paths(self._root_dir, dir_path) \
             if dir_path \
             else self._root_dir
         self._logger.debug(f"Finding files in directory {dir_to_search}")
         all_files = self._file_system_helper.find_all_files_in_folder(dir_to_search)
-        regex = re.compile(r'.*\.ditto_archived')
-        list_of_files = list(filter(lambda name: not regex.match(name), all_files))
+        list_of_files = list(filter(lambda name: not name.endswith(self._archive_file_name), all_files))
         self._logger.debug(f"Found {len(list_of_files)} files, converting to file information objects")
         file_information_list = [self.build_file_information(full_file_name) for full_file_name in list_of_files]
         self._logger.info(f"{len(file_information_list)} files found in {dir_path}")
@@ -29,16 +27,13 @@ def build_file_information(self, file_path):
         file_name = self._file_system_helper.file_name(abs_path_to_file)
         return FileInformation(abs_path_to_file, rel_path_to_file, file_name)
 
-    def create_archive_file(self, dir_path, file_summary):
+    def archive_file_transfer(self, dir_path, file_summary):
         full_dir_path = self._file_system_helper.join_paths(self._root_dir, dir_path) if dir_path else self._root_dir
-        file_path = self._file_system_helper.join_paths(full_dir_path, ".ditto_archived")
+        file_path = self._file_system_helper.join_paths(full_dir_path, self._archive_file_name)
         if self._file_system_helper.does_file_exist(file_path):
-            old_content = self._file_system_helper.load_content(file_path)
-            content_to_write = self._archiver.archive_transfer(old_content, file_summary)
+            self._archiver.update_archive(file_path, file_summary)
         else:
-            content_to_write = self._archiver.archive_transfer(None, file_summary)
-        self._logger.info("Creating archive file")
-        self._file_system_helper.create_file(file_path, content_to_write)
+            self._archiver.write_archive(file_path, file_summary)
 
     def archive_file_deletion(self, file_rel_path):
         pass

ditto_web_api/DittoWebApi/src/utils/configurations.py

@@ -72,6 +72,10 @@ def root_dir(self):
     def bucket_standard(self):
         return self._bucket_standard
 
+    @property
+    def archive_file_name(self):
+        return self._archive_file_name
+
     def _parse(self, path):
         config = configparser.ConfigParser()
         config.read(path)
@@ -94,6 +98,7 @@ def _parse(self, path):
         # Local data
         self._root_dir = os.path.abspath(settings['RootDirectory'])
         self._bucket_standard = settings['BucketStandardisation']
+        self._archive_file_name = settings['ArchiveFileName']
 
     @staticmethod
     def get_directory(settings, key):

ditto_web_api/DittoWebApi/src/utils/file_system/files_system_helpers.py

@@ -30,13 +30,15 @@ def file_size(self, abs_file_path):
         return os.stat(abs_file_path).st_size
 
     def create_file(self, file_path, content):
-        with open(file_path, "w+") as file:
-            json.dump(content, file)
+        with open(file_path, "w") as file:
+            file.write(content)
 
     def does_file_exist(self, file_path):
         return os.path.exists(file_path)
 
-    def load_content(self, file_path):
-        with open(file_path, 'rt') as file:
-            content = json.loads(file)
-        return content
+    def open_file(self, file_path):
+        return open(file_path, 'w')
+
+    def close_file(self, open_file):
+        if not open_file.closed:
+            open_file.close()

ditto_web_api/DittoWebApi/tests/services/external_data_services_test.py

@@ -29,9 +29,9 @@ def setup(self):
         self.mock_file_system_helper = mock.create_autospec(FileSystemHelper)
         self.test_service = ExternalDataService(
             mock_configuration,
+            self.mock_file_system_helper,
             self.mock_logger,
-            self.mock_s3_client,
-            self.mock_file_system_helper
+            self.mock_s3_client
         )
 
     @staticmethod

ditto_web_api/DittoWebApi/tests/services/internal_data_services_test.py

@@ -6,6 +6,7 @@
 from DittoWebApi.src.services.internal.internal_data_service import InternalDataService
 from DittoWebApi.src.services.internal.archiver import Archiver
 from DittoWebApi.src.utils.file_system.files_system_helpers import FileSystemHelper
+from DittoWebApi.src.models.file_storage_summary import FilesStorageSummary
 
 
 class TestInternalDataServices(unittest.TestCase):
@@ -16,6 +17,7 @@ def setup(self):
         self.mock_logger = mock.create_autospec(logging.RootLogger, spec_set=False)
         self.mock_archiver = mock.create_autospec(Archiver)
         mock_configuration.root_dir = "test_root_dir"
+        mock_configuration.archive_file_name = ".ditto_archived"
         self.mock_file_system_helper.join_paths.return_value = "test_root_dir/file_1"
         self.mock_file_system_helper.find_all_files_in_folder.return_value = ["test_root_dir/file_1.txt",
                                                                               "test_root_dir/file_2.txt"]
@@ -25,9 +27,9 @@ def setup(self):
                                                                        "test_root_dir/file_2.txt"]
         self.mock_file_system_helper.file_name.side_effect = ["file_1.txt",
                                                               "file_2.txt"]
-        self.internal_data_services = InternalDataService(mock_configuration,
+        self.internal_data_services = InternalDataService(self.mock_archiver,
+                                                          mock_configuration,
                                                           self.mock_file_system_helper,
-                                                          self.mock_archiver,
                                                           self.mock_logger)
 
     def test_find_files_finds_files_in_a_directories(self):
@@ -79,20 +81,19 @@ def test_create_archive_file_calls_archiver_with_given_content_when_archive_file
         # Arrange
         self.mock_file_system_helper.does_file_exist.return_value = False
         self.mock_file_system_helper.join_paths.return_value = "root/.ditto_archived"
+        mock_file_summary = mock.create_autospec(FilesStorageSummary)
         # Act
-        self.internal_data_services.create_archive_file(None, "test_content")
+        self.internal_data_services.archive_file_transfer(None, mock_file_summary)
         # Assert
-        self.mock_file_system_helper.create_file.assert_called_once_with("root/.ditto_archived", "test_content")
+        self.mock_archiver.write_archive.assert_called_once_with("root/.ditto_archived", mock_file_summary)
 
     def test_create_archive_file_calls_archiver_with_updated_content_when_archive_filet_exist(self):
         # Arrange
         self.mock_file_system_helper.does_file_exist.return_value = True
         self.mock_file_system_helper.join_paths.return_value = "root/.ditto_archived"
-        self.mock_file_system_helper.load_content.return_value = "Some old content "
-        self.mock_archiver.update_content.return_value = "Some old content test_content"
+        self.mock_archiver.update_archive.return_value = "Some old content test_content"
+        mock_file_summary = mock.create_autospec(FilesStorageSummary)
         # Act
-        self.internal_data_services.create_archive_file(None, "test_content")
+        self.internal_data_services.archive_file_transfer(None, mock_file_summary)
         # Assert
-        self.mock_archiver.update_content.assert_called_once_with("Some old content ", "test_content")
-        self.mock_file_system_helper.create_file.assert_called_once_with("root/.ditto_archived",
-                                                                         "Some old content test_content")
+        self.mock_archiver.update_archive.assert_called_once_with("root/.ditto_archived", mock_file_summary)

ditto_web_api/DittoWebApi/tests/utils/configuration_test.py

@@ -18,7 +18,8 @@ def create_configuration(log_folder_loc,
                              s3_secret_key,
                              s3_use_secure,
                              root_dir,
-                             bucket_standard):
+                             bucket_standard,
+                             archive_file_name):
         template = "[Settings]\n"
         template = SampleConfigurationCreator.add_element_to_temp_file(template,
                                                                        "LogFolderLocation",
@@ -50,6 +51,9 @@ def create_configuration(log_folder_loc,
         template = SampleConfigurationCreator.add_element_to_temp_file(template,
                                                                        "BucketStandardisation",
                                                                        bucket_standard)
+        template = SampleConfigurationCreator.add_element_to_temp_file(template,
+                                                                       "ArchiveFileName",
+                                                                       archive_file_name)
         return SampleConfigurationCreator.write_sample_configuration_file(template)
 
     @staticmethod
@@ -92,7 +96,8 @@ def test_configuration_can_be_read_when_s3_secure():
                                                                          "secret",
                                                                          "true",
                                                                          current,
-                                                                         "test")
+                                                                         "test",
+                                                                         "test2")
 
     # Act
     configuration = Configuration(configuration_path)
@@ -106,6 +111,7 @@ def test_configuration_can_be_read_when_s3_secure():
     assert configuration.s3_use_secure is True
     assert configuration.root_dir == current
     assert configuration.bucket_standard == "test"
+    assert configuration.archive_file_name == "test2"
 
     # Clean up
     SampleConfigurationCreator.remove_file(configuration_path)
@@ -123,7 +129,8 @@ def test_configuration_can_be_read_when_s3_not_secure():
                                                                          "secret",
                                                                          "false",
                                                                          current,
-                                                                         "test")
+                                                                         "test",
+                                                                         "test2")
 
     # Act
     configuration = Configuration(configuration_path)
@@ -137,6 +144,7 @@ def test_configuration_can_be_read_when_s3_not_secure():
     assert configuration.s3_use_secure is False
     assert configuration.root_dir == current
     assert configuration.bucket_standard == "test"
+    assert configuration.archive_file_name == "test2"
 
     # Clean up
     SampleConfigurationCreator.remove_file(configuration_path)

systemTests/testScenarios/context.py

@@ -84,6 +84,18 @@ def local_data_folder_path(self):
     def s3_data_folder_path(self):
         return '/opt/minio/data'
 
+    @property
+    def authentication_username(self):
+        return 'Tom'
+
+    @property
+    def authentication_password(self):
+        return 'potato'
+
+    @property
+    def authentication_groups(self):
+        return 'group1'
+
     def _response_body_as_json(self):
         return json.loads(self.http_client_response.text)
 
@@ -109,6 +121,7 @@ def standard_bucket_name(self):
     def simple_file_name(self):
         return "testA.txt"
 
+
 class BaseSystemTest(unittest.TestCase):
     @pytest.fixture(autouse=True)
     def setup(self, request):

systemTests/testScenarios/given/ditto_api_server.py

@@ -9,6 +9,7 @@ def __init__(self, context):
 
     def is_started(self):
         self._write_configuration()
+        self._write_security()
         self._start_ditto()
 
     def is_started_without_configuration(self):
@@ -37,6 +38,21 @@ def _write_configuration(self):
         with open(config_file_path, 'w') as config_file:
             config_file.write(file_contents)
 
+    def _write_security(self):
+        file_contents = \
+            f'[{self._context.authentication_username}]\n' \
+            f'password = {self._context.authentication_password}\n' \
+            f'groups = {self._context.authentication_groups}\n'
+
+        security_file_path = os.path.join(
+            self._context.ditto_web_api_folder_path,
+            'DittoWebApi',
+            'security_configuration.ini'
+        )
+
+        with open(security_file_path, 'w') as security_file:
+            security_file.write(file_contents)
+
     def _start_ditto(self):
         path_of_file = os.path.dirname(os.path.realpath(__file__))
         web_api_script = os.path.join(path_of_file, 'runDittoWebApi.sh')