Merge pull request #97 from ukaea/feature/additional-content-in-archive/

#8

Feature/additional content in archive/#8

.travis.yml

@@ -8,6 +8,7 @@ install:
   - pip install -r requirements.txt
 before_script:
   - export PYTHONPATH=$PYTHONPATH:$(pwd)
+  - sh build_version_number.sh
 script:
   - flake8 ./DittoWebApi
   - python lint_threshold_check.py ./DittoWebApi

ditto_web_api/DittoWebApi/src/services/data_replication/data_replication_service.py

@@ -67,7 +67,7 @@ def copy_dir(self, bucket_name, dir_path):
         file_summary = self._storage_difference_processor.return_difference_comparison([], files_in_directory)
         transfer_summary = self._external_data_service.perform_transfer(bucket_name, file_summary)
         archive_root_dir = self._bucket_settings_service.bucket_archive_root_directory(bucket_name)
-        self._internal_data_service.archive_file_transfer(file_summary, archive_root_dir)
+        self._internal_data_service.archive_file_transfer(bucket_name, file_summary, archive_root_dir)
         return return_transfer_summary(**transfer_summary)
 
     def create_bucket(self, bucket_name, groups, archive_root_dir, data_root_dir):
@@ -141,7 +141,7 @@ def copy_new(self, bucket_name, dir_path):
                                            files_skipped=len(files_in_directory))
         transfer_summary = self._external_data_service.perform_transfer(bucket_name, files_summary)
         archive_root_dir = self._bucket_settings_service.bucket_archive_root_directory(bucket_name)
-        self._internal_data_service.archive_file_transfer(files_summary, archive_root_dir)
+        self._internal_data_service.archive_file_transfer(bucket_name, files_summary, archive_root_dir)
         return return_transfer_summary(**transfer_summary)
 
     def copy_new_and_update(self, bucket_name, dir_path):
@@ -170,7 +170,7 @@ def copy_new_and_update(self, bucket_name, dir_path):
                                            files_skipped=len(files_summary.files_to_be_skipped()))
         transfer_summary = self._external_data_service.perform_transfer(bucket_name, files_summary)
         archive_root_dir = self._bucket_settings_service.bucket_archive_root_directory(bucket_name)
-        self._internal_data_service.archive_file_transfer(files_summary, archive_root_dir)
+        self._internal_data_service.archive_file_transfer(bucket_name, files_summary, archive_root_dir)
         return return_transfer_summary(**transfer_summary)
 
 

ditto_web_api/DittoWebApi/src/services/internal/archiver.py

@@ -1,4 +1,12 @@
 from DittoWebApi.src.utils.system_helper import current_time_in_utc
+try:
+    from DittoWebApi.version import __version__
+except ModuleNotFoundError:
+    import sys
+    print("")
+    print("Version not found, please follow instructions in GitHub wiki to build version number")
+    print("")
+    sys.exit()
 
 
 class Archiver:
@@ -7,13 +15,13 @@ def __init__(self, file_read_write_helper, file_system_helper, logger):
         self._file_system_helper = file_system_helper
         self._file_read_write_helper = file_read_write_helper
 
-    def write_archive(self, archive_file_path, file_summary):
+    def write_archive(self, archive_file_path, bucket_name, file_summary):
         try:
             self._create_directory_for_archive(archive_file_path)
             content = {}
             new_archive_file = self._file_system_helper.create_and_open_file_for_writing(archive_file_path)
 
-            self._archive_file_summary(file_summary, content)
+            self._archive_file_summary(bucket_name, file_summary, content)
 
             self._file_read_write_helper.write_json_to_file(new_archive_file, content)
         except Exception as exception:
@@ -23,13 +31,13 @@ def write_archive(self, archive_file_path, file_summary):
             self._file_system_helper.close_file(new_archive_file)
         self._logger.debug(f"Archive file created: {archive_file_path}")
 
-    def update_archive(self, archive_file_path, file_summary):
+    def update_archive(self, archive_file_path, bucket_name, file_summary):
         try:
             archived_file = self._file_system_helper.open_file_for_reading_and_writing(archive_file_path)
             content = self._file_read_write_helper.read_file_as_json(archived_file)
             self._file_read_write_helper.clear_file(archived_file)
 
-            self._archive_file_summary(file_summary, content)
+            self._archive_file_summary(bucket_name, file_summary, content)
 
             self._file_read_write_helper.write_json_to_file(archived_file, content)
 
@@ -40,20 +48,22 @@ def update_archive(self, archive_file_path, file_summary):
             self._file_system_helper.close_file(archived_file)
         self._logger.debug(f"Archive file updated: {archive_file_path}")
 
-    def _archive_file(self, file, time_of_transfer, type_of_transfer):
+    def _archive_file(self, bucket_name, file, time_of_archive, type_of_transfer):
         size = self._file_system_helper.file_size(file.abs_path)
         return {"file": file.file_name,
+                "bucket": bucket_name,
                 "size": size,
-                "latest update": time_of_transfer,
-                "type of transfer": type_of_transfer}
+                "last archived": time_of_archive,
+                "type of transfer": type_of_transfer,
+                "ditto version": __version__}
 
-    def _archive_file_summary(self, file_summary, content):
+    def _archive_file_summary(self, bucket_name, file_summary, content):
         time_of_transfer = str(current_time_in_utc())
         for file in file_summary.new_files:
-            content[file.file_name] = self._archive_file(file, time_of_transfer, "new upload")
+            content[file.file_name] = self._archive_file(bucket_name, file, time_of_transfer, "new upload")
 
         for file in file_summary.updated_files:
-            content[file.file_name] = self._archive_file(file, time_of_transfer, "file update")
+            content[file.file_name] = self._archive_file(bucket_name, file, time_of_transfer, "file update")
 
     def _create_directory_for_archive(self, archive_file_path):
         archive_file_directory_path = self._file_system_helper.file_directory(archive_file_path)

ditto_web_api/DittoWebApi/src/services/internal/internal_data_service.py

@@ -29,7 +29,7 @@ def _build_file_information(self, root_dir, file_path):
         file_name = self._file_system_helper.file_name(abs_path_to_file)
         return FileInformation(abs_path_to_file, rel_path_to_file, file_name)
 
-    def archive_file_transfer(self, file_summary, root_dir):
+    def archive_file_transfer(self, bucket_name, file_summary, root_dir):
         sub_directory_file_summaries = self._split_file_summary_by_sub_dir(file_summary)
 
         for sub_dir in sub_directory_file_summaries:
@@ -40,9 +40,9 @@ def archive_file_transfer(self, file_summary, root_dir):
             sub_dir_file_summary = sub_directory_file_summaries[sub_dir]
 
             if self._file_system_helper.does_path_exist(archive_file_path):
-                self._archiver.update_archive(archive_file_path, sub_dir_file_summary)
+                self._archiver.update_archive(archive_file_path, bucket_name, sub_dir_file_summary)
             else:
-                self._archiver.write_archive(archive_file_path, sub_dir_file_summary)
+                self._archiver.write_archive(archive_file_path, bucket_name, sub_dir_file_summary)
 
     def _split_file_summary_by_sub_dir(self, file_summary):
         dict_of_sub_dir_summaries = {}

ditto_web_api/DittoWebApi/tests/services/archiver_test.py

@@ -4,6 +4,7 @@
 import pytest
 import mock
 
+from DittoWebApi.version import __version__ as version
 from DittoWebApi.src.utils.file_system.files_system_helpers import FileSystemHelper
 from DittoWebApi.src.models.file_storage_summary import FilesStorageSummary
 from DittoWebApi.src.models.file_information import FileInformation
@@ -37,20 +38,24 @@ def test_write_archive_creates_archive_file(self, time):
         self.mock_file_summary.new_files = [self.mock_file_1]
         self.mock_file_summary.updated_files = [self.mock_file_2]
         # Act
-        self.test_archiver.write_archive("some_file_path", self.mock_file_summary)
+        self.test_archiver.write_archive("some_file_path", "some_bucket", self.mock_file_summary)
         # Assert
         self._logger.debug.assert_called_with("Archive file created: some_file_path")
         self._file_read_write_helper.write_json_to_file.assert_called_once_with(
             self.mock_open_file, {self.mock_file_1.file_name: {
                 'file': self.mock_file_1.file_name,
+                'bucket': "some_bucket",
                 'size': 100,
-                'latest update': '1970-01-01 03:25:45',
-                'type of transfer': 'new upload'},
+                'last archived': '1970-01-01 03:25:45',
+                'type of transfer': 'new upload',
+                'ditto version': version},
                                   self.mock_file_2.file_name: {
                                       'file': self.mock_file_2.file_name,
+                                      'bucket': "some_bucket",
                                       'size': 50,
-                                      'latest update': '1970-01-01 03:25:45',
-                                      'type of transfer': 'file update'}}
+                                      'last archived': '1970-01-01 03:25:45',
+                                      'type of transfer': 'file update',
+                                      'ditto version': version}}
         )
 
     @mock.patch('DittoWebApi.src.utils.system_helper.time.time', return_value=12345)
@@ -61,26 +66,32 @@ def test_update_archive_updates_an_archive_file(self, time):
         self._file_read_write_helper.read_file_as_json.return_value = {
             self.mock_file_2.file_name: {
                 'file': self.mock_file_2.file_name,
+                'bucket': "some_bucket",
                 'size': 50,
-                'latest update': '1970-01-01 03:25:45',
-                'type of transfer': 'file update'}
+                'last archived': '1970-01-01 03:25:45',
+                'type of transfer': 'file update',
+                'ditto version': version}
         }
         # Act
-        self.test_archiver.update_archive("some_file_path", self.mock_file_summary)
+        self.test_archiver.update_archive("some_file_path", "some_bucket", self.mock_file_summary)
         # Assert
         self._logger.debug.assert_called_with("Archive file updated: some_file_path")
         self._file_read_write_helper.write_json_to_file.assert_called_once_with(
             self.mock_open_file, {
                 self.mock_file_2.file_name: {
                     'file': self.mock_file_2.file_name,
+                    'bucket': "some_bucket",
                     'size': 50,
-                    'latest update': '1970-01-01 03:25:45',
-                    'type of transfer': 'file update'},
+                    'last archived': '1970-01-01 03:25:45',
+                    'type of transfer': 'file update',
+                    'ditto version': version},
                 self.mock_file_1.file_name: {
                     'file': self.mock_file_1.file_name,
+                    'bucket': "some_bucket",
                     'size': 100,
-                    'latest update': '1970-01-01 03:25:45',
-                    'type of transfer': 'new upload'}
+                    'last archived': '1970-01-01 03:25:45',
+                    'type of transfer': 'new upload',
+                    'ditto version': version}
             })
 
     @mock.patch('DittoWebApi.src.utils.system_helper.time.time', return_value=12345)
@@ -91,25 +102,30 @@ def test_update_archive_updates_an_archive_file_when_no_new_files(self, time):
         self._file_read_write_helper.read_file_as_json.return_value = {
             self.mock_file_2.file_name: {
                 'file': self.mock_file_2.file_name,
+                'bucket': "some_bucket",
                 'size': 50,
-                'latest update': '1970-01-01 03:25:45',
-                'type of transfer': 'new upload'}
+                'last archived': '1970-01-01 03:25:45',
+                'type of transfer': 'new upload',
+                'ditto version': version}
         }
         # Act
-        self.test_archiver.update_archive("some_file_path", self.mock_file_summary)
+        self.test_archiver.update_archive("some_file_path", "some_bucket", self.mock_file_summary)
         # Assert
         self._logger.debug.assert_called_with("Archive file updated: some_file_path")
         self._file_read_write_helper.write_json_to_file.assert_called_once_with(
             self.mock_open_file, {
                 self.mock_file_1.file_name: {
                     'file': self.mock_file_1.file_name,
+                    'bucket': "some_bucket",
                     'size': 100,
-                    'latest update': '1970-01-01 03:25:45',
-                    'type of transfer': 'file update'},
+                    'last archived': '1970-01-01 03:25:45',
+                    'type of transfer': 'file update',
+                    'ditto version': version},
                 self.mock_file_2.file_name: {
                     'file': self.mock_file_2.file_name,
+                    'bucket': "some_bucket",
                     'size': 50,
-                    'latest update': '1970-01-01 03:25:45',
-                    'type of transfer': 'file update'}
-
+                    'last archived': '1970-01-01 03:25:45',
+                    'type of transfer': 'file update',
+                    'ditto version': version}
             })

ditto_web_api/DittoWebApi/tests/services/internal_data_services_test.py

@@ -212,7 +212,7 @@ def test_create_archive_file_called_for_each_subdir_for_files_transferred_when_a
         self.mock_file_summary.new_files = [self.mock_file_1, self.mock_file_2]
         self.mock_file_summary.updated_files = []
         # Act
-        self.internal_data_services.archive_file_transfer(self.mock_file_summary, "/usr/tmp/data")
+        self.internal_data_services.archive_file_transfer("some_bucket", self.mock_file_summary, "/usr/tmp/data")
         # Assert
         assert self.mock_archiver.write_archive.call_count == 1
         assert self.mock_archiver.update_archive.call_count == 0
@@ -224,7 +224,7 @@ def test_create_archive_file_calls_archiver_with_updated_content_when_archive_fi
         self.mock_file_summary.new_files = []
         self.mock_file_summary.updated_files = [self.mock_file_1, self.mock_file_2]
         # Act
-        self.internal_data_services.archive_file_transfer(self.mock_file_summary, "/usr/tmp/data")
+        self.internal_data_services.archive_file_transfer("some_bucket", self.mock_file_summary, "/usr/tmp/data")
         # Assert
         assert self.mock_archiver.write_archive.call_count == 0
         assert self.mock_archiver.update_archive.call_count == 1
@@ -237,7 +237,7 @@ def test_create_archive_file_calls_archiver_with_updated_content_when_archive_fi
         self.mock_file_summary.new_files = []
         self.mock_file_summary.updated_files = [self.mock_file_1, self.mock_file_2]
         # Act
-        self.internal_data_services.archive_file_transfer(self.mock_file_summary, "/usr/tmp/data")
+        self.internal_data_services.archive_file_transfer("some_bucket", self.mock_file_summary, "/usr/tmp/data")
         # Assert
         assert self.mock_archiver.write_archive.call_count == 0
         assert self.mock_archiver.update_archive.call_count == 1
@@ -251,7 +251,7 @@ def test_create_archive_file_updates_archive_when_exists_and_creates_new_when_no
         self.mock_file_summary.new_files = [self.mock_file_2]
         self.mock_file_summary.updated_files = [self.mock_file_1]
         # Act
-        self.internal_data_services.archive_file_transfer(self.mock_file_summary, "/usr/tmp/data")
+        self.internal_data_services.archive_file_transfer("some_bucket", self.mock_file_summary, "/usr/tmp/data")
         # Assert
         assert self.mock_archiver.write_archive.call_count == 1
         assert self.mock_archiver.update_archive.call_count == 1
@@ -264,7 +264,7 @@ def test_create_archive_file_updates_archive_when_exists_even_when_all_files_are
         self.mock_file_summary.new_files = [self.mock_file_1, self.mock_file_2]
         self.mock_file_summary.updated_files = []
         # Act
-        self.internal_data_services.archive_file_transfer(self.mock_file_summary, "/usr/tmp/data")
+        self.internal_data_services.archive_file_transfer("some_bucket", self.mock_file_summary, "/usr/tmp/data")
         # Assert
         assert self.mock_archiver.write_archive.call_count == 0
         assert self.mock_archiver.update_archive.call_count == 1
@@ -277,7 +277,7 @@ def test_create_archive_file_creates_archive_that_do_not_exist_even_when_all_fil
         self.mock_file_summary.new_files = []
         self.mock_file_summary.updated_files = [self.mock_file_1, self.mock_file_2]
         # Act
-        self.internal_data_services.archive_file_transfer(self.mock_file_summary, "/usr/tmp/data")
+        self.internal_data_services.archive_file_transfer("some_bucket", self.mock_file_summary, "/usr/tmp/data")
         # Assert
         assert self.mock_archiver.write_archive.call_count == 1
         assert self.mock_archiver.update_archive.call_count == 0