Fix RETRY - 1 issue

src/core/src/bootstrap/EnvLayer.py

@@ -348,7 +348,7 @@ def open(self, file_path, mode, raise_if_not_found=True):
                 try:
                     return open(real_path, mode)
                 except Exception as error:
-                    if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT:
+                    if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT - 1:
                         time.sleep(i + 1)
                     else:
                         error_message = "Unable to open file (retries exhausted). [File={0}][Error={1}][RaiseIfNotFound={2}].".format(str(real_path), repr(error), str(raise_if_not_found))
@@ -382,7 +382,7 @@ def read_with_retry(self, file_path_or_handle, raise_if_not_found=True):
                         self.__write_record(operation, code=0, output=value, delay=0)
                         return value
                     except Exception as error:
-                        if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT:
+                        if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT - 1:
                             time.sleep(i + 1)
                         else:
                             error_message = "Unable to read file (retries exhausted). [File={0}][Error={1}][RaiseIfNotFound={2}].".format(str(file_path_or_handle), repr(error), str(raise_if_not_found))
@@ -404,7 +404,7 @@ def write_with_retry(self, file_path_or_handle, data, mode='a+'):
                     file_handle.write(str(data))
                     break
                 except Exception as error:
-                    if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT:
+                    if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT - 1:
                         time.sleep(i + 1)
                     else:
                         raise Exception("Unable to write to {0} (retries exhausted). Error: {1}.".format(str(file_handle.name), repr(error)))
@@ -423,7 +423,7 @@ def write_with_retry_using_temp_file(file_path, data, mode='w'):
                     shutil.move(tempname, file_path)
                     break
                 except Exception as error:
-                    if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT:
+                    if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT - 1:
                         time.sleep(i + 1)
                     else:
                         raise Exception("Unable to write to {0} (retries exhausted). Error: {1}.".format(str(file_path), repr(error)))

src/core/src/core_logic/PatchAssessor.py

@@ -62,7 +62,7 @@ def start_assessment(self):
                 self.status_handler.set_assessment_substatus_json(status=Constants.STATUS_SUCCESS)
                 break
             except Exception as error:
-                if i < Constants.MAX_ASSESSMENT_RETRY_COUNT:
+                if i < Constants.MAX_ASSESSMENT_RETRY_COUNT - 1:
                     error_msg = 'Retryable error retrieving available patches: ' + repr(error)
                     self.composite_logger.log_warning(error_msg)
                     self.status_handler.add_error_to_status(error_msg, Constants.PatchOperationErrorCodes.DEFAULT_ERROR)

src/core/src/service_interfaces/LifecycleManager.py

@@ -56,7 +56,7 @@ def read_extension_sequence(self):
                 with self.env_layer.file_system.open(self.ext_state_file_path, mode="r") as file_handle:
                     return json.load(file_handle)['extensionSequence']
             except Exception as error:
-                if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT:
+                if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT - 1:
                     self.composite_logger.log_warning("Exception on extension sequence read. [Exception={0}] [RetryCount={1}]".format(repr(error), str(i)))
                     time.sleep(i+1)
                 else:
@@ -115,7 +115,7 @@ def read_core_sequence(self):
 
                 return core_sequence
             except Exception as error:
-                if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT:
+                if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT - 1:
                     self.composite_logger.log_warning("Exception on core sequence read. [Exception={0}] [RetryCount={1}]".format(repr(error), str(i)))
                     time.sleep(i + 1)
                 else:
@@ -145,7 +145,7 @@ def update_core_sequence(self, completed=False):
                 with self.env_layer.file_system.open(self.core_state_file_path, 'w+') as file_handle:
                     file_handle.write(core_state_payload)
             except Exception as error:
-                if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT:
+                if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT - 1:
                     self.composite_logger.log_warning("Exception on core sequence update. [Exception={0}] [RetryCount={1}]".format(repr(error), str(i)))
                     time.sleep(i + 1)
                 else:

src/core/src/service_interfaces/StatusHandler.py

@@ -504,7 +504,7 @@ def __load_status_file_components(self, initial_load=False):
                 with self.env_layer.file_system.open(self.status_file_path, 'r') as file_handle:
                     status_file_data_raw = json.load(file_handle)[0]    # structure is array of 1
             except Exception as error:
-                if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT:
+                if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT - 1:
                     time.sleep(i + 1)
                 else:
                     self.composite_logger.log_error("Unable to read status file (retries exhausted). Error: {0}.".format(repr(error)))

src/core/tests/Test_LifecycleManagerArc.py

@@ -54,7 +54,7 @@ def test_read_extension_sequence_fail(self):
         # file open throws exception
         self.lifecycle_manager.ext_state_file_path = old_ext_state_file_path
         self.runtime.env_layer.file_system.open = self.mock_file_open_throw_exception
-        ext_state_json = self.lifecycle_manager.read_extension_sequence()
+        ext_state_json = self.assertRaises(Exception, self.lifecycle_manager.read_extension_sequence)
         self.assertEquals(ext_state_json, None)
 
     def test_read_extension_sequence_success(self):
@@ -65,7 +65,7 @@ def test_read_extension_sequence_success(self):
     def test_read_core_sequence_fail(self):
         # file open throws exception
         self.runtime.env_layer.file_system.open = self.mock_file_open_throw_exception
-        core_sequence_json = self.lifecycle_manager.read_core_sequence()
+        core_sequence_json = self.assertRaises(Exception, self.lifecycle_manager.read_core_sequence)
         self.assertEquals(core_sequence_json, None)
 
     def test_read_core_sequence_success(self):

src/core/tests/Test_LifecycleManagerAzure.py

@@ -51,7 +51,7 @@ def test_read_extension_sequence_fail(self):
         # file open throws exception
         self.lifecycle_manager.ext_state_file_path = old_ext_state_file_path
         self.runtime.env_layer.file_system.open = self.mock_file_open_throw_exception
-        ext_state_json = self.lifecycle_manager.read_extension_sequence()
+        ext_state_json = self.assertRaises(Exception, self.lifecycle_manager.read_extension_sequence)
         self.assertEquals(ext_state_json, None)
 
     def test_read_extension_sequence_success(self):
@@ -61,9 +61,21 @@ def test_read_extension_sequence_success(self):
 
     def test_read_core_sequence_fail(self):
         # file open throws exception
+        backup_open = self.runtime.env_layer.file_system.open
         self.runtime.env_layer.file_system.open = self.mock_file_open_throw_exception
-        core_sequence_json = self.lifecycle_manager.read_core_sequence()
+        core_sequence_json = self.assertRaises(Exception, self.lifecycle_manager.read_core_sequence)
+        self.assertEquals(core_sequence_json, None)
+        # Unable to write to core state file (retries exhausted)
+
+        # json throws exception, but in a different area to test retries exhausted
+        import json
+        backup_json_load = json.load
+        json.load = None
+        self.runtime.env_layer.file_system.open = backup_open
+        core_sequence_json = self.assertRaises(Exception, self.lifecycle_manager.read_core_sequence)
         self.assertEquals(core_sequence_json, None)
+        json.load = backup_json_load
+        # Unable to read core state file (retries exhausted).
 
     def test_read_core_sequence_success(self):
         old_core_state_file_path = self.lifecycle_manager.core_state_file_path

src/core/tests/Test_PatchAssessor.py

@@ -46,7 +46,7 @@ def test_get_all_security_updates_fail(self):
     def test_assessment_fail_with_status_update(self):
         self.runtime.package_manager.refresh_repo = self.mock_refresh_repo
         self.runtime.set_legacy_test_type('UnalignedPath')
-        self.runtime.patch_assessor.start_assessment()
+        self.assertRaises(Exception, self.runtime.patch_assessor.start_assessment)
         with open(self.runtime.execution_config.status_file_path, 'r') as file_handle:
             file_contents = json.loads(file_handle.read())
             self.assertTrue('Unexpected return code (100) from package manager on command: LANG=en_US.UTF8 sudo apt-get -s dist-upgrade' in str(file_contents))

src/core/tests/Test_StatusHandler.py

@@ -248,6 +248,24 @@ def test_add_duplicate_error(self):
         self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"][0]["code"], Constants.PatchOperationErrorCodes.OPERATION_FAILED)
         self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"][1]["code"], Constants.PatchOperationErrorCodes.DEFAULT_ERROR)
 
+    def test_add_error_fail(self):
+        self.runtime.status_handler.set_current_operation(Constants.ASSESSMENT)
+
+        import tempfile
+        tempfile_backup = tempfile.NamedTemporaryFile
+        tempfile.NamedTemporaryFile = None
+
+        # Error within retries for writing temporary file
+        error_raised = False
+        try:
+            self.runtime.status_handler.add_error_to_status("test")
+        except Exception as error:
+            error_raised = True
+            self.assertTrue("retries exhausted" in str(error))
+
+        self.assertTrue(error_raised)
+        tempfile.NamedTemporaryFile = tempfile_backup
+
     def test_status_file_initial_load(self):
         # for non autopatching request, with Reboot started
         self.runtime.status_handler.set_installation_reboot_status(Constants.RebootStatus.STARTED)
@@ -274,6 +292,19 @@ def test_status_file_initial_load(self):
         self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["patchVersion"], Constants.PATCH_VERSION_UNKNOWN)
         self.assertEqual(substatus_file_data["status"].lower(), Constants.STATUS_SUCCESS.lower())
 
+        # fail to load status file
+        self.runtime.status_handler.set_installation_reboot_status(Constants.RebootStatus.STARTED)
+        backup_file_system_open = self.runtime.env_layer.file_system.open
+        self.runtime.env_layer.file_system.open = None
+        status_handler_failed = False
+        try:
+            status_handler = StatusHandler(self.runtime.env_layer, self.runtime.execution_config, self.runtime.composite_logger, self.runtime.telemetry_writer, self.runtime.vm_cloud_type)
+        except Exception as error:
+            status_handler_failed = True
+
+        self.assertTrue(status_handler_failed)
+        self.runtime.env_layer.file_system.open = backup_file_system_open
+
     def test_set_patch_metadata_for_healthstore_substatus_json(self):
         # setting healthstore properties
         self.runtime.status_handler.set_patch_metadata_for_healthstore_substatus_json(status=Constants.STATUS_SUCCESS, patch_version="2020-07-08", report_to_healthstore=True, wait_after_update=True)

src/core/tests/Test_ZypperPackageManager.py

@@ -16,6 +16,8 @@
 import json
 import os
 import unittest
+import tempfile
+import shutil
 from core.src.bootstrap.Constants import Constants
 from core.tests.library.ArgumentComposer import ArgumentComposer
 from core.tests.library.RuntimeCompositor import RuntimeCompositor
@@ -283,6 +285,20 @@ def test_get_process_tree_from_package_manager_output_failure_cmd_empty_output(s
     def test_env_var_set_get(self):
         zypp_lock_timeout_var_name = "ZYPP_LOCK_TIMEOUT"
 
+        self.temp_dir = tempfile.mkdtemp()
+        self.temp_env_file = self.temp_dir + "/" + "mockEnv"
+        open(self.temp_env_file, 'w+').close() # create temp file
+        self.runtime.env_layer.etc_environment_file_path = self.temp_env_file
+        write_with_retry_backup = self.runtime.env_layer.file_system.write_with_retry
+
+        def write_with_retry_fail(file_path_or_handle, data, mode='a+'):
+            self.runtime.env_layer.file_system.open = lambda file_path, mode, raise_if_not_found: None
+            write_with_retry_backup(file_path_or_handle, data, mode='a+')
+
+        self.runtime.env_layer.file_system.write_with_retry = write_with_retry_fail
+        self.assertRaises(Exception, self.runtime.env_layer.set_env_var(zypp_lock_timeout_var_name, "5"))
+        self.runtime.env_layer.file_system.write_with_retry = write_with_retry_backup
+
         # normal case without preexisting var
         self.runtime.env_layer.file_system.read_with_retry = self.mock_read_with_retry_not_has_zypper_lock_var
         self.runtime.env_layer.file_system.write_with_retry = self.mock_write_with_retry_valid
@@ -339,6 +355,8 @@ def test_env_var_set_get(self):
         self.assertEqual(self.runtime.env_layer.get_env_var(zypp_lock_timeout_var_name), "=10")
         self.runtime.env_layer.set_env_var(zypp_lock_timeout_var_name, 5)
 
+        shutil.rmtree(self.temp_dir)
+
     def test_disable_auto_os_updates_with_uninstalled_services(self):
         # no services are installed on the machine. expected o/p: function will complete successfully. Backup file will be created with default values, no auto OS update configuration settings will be updated as there are none
         self.runtime.set_legacy_test_type('SadPath')

src/extension/src/EnvLayer.py

@@ -202,7 +202,7 @@ def open(self, file_path, mode):
                 try:
                     return open(file_path, mode)
                 except Exception as error:
-                    if i < self.retry_count:
+                    if i < self.retry_count - 1:
                         time.sleep(i + 1)
                     else:
                         raise Exception("Unable to open {0} (retries exhausted). Error: {1}.".format(str(file_path), repr(error)))
@@ -236,7 +236,7 @@ def write_with_retry(self, file_path_or_handle, data, mode='a+'):
                     file_handle.write(str(data))
                     break
                 except Exception as error:
-                    if i < self.retry_count:
+                    if i < self.retry_count - 1:
                         time.sleep(i + 1)
                     else:
                         raise Exception("Unable to write to {0} (retries exhausted). Error: {1}.".format(str(file_handle.name), repr(error)))

src/extension/tests/Test_EnvHealthManager.py

@@ -202,6 +202,26 @@ def test_ensure_tty_not_required_when_tty_set_to_required_in_default_sudoers_and
         # wrap up
         self.__wrap_up_ensure_tty_not_required_test(backup_etc_sudoers_file_path, backup_etc_sudoers_linux_patch_extension_file_path)
 
+    def test_read_with_retry_fail(self):
+        open = None
+        has_error = False
+        try:
+            self.env_layer.file_system.read_with_retry(self.env_layer.etc_sudoers_linux_patch_extension_file_path)
+        except Exception as error:
+            has_error = True
+
+        self.assertTrue(has_error)
+
+    def test_write_with_retry_fail(self):
+        self.env_layer.file_system.open = lambda file_path, mode: None
+        has_error = False
+        try:
+            self.env_layer.file_system.write_with_retry(self.env_layer.etc_sudoers_linux_patch_extension_file_path, "test")
+        except Exception as error:
+            has_error = True
+
+        self.assertTrue(has_error)
+
     def __ensure_tty_not_required_test_setup(self):
         mock_sudoers_file_path = os.path.join(self.temp_dir, "etc-sudoers")
         backup_etc_sudoers_file_path = self.env_layer.etc_sudoers_file_path