Ticket 7914 - pull latest instrument scripts master

installation_and_upgrade/ibex_install_utils/install_tasks.py

@@ -243,10 +243,10 @@ def run_instrument_deploy_main(self) -> None:
         self._python_tasks.install_genie_python3()
         self._mysql_tasks.install_mysql()
         self._system_tasks.install_or_upgrade_vc_redist()
-
         self._client_tasks.install_ibex_client()
         self._git_tasks.checkout_to_release_branch()
         self._server_tasks.upgrade_instrument_configuration()
+        self._server_tasks.update_shared_scripts_repository()
         self._server_tasks.update_calibrations_repository()
         self._system_tasks.clear_or_reapply_hotfixes()
         self._python_tasks.update_script_definitions()

installation_and_upgrade/ibex_install_utils/motor_params.py

@@ -5,7 +5,7 @@
 """
 
 import csv
-from typing import BinaryIO
+from typing import TextIO
 
 from aioca import CANothing, caget
 from ibex_install_utils.ca_utils import get_machine_details_from_identifier
@@ -111,14 +111,12 @@
 }
 
 
-async def get_params_and_save_to_file(
-    file_reference: BinaryIO, num_of_controllers: int = 8
-) -> None:
+async def get_params_and_save_to_file(file_reference: TextIO, num_of_controllers: int = 8) -> None:
     """
     Gets all the motor parameters and saves them to an open file reference as a csv.
 
     Args:
-        file_reference (BinaryIO): The csv file to save the data to.
+        file_reference (TextIO): The csv file to save the data to.
         num_of_controllers (int, optional): The number of motor controllers on the instrument
     """
     list_of_axis_pvs = []

installation_and_upgrade/ibex_install_utils/tasks/__init__.py

@@ -5,24 +5,25 @@
 from ibex_install_utils.ca_utils import CaWrapper
 from ibex_install_utils.file_utils import FileUtils
 from ibex_install_utils.tasks.common_paths import BACKUP_DATA_DIR, BACKUP_DIR
+from ibex_install_utils.user_prompt import UserPrompt
 
 
 class BaseTasks:
     _backup_dir = None
 
     def __init__(
         self,
-        user_prompt,
-        server_source_dir,
-        client_source_dir,
-        genie_python3_dir,
-        ibex_version,
-        file_utils=FileUtils(),
-    ):
+        user_prompt: UserPrompt,
+        server_source_dir: str,
+        client_source_dir: str,
+        genie_python3_dir: str,
+        ibex_version: str,
+        file_utils: FileUtils = FileUtils(),
+    ) -> None:
         """
         Initializer.
         Args:
-            user_prompt (ibex_install_utils.user_prompt.UserPrompt): a object to allow prompting of the user
+            user_prompt (UserPrompt): an object to allow prompting of the user
             server_source_dir: directory to install ibex server from
             client_source_dir: directory to install ibex client from
             genie_python3_dir: directory to install genie python from
@@ -39,7 +40,7 @@ def __init__(
         self._ca = CaWrapper()
 
     @staticmethod
-    def _get_machine_name():
+    def _get_machine_name() -> str:
         """
         Returns:
             The current machine name
@@ -48,23 +49,23 @@ def _get_machine_name():
         return socket.gethostname()
 
     @staticmethod
-    def _get_instrument_name():
+    def _get_instrument_name() -> str:
         """
         Returns:
             The name of the current instrument
         """
         return BaseTasks._get_machine_name().replace("NDX", "")
 
     @staticmethod
-    def _today_date_for_filenames():
+    def _today_date_for_filenames() -> str:
         return date.today().strftime("%Y_%m_%d")
 
     @staticmethod
-    def _generate_backup_dir_name():
+    def _generate_backup_dir_name() -> str:
         return f"ibex_backup_{BaseTasks._today_date_for_filenames()}"
 
     @staticmethod
-    def _get_backup_dir():
+    def _get_backup_dir() -> str:
         """
         The backup directory contains the date of backup, if this script is
         running over multiple days this will return the date this method was first called.
@@ -79,7 +80,8 @@ def _get_backup_dir():
         new_backup_dir = os.path.join(BACKUP_DIR, BaseTasks._generate_backup_dir_name())
 
         if not os.path.exists(BACKUP_DATA_DIR):
-            # data dir is a linked directory on real instrument machines so can't just simply be created with mkdir
+            # data dir is a linked directory on real instrument machines
+            # so can't just simply be created with mkdir
             raise IOError(
                 f"Base directory does not exist {BACKUP_DATA_DIR} should be a provided linked dir"
             )

installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py

@@ -1,9 +1,13 @@
 import re
 import subprocess
 
+import git
+from git import PathLike
+from ibex_install_utils.exceptions import ErrorInTask
 from ibex_install_utils.task import task
 from ibex_install_utils.tasks import BaseTasks
 from ibex_install_utils.tasks.common_paths import EPICS_PATH
+from ibex_install_utils.user_prompt import UserPrompt
 
 
 class GitTasks(BaseTasks):
@@ -86,3 +90,47 @@ def checkout_to_release_branch(self) -> None:
         except subprocess.CalledProcessError as e:
             print(f"Error checking out to new release branch and push: {e}")
             print("Branch may previously exist either locally or remotely - intervention required")
+
+
+def try_to_merge_master_into_repo(
+    prompt: UserPrompt, repo_path: str | PathLike | None, pull_first: bool = False
+) -> None:
+    manual_prompt = (
+        "Merge the master branch into the local branch."
+        f"From {repo_path} run:\n"
+        "    0. Clean up any in progress merge (e.g. git merge --abort)\n"
+        "    1. git checkout master\n"
+        "    2. git pull\n"
+        "    3. git checkout [machine name]\n"
+        "    4. git merge master\n"
+        "    5. Resolve any merge conflicts\n"
+        "    6. git push\n"
+    )
+    automatic_prompt = "Attempt automatic branch merge?"
+    if prompt.confirm_step(automatic_prompt):
+        try:
+            repo = git.Repo(repo_path)
+            if pull_first:
+                repo.git.pull()
+            if repo.active_branch.name != BaseTasks._get_machine_name():
+                print(
+                    f"Git branch, '{repo.active_branch}', is not the same as"
+                    f" machine name ,'{BaseTasks._get_machine_name()}' "
+                )
+                raise ErrorInTask("Git branch is not the same as machine name")
+            try:
+                print(f"     fetch: {repo.git.fetch()}")
+                print(f"     merge: {repo.git.merge('origin/master')}")
+            except git.GitCommandError as e:
+                # do gc and prune to remove issues with stale references
+                # this does a pack that takes a while, hence not do every time
+                print(f"Retrying git operations after a prune due to {e}")
+                print(f"        gc: {repo.git.gc(prune='now')}")
+                print(f"     prune: {repo.git.remote('prune', 'origin')}")
+                print(f"     fetch: {repo.git.fetch()}")
+                print(f"     merge: {repo.git.merge('origin/master')}")
+        except git.GitCommandError as e:
+            print(f"Error doing automatic merge, please perform the merge manually: {e}")
+            prompt.prompt_and_raise_if_not_yes(manual_prompt)
+    else:
+        prompt.prompt_and_raise_if_not_yes(manual_prompt)

installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py

@@ -6,7 +6,8 @@
 import shutil
 import subprocess
 import tempfile
-from typing import Generator, LiteralString, TextIO
+from pathlib import Path
+from typing import Generator, TextIO
 
 import lxml.etree
 
@@ -17,7 +18,7 @@
 
 import git
 from ibex_install_utils.admin_runner import AdminCommandBuilder
-from ibex_install_utils.exceptions import ErrorInRun, ErrorInTask
+from ibex_install_utils.exceptions import ErrorInRun
 from ibex_install_utils.file_utils import LABVIEW_DAE_DIR, FileUtils
 from ibex_install_utils.motor_params import get_params_and_save_to_file
 from ibex_install_utils.run_process import RunProcess
@@ -32,6 +33,7 @@
     SETTINGS_CONFIG_PATH,
     VAR_DIR,
 )
+from ibex_install_utils.tasks.git_tasks import try_to_merge_master_into_repo
 
 CONFIG_UPGRADE_SCRIPT_DIR = os.path.join(EPICS_PATH, "misc", "upgrade", "master")
 
@@ -73,14 +75,12 @@ class ServerTasks(BaseTasks):
     its associated configuration files."""
 
     @staticmethod
-    def _get_config_path() -> LiteralString | str | bytes:
+    def _get_config_path() -> os.PathLike[str]:
         """Returns:
         The path to the instrument's configurations directory
 
         """
-        return os.path.join(
-            INSTRUMENT_BASE_DIR, SETTINGS_CONFIG_FOLDER, ServerTasks._get_machine_name()
-        )
+        return Path(INSTRUMENT_BASE_DIR, SETTINGS_CONFIG_FOLDER, BaseTasks._get_machine_name())
 
     @task("Removing old settings file")
     def remove_settings(self) -> None:
@@ -97,7 +97,7 @@ def install_settings(self) -> None:
 
         """
         self._file_utils.mkdir_recursive(SETTINGS_CONFIG_PATH)
-        settings_path = os.path.join(SETTINGS_CONFIG_PATH, ServerTasks._get_machine_name())
+        settings_path = os.path.join(SETTINGS_CONFIG_PATH, BaseTasks._get_machine_name())
 
         shutil.copytree(SOURCE_MACHINE_SETTINGS_CONFIG_PATH, settings_path)
 
@@ -119,7 +119,7 @@ def install_settings(self) -> None:
         )
 
     @task("Installing IBEX Server")
-    def install_ibex_server(self, use_old_galil: bool = None) -> None:
+    def install_ibex_server(self, use_old_galil: bool | None = None) -> None:
         """Install ibex server.
 
         Args:
@@ -248,45 +248,11 @@ def setup_config_repository(self) -> None:
 
     @task("Upgrading instrument configuration")
     def upgrade_instrument_configuration(self) -> None:
-        """Update the configuration on the instrument using its upgrade config script."""
-        manual_prompt = (
-            "Merge the master configurations branch into the instrument configuration. "
-            "From C:\\Instrument\\Settings\\config\\[machine name] run:\n"
-            "    0. Clean up any in progress merge (e.g. git merge --abort)\n"
-            "    1. git checkout master\n"
-            "    2. git pull\n"
-            "    3. git checkout [machine name]\n"
-            "    4. git merge master\n"
-            "    5. Resolve any merge conflicts\n"
-            "    6. git push\n"
-        )
-        automatic_prompt = "Attempt automatic configuration merge?"
-        if self.prompt.confirm_step(automatic_prompt):
-            try:
-                repo = git.Repo(os.path.join(SETTINGS_CONFIG_PATH, BaseTasks._get_machine_name()))
-                if repo.active_branch.name != BaseTasks._get_machine_name():
-                    print(
-                        f"Git branch, '{repo.active_branch}', is not the same as"
-                        f" machine name ,'{BaseTasks._get_machine_name()}' "
-                    )
-                    raise ErrorInTask("Git branch is not the same as machine name")
-                try:
-                    print(f"     fetch: {repo.git.fetch()}")
-                    print(f"     merge: {repo.git.merge('origin/master')}")
-                except git.GitCommandError as e:
-                    # do gc and prune to remove issues with stale references
-                    # this does a pack that takes a while, hence not do every time
-                    print(f"Retrying git operations after a prune due to {e}")
-                    print(f"        gc: {repo.git.gc(prune='now')}")
-                    print(f"     prune: {repo.git.remote('prune', 'origin')}")
-                    print(f"     fetch: {repo.git.fetch()}")
-                    print(f"     merge: {repo.git.merge('origin/master')}")
-                # no longer push let the instrument do that on start up if needed
-            except git.GitCommandError as e:
-                print(f"Error doing automatic merge, please perform the merge manually: {e}")
-                self.prompt.prompt_and_raise_if_not_yes(manual_prompt)
-        else:
-            self.prompt.prompt_and_raise_if_not_yes(manual_prompt)
+        """Update the configuration on the instrument by attempting to merge the master branch
+        and running its upgrade config script."""
+
+        repo_path = os.path.join(SETTINGS_CONFIG_PATH, BaseTasks._get_machine_name())
+        try_to_merge_master_into_repo(self.prompt, repo_path)
 
         RunProcess(CONFIG_UPGRADE_SCRIPT_DIR, "upgrade.bat", capture_pipes=False).run()
 
@@ -312,12 +278,12 @@ def install_shared_scripts_repository(self) -> None:
                 prog_args=["clone", repo_url, INST_SCRIPTS_PATH],
             ).run()
 
-    @task("Set up shared instrument scripts library")
+    @task("Merge master branch into local branch of shared instrument scripts library")
     def update_shared_scripts_repository(self) -> None:
-        """Update the shared instrument scripts repository containing"""
+        """Update the shared instrument scripts repository
+        containing scripts useful across all instruments"""
         try:
-            repo = git.Repo(INST_SCRIPTS_PATH)
-            repo.git.pull()
+            try_to_merge_master_into_repo(self.prompt, INST_SCRIPTS_PATH, True)
         except git.GitCommandError:
             self.prompt.prompt_and_raise_if_not_yes(
                 "There was an error pulling the shared scripts repo.\n"
@@ -376,21 +342,21 @@ def perform_server_tests(self) -> None:
         print(
             f"Checking that configurations are being pushed to"
             f" the appropriate repository "
-            f"({INSTCONFIGS_GIT_URL.format(ServerTasks._get_machine_name())})"
+            f"({INSTCONFIGS_GIT_URL.format(BaseTasks._get_machine_name())})"
         )
         repo = git.Repo(self._get_config_path())
         repo.git.fetch()
         status = repo.git.status()
         print(f"Current repository status is: {status}")
-        if f"up to date with 'origin/{self._get_machine_name()}'" in status:
+        if f"up to date with 'origin/{BaseTasks._get_machine_name()}'" in status:
             print("Configurations updating correctly")
         else:
             self.prompt.prompt_and_raise_if_not_yes(
                 f"Repository status shown above is either not 'up to date' or "
                 f"not attached to correct branch. "
                 f"Please confirm that configurations are being pushed to the appropriate "
                 f"remote repository branch"
-                f" ({INSTCONFIGS_GIT_URL.format(ServerTasks._get_machine_name())})"
+                f" ({INSTCONFIGS_GIT_URL.format(BaseTasks._get_machine_name())})"
             )
 
         self.prompt.prompt_and_raise_if_not_yes(
@@ -555,7 +521,9 @@ def _pretty_print(data: str | None) -> str:
 
         with self.timestamped_pv_backups_file(directory="inst_servers", name=name) as f:
             try:
-                f.write(f"{_pretty_print(self._ca.get_object_from_compressed_hexed_json(pv))}\r\n")
+                pvs = self._ca.get_object_from_compressed_hexed_json(pv)
+                if pvs is not None:
+                    f.write(f"{_pretty_print(data=pvs.decode('utf-8'))}\r\n")
             except:  # noqa: E722
                 pass