From 6a77f271697d3699728093c2bb52ed22fa54746c Mon Sep 17 00:00:00 2001 From: David Keymer Date: Thu, 16 Oct 2025 15:40:41 +0100 Subject: [PATCH 1/7] Fixed typing --- .../ibex_install_utils/install_tasks.py | 2 +- .../ibex_install_utils/motor_params.py | 9 +-- .../ibex_install_utils/tasks/__init__.py | 2 +- .../ibex_install_utils/tasks/git_tasks.py | 49 ++++++++++++ .../ibex_install_utils/tasks/server_tasks.py | 76 ++++++------------- 5 files changed, 77 insertions(+), 61 deletions(-) diff --git a/installation_and_upgrade/ibex_install_utils/install_tasks.py b/installation_and_upgrade/ibex_install_utils/install_tasks.py index 3c5da87f..29942316 100644 --- a/installation_and_upgrade/ibex_install_utils/install_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/install_tasks.py @@ -243,10 +243,10 @@ def run_instrument_deploy_main(self) -> None: self._python_tasks.install_genie_python3() self._mysql_tasks.install_mysql() self._system_tasks.install_or_upgrade_vc_redist() - self._client_tasks.install_ibex_client() self._git_tasks.checkout_to_release_branch() self._server_tasks.upgrade_instrument_configuration() + self._server_tasks.update_shared_scripts_repository() self._server_tasks.update_calibrations_repository() self._system_tasks.clear_or_reapply_hotfixes() self._python_tasks.update_script_definitions() diff --git a/installation_and_upgrade/ibex_install_utils/motor_params.py b/installation_and_upgrade/ibex_install_utils/motor_params.py index d8d112a9..af5858b9 100644 --- a/installation_and_upgrade/ibex_install_utils/motor_params.py +++ b/installation_and_upgrade/ibex_install_utils/motor_params.py @@ -5,9 +5,10 @@ """ import csv -from typing import BinaryIO +from typing import TextIO from aioca import CANothing, caget + from ibex_install_utils.ca_utils import get_machine_details_from_identifier VELOCITY_UNITS = "EGU per sec" @@ -111,14 +112,12 @@ } -async def get_params_and_save_to_file( - file_reference: BinaryIO, num_of_controllers: int = 8 -) -> None: +async def get_params_and_save_to_file(file_reference: TextIO, num_of_controllers: int = 8) -> None: """ Gets all the motor parameters and saves them to an open file reference as a csv. Args: - file_reference (BinaryIO): The csv file to save the data to. + file_reference (TextIO): The csv file to save the data to. num_of_controllers (int, optional): The number of motor controllers on the instrument """ list_of_axis_pvs = [] diff --git a/installation_and_upgrade/ibex_install_utils/tasks/__init__.py b/installation_and_upgrade/ibex_install_utils/tasks/__init__.py index 955333e1..4769120f 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/__init__.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/__init__.py @@ -39,7 +39,7 @@ def __init__( self._ca = CaWrapper() @staticmethod - def _get_machine_name(): + def _get_machine_name() -> str: """ Returns: The current machine name diff --git a/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py b/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py index d749fc9d..0543bc04 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py @@ -1,9 +1,14 @@ import re import subprocess +import git +from git import PathLike + +from ibex_install_utils.exceptions import ErrorInTask from ibex_install_utils.task import task from ibex_install_utils.tasks import BaseTasks from ibex_install_utils.tasks.common_paths import EPICS_PATH +from ibex_install_utils.user_prompt import UserPrompt class GitTasks(BaseTasks): @@ -86,3 +91,47 @@ def checkout_to_release_branch(self) -> None: except subprocess.CalledProcessError as e: print(f"Error checking out to new release branch and push: {e}") print("Branch may previously exist either locally or remotely - intervention required") + + +def _try_to_merge_master_into_repo( + prompt: UserPrompt, repo_path: str | PathLike | None, pull_first: bool = False +) -> None: + manual_prompt = ( + "Merge the master branch into the local branch." + f"From {repo_path} run:\n" + " 0. Clean up any in progress merge (e.g. git merge --abort)\n" + " 1. git checkout master\n" + " 2. git pull\n" + " 3. git checkout [machine name]\n" + " 4. git merge master\n" + " 5. Resolve any merge conflicts\n" + " 6. git push\n" + ) + automatic_prompt = "Attempt automatic branch merge?" + if prompt.confirm_step(automatic_prompt): + try: + repo = git.Repo(repo_path) + if pull_first: + repo.git.pull() + if repo.active_branch.name != BaseTasks._get_machine_name(): + print( + f"Git branch, '{repo.active_branch}', is not the same as" + f" machine name ,'{BaseTasks._get_machine_name()}' " + ) + raise ErrorInTask("Git branch is not the same as machine name") + try: + print(f" fetch: {repo.git.fetch()}") + print(f" merge: {repo.git.merge('origin/master')}") + except git.GitCommandError as e: + # do gc and prune to remove issues with stale references + # this does a pack that takes a while, hence not do every time + print(f"Retrying git operations after a prune due to {e}") + print(f" gc: {repo.git.gc(prune='now')}") + print(f" prune: {repo.git.remote('prune', 'origin')}") + print(f" fetch: {repo.git.fetch()}") + print(f" merge: {repo.git.merge('origin/master')}") + except git.GitCommandError as e: + print(f"Error doing automatic merge, please perform the merge manually: {e}") + prompt.prompt_and_raise_if_not_yes(manual_prompt) + else: + prompt.prompt_and_raise_if_not_yes(manual_prompt) diff --git a/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py b/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py index 3105c95a..30c39a61 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py @@ -6,7 +6,8 @@ import shutil import subprocess import tempfile -from typing import Generator, LiteralString, TextIO +from pathlib import Path +from typing import Generator, TextIO import lxml.etree @@ -16,8 +17,9 @@ from contextlib2 import contextmanager import git + from ibex_install_utils.admin_runner import AdminCommandBuilder -from ibex_install_utils.exceptions import ErrorInRun, ErrorInTask +from ibex_install_utils.exceptions import ErrorInRun from ibex_install_utils.file_utils import LABVIEW_DAE_DIR, FileUtils from ibex_install_utils.motor_params import get_params_and_save_to_file from ibex_install_utils.run_process import RunProcess @@ -32,6 +34,7 @@ SETTINGS_CONFIG_PATH, VAR_DIR, ) +from ibex_install_utils.tasks.git_tasks import _try_to_merge_master_into_repo CONFIG_UPGRADE_SCRIPT_DIR = os.path.join(EPICS_PATH, "misc", "upgrade", "master") @@ -73,14 +76,12 @@ class ServerTasks(BaseTasks): its associated configuration files.""" @staticmethod - def _get_config_path() -> LiteralString | str | bytes: + def _get_config_path() -> os.PathLike[str]: """Returns: The path to the instrument's configurations directory """ - return os.path.join( - INSTRUMENT_BASE_DIR, SETTINGS_CONFIG_FOLDER, ServerTasks._get_machine_name() - ) + return Path(INSTRUMENT_BASE_DIR, SETTINGS_CONFIG_FOLDER, BaseTasks._get_machine_name()) @task("Removing old settings file") def remove_settings(self) -> None: @@ -97,7 +98,7 @@ def install_settings(self) -> None: """ self._file_utils.mkdir_recursive(SETTINGS_CONFIG_PATH) - settings_path = os.path.join(SETTINGS_CONFIG_PATH, ServerTasks._get_machine_name()) + settings_path = os.path.join(SETTINGS_CONFIG_PATH, BaseTasks._get_machine_name()) shutil.copytree(SOURCE_MACHINE_SETTINGS_CONFIG_PATH, settings_path) @@ -119,7 +120,7 @@ def install_settings(self) -> None: ) @task("Installing IBEX Server") - def install_ibex_server(self, use_old_galil: bool = None) -> None: + def install_ibex_server(self, use_old_galil: bool | None = None) -> None: """Install ibex server. Args: @@ -248,45 +249,11 @@ def setup_config_repository(self) -> None: @task("Upgrading instrument configuration") def upgrade_instrument_configuration(self) -> None: - """Update the configuration on the instrument using its upgrade config script.""" - manual_prompt = ( - "Merge the master configurations branch into the instrument configuration. " - "From C:\\Instrument\\Settings\\config\\[machine name] run:\n" - " 0. Clean up any in progress merge (e.g. git merge --abort)\n" - " 1. git checkout master\n" - " 2. git pull\n" - " 3. git checkout [machine name]\n" - " 4. git merge master\n" - " 5. Resolve any merge conflicts\n" - " 6. git push\n" - ) - automatic_prompt = "Attempt automatic configuration merge?" - if self.prompt.confirm_step(automatic_prompt): - try: - repo = git.Repo(os.path.join(SETTINGS_CONFIG_PATH, BaseTasks._get_machine_name())) - if repo.active_branch.name != BaseTasks._get_machine_name(): - print( - f"Git branch, '{repo.active_branch}', is not the same as" - f" machine name ,'{BaseTasks._get_machine_name()}' " - ) - raise ErrorInTask("Git branch is not the same as machine name") - try: - print(f" fetch: {repo.git.fetch()}") - print(f" merge: {repo.git.merge('origin/master')}") - except git.GitCommandError as e: - # do gc and prune to remove issues with stale references - # this does a pack that takes a while, hence not do every time - print(f"Retrying git operations after a prune due to {e}") - print(f" gc: {repo.git.gc(prune='now')}") - print(f" prune: {repo.git.remote('prune', 'origin')}") - print(f" fetch: {repo.git.fetch()}") - print(f" merge: {repo.git.merge('origin/master')}") - # no longer push let the instrument do that on start up if needed - except git.GitCommandError as e: - print(f"Error doing automatic merge, please perform the merge manually: {e}") - self.prompt.prompt_and_raise_if_not_yes(manual_prompt) - else: - self.prompt.prompt_and_raise_if_not_yes(manual_prompt) + """Update the configuration on the instrument by attempting to merge the master branch + and running its upgrade config script.""" + + repo_path = os.path.join(SETTINGS_CONFIG_PATH, BaseTasks._get_machine_name()) + _try_to_merge_master_into_repo(self.prompt, repo_path) RunProcess(CONFIG_UPGRADE_SCRIPT_DIR, "upgrade.bat", capture_pipes=False).run() @@ -312,12 +279,11 @@ def install_shared_scripts_repository(self) -> None: prog_args=["clone", repo_url, INST_SCRIPTS_PATH], ).run() - @task("Set up shared instrument scripts library") + @task("Merge master into local shared instrument scripts library") def update_shared_scripts_repository(self) -> None: """Update the shared instrument scripts repository containing""" try: - repo = git.Repo(INST_SCRIPTS_PATH) - repo.git.pull() + _try_to_merge_master_into_repo(self.prompt, INST_SCRIPTS_PATH, True) except git.GitCommandError: self.prompt.prompt_and_raise_if_not_yes( "There was an error pulling the shared scripts repo.\n" @@ -376,13 +342,13 @@ def perform_server_tests(self) -> None: print( f"Checking that configurations are being pushed to" f" the appropriate repository " - f"({INSTCONFIGS_GIT_URL.format(ServerTasks._get_machine_name())})" + f"({INSTCONFIGS_GIT_URL.format(BaseTasks._get_machine_name())})" ) repo = git.Repo(self._get_config_path()) repo.git.fetch() status = repo.git.status() print(f"Current repository status is: {status}") - if f"up to date with 'origin/{self._get_machine_name()}'" in status: + if f"up to date with 'origin/{BaseTasks._get_machine_name()}'" in status: print("Configurations updating correctly") else: self.prompt.prompt_and_raise_if_not_yes( @@ -390,7 +356,7 @@ def perform_server_tests(self) -> None: f"not attached to correct branch. " f"Please confirm that configurations are being pushed to the appropriate " f"remote repository branch" - f" ({INSTCONFIGS_GIT_URL.format(ServerTasks._get_machine_name())})" + f" ({INSTCONFIGS_GIT_URL.format(BaseTasks._get_machine_name())})" ) self.prompt.prompt_and_raise_if_not_yes( @@ -555,7 +521,9 @@ def _pretty_print(data: str | None) -> str: with self.timestamped_pv_backups_file(directory="inst_servers", name=name) as f: try: - f.write(f"{_pretty_print(self._ca.get_object_from_compressed_hexed_json(pv))}\r\n") + pvs = self._ca.get_object_from_compressed_hexed_json(pv) + if pvs is not None: + f.write(f"{_pretty_print(data=pvs.decode('utf-8'))}\r\n") except: # noqa: E722 pass From 4da0000e7e4f247ea549a262d43c9cfa1a60ff18 Mon Sep 17 00:00:00 2001 From: David Keymer Date: Thu, 16 Oct 2025 15:50:35 +0100 Subject: [PATCH 2/7] Fixed typing --- .../ibex_install_utils/tasks/__init__.py | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/installation_and_upgrade/ibex_install_utils/tasks/__init__.py b/installation_and_upgrade/ibex_install_utils/tasks/__init__.py index 4769120f..dd4c621e 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/__init__.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/__init__.py @@ -5,6 +5,7 @@ from ibex_install_utils.ca_utils import CaWrapper from ibex_install_utils.file_utils import FileUtils from ibex_install_utils.tasks.common_paths import BACKUP_DATA_DIR, BACKUP_DIR +from ibex_install_utils.user_prompt import UserPrompt class BaseTasks: @@ -12,17 +13,17 @@ class BaseTasks: def __init__( self, - user_prompt, - server_source_dir, - client_source_dir, - genie_python3_dir, - ibex_version, - file_utils=FileUtils(), - ): + user_prompt: UserPrompt, + server_source_dir: str, + client_source_dir: str, + genie_python3_dir: str, + ibex_version: str, + file_utils: FileUtils = FileUtils(), + ) -> None: """ Initializer. Args: - user_prompt (ibex_install_utils.user_prompt.UserPrompt): a object to allow prompting of the user + user_prompt (UserPrompt): an object to allow prompting of the user server_source_dir: directory to install ibex server from client_source_dir: directory to install ibex client from genie_python3_dir: directory to install genie python from @@ -48,7 +49,7 @@ def _get_machine_name() -> str: return socket.gethostname() @staticmethod - def _get_instrument_name(): + def _get_instrument_name() -> str: """ Returns: The name of the current instrument @@ -56,15 +57,15 @@ def _get_instrument_name(): return BaseTasks._get_machine_name().replace("NDX", "") @staticmethod - def _today_date_for_filenames(): + def _today_date_for_filenames() -> str: return date.today().strftime("%Y_%m_%d") @staticmethod - def _generate_backup_dir_name(): + def _generate_backup_dir_name() -> str: return f"ibex_backup_{BaseTasks._today_date_for_filenames()}" @staticmethod - def _get_backup_dir(): + def _get_backup_dir() -> str: """ The backup directory contains the date of backup, if this script is running over multiple days this will return the date this method was first called. @@ -79,7 +80,8 @@ def _get_backup_dir(): new_backup_dir = os.path.join(BACKUP_DIR, BaseTasks._generate_backup_dir_name()) if not os.path.exists(BACKUP_DATA_DIR): - # data dir is a linked directory on real instrument machines so can't just simply be created with mkdir + # data dir is a linked directory on real instrument machines + # so can't just simply be created with mkdir raise IOError( f"Base directory does not exist {BACKUP_DATA_DIR} should be a provided linked dir" ) From 327968cb2ee18b180a0604ad81d6f93014c988c5 Mon Sep 17 00:00:00 2001 From: David Keymer Date: Thu, 16 Oct 2025 16:17:34 +0100 Subject: [PATCH 3/7] Made auto-merge function public --- .../ibex_install_utils/tasks/git_tasks.py | 2 +- .../ibex_install_utils/tasks/server_tasks.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py b/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py index 0543bc04..c286dbe3 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py @@ -93,7 +93,7 @@ def checkout_to_release_branch(self) -> None: print("Branch may previously exist either locally or remotely - intervention required") -def _try_to_merge_master_into_repo( +def try_to_merge_master_into_repo( prompt: UserPrompt, repo_path: str | PathLike | None, pull_first: bool = False ) -> None: manual_prompt = ( diff --git a/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py b/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py index 30c39a61..ca4b66d6 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py @@ -34,7 +34,7 @@ SETTINGS_CONFIG_PATH, VAR_DIR, ) -from ibex_install_utils.tasks.git_tasks import _try_to_merge_master_into_repo +from ibex_install_utils.tasks.git_tasks import try_to_merge_master_into_repo CONFIG_UPGRADE_SCRIPT_DIR = os.path.join(EPICS_PATH, "misc", "upgrade", "master") @@ -253,7 +253,7 @@ def upgrade_instrument_configuration(self) -> None: and running its upgrade config script.""" repo_path = os.path.join(SETTINGS_CONFIG_PATH, BaseTasks._get_machine_name()) - _try_to_merge_master_into_repo(self.prompt, repo_path) + try_to_merge_master_into_repo(self.prompt, repo_path) RunProcess(CONFIG_UPGRADE_SCRIPT_DIR, "upgrade.bat", capture_pipes=False).run() @@ -283,7 +283,7 @@ def install_shared_scripts_repository(self) -> None: def update_shared_scripts_repository(self) -> None: """Update the shared instrument scripts repository containing""" try: - _try_to_merge_master_into_repo(self.prompt, INST_SCRIPTS_PATH, True) + try_to_merge_master_into_repo(self.prompt, INST_SCRIPTS_PATH, True) except git.GitCommandError: self.prompt.prompt_and_raise_if_not_yes( "There was an error pulling the shared scripts repo.\n" From 214814f0c0de68c2604c7a26c859e15013409864 Mon Sep 17 00:00:00 2001 From: David Keymer Date: Thu, 16 Oct 2025 16:18:13 +0100 Subject: [PATCH 4/7] Made auto-merge function public --- installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py | 1 - .../ibex_install_utils/tasks/server_tasks.py | 1 - 2 files changed, 2 deletions(-) diff --git a/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py b/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py index c286dbe3..0d627f0b 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py @@ -3,7 +3,6 @@ import git from git import PathLike - from ibex_install_utils.exceptions import ErrorInTask from ibex_install_utils.task import task from ibex_install_utils.tasks import BaseTasks diff --git a/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py b/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py index ca4b66d6..9bc9bd3c 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py @@ -17,7 +17,6 @@ from contextlib2 import contextmanager import git - from ibex_install_utils.admin_runner import AdminCommandBuilder from ibex_install_utils.exceptions import ErrorInRun from ibex_install_utils.file_utils import LABVIEW_DAE_DIR, FileUtils From 2f74d61e6f007859d53331e51ed8907b3d59ca1b Mon Sep 17 00:00:00 2001 From: David Keymer Date: Mon, 20 Oct 2025 14:04:09 +0100 Subject: [PATCH 5/7] Fix Ruff checks --- installation_and_upgrade/ibex_install_utils/motor_params.py | 1 - 1 file changed, 1 deletion(-) diff --git a/installation_and_upgrade/ibex_install_utils/motor_params.py b/installation_and_upgrade/ibex_install_utils/motor_params.py index af5858b9..9f195f52 100644 --- a/installation_and_upgrade/ibex_install_utils/motor_params.py +++ b/installation_and_upgrade/ibex_install_utils/motor_params.py @@ -8,7 +8,6 @@ from typing import TextIO from aioca import CANothing, caget - from ibex_install_utils.ca_utils import get_machine_details_from_identifier VELOCITY_UNITS = "EGU per sec" From 436c71c92ec3f399c668bf3c13d71f69f50d1f3d Mon Sep 17 00:00:00 2001 From: David Keymer Date: Mon, 20 Oct 2025 14:12:41 +0100 Subject: [PATCH 6/7] clarified comment --- .../ibex_install_utils/tasks/server_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py b/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py index 9bc9bd3c..73e28e28 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py @@ -278,7 +278,7 @@ def install_shared_scripts_repository(self) -> None: prog_args=["clone", repo_url, INST_SCRIPTS_PATH], ).run() - @task("Merge master into local shared instrument scripts library") + @task("Merge master branch into local branch of shared instrument scripts library") def update_shared_scripts_repository(self) -> None: """Update the shared instrument scripts repository containing""" try: From e72ef8e6b96a9806c77e9fea8e2cec9043304510 Mon Sep 17 00:00:00 2001 From: David Keymer Date: Mon, 20 Oct 2025 17:09:20 +0100 Subject: [PATCH 7/7] complete function description --- .../ibex_install_utils/tasks/server_tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py b/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py index 73e28e28..f533ba60 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py @@ -280,7 +280,8 @@ def install_shared_scripts_repository(self) -> None: @task("Merge master branch into local branch of shared instrument scripts library") def update_shared_scripts_repository(self) -> None: - """Update the shared instrument scripts repository containing""" + """Update the shared instrument scripts repository + containing scripts useful across all instruments""" try: try_to_merge_master_into_repo(self.prompt, INST_SCRIPTS_PATH, True) except git.GitCommandError: