From e1f70683a85e7cff7c7ade24bcde5286400a6b16 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Sat, 22 Nov 2025 13:21:46 +0100 Subject: [PATCH] Split sync ironic and implement sync netbox with URL filtering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split the bidirectional sync ironic command into two separate commands: - sync ironic: NetBox → Ironic synchronization (unchanged direction) - sync netbox: Ironic → NetBox synchronization (new command) The sync ironic command now only synchronizes from NetBox to Ironic, creating/updating/deleting Ironic nodes based on NetBox devices. The previously embedded Ironic → NetBox state updates have been removed and moved to the new sync netbox command. Implement sync netbox command to synchronize Ironic node states to NetBox: - Updates provision_state, power_state, and maintenance custom fields - Syncs to primary NetBox and all configured secondary instances - Supports optional node_name parameter to sync specific nodes - Add --netbox-filter parameter for selective NetBox instance updates - Filter uses case-insensitive substring matching on base_url The URL filter enables targeted synchronization scenarios: - Sync only to primary NetBox: --netbox-filter primary - Sync only to specific secondary: --netbox-filter secondary-1 - Sync to environment-specific instances: --netbox-filter prod This separation allows independent execution of each sync direction and provides fine-grained control over which NetBox instances to update. AI-assisted: Claude Code Signed-off-by: Christian Berendt --- osism/commands/netbox.py | 56 +++++++++++++++++++++--- osism/tasks/conductor/__init__.py | 7 +-- osism/tasks/conductor/ironic.py | 72 ++++++++++++++++++++++++++++--- osism/tasks/netbox.py | 55 ++++++++++++++++++++--- 4 files changed, 170 insertions(+), 20 deletions(-) diff --git a/osism/commands/netbox.py b/osism/commands/netbox.py index ddd3124a..c73e2caf 100644 --- a/osism/commands/netbox.py +++ b/osism/commands/netbox.py @@ -83,11 +83,28 @@ def take_action(self, parsed_args): class Sync(Command): def get_parser(self, prog_name): parser = super(Sync, self).get_parser(prog_name) + parser.add_argument( + "node", + nargs="?", + help="Optional node name to sync only a specific node", + ) parser.add_argument( "--no-wait", help="Do not wait until the sync has been completed", action="store_true", ) + parser.add_argument( + "--task-timeout", + default=os.environ.get("OSISM_TASK_TIMEOUT", 300), + type=int, + help="Timeout for a scheduled task that has not been executed yet", + ) + parser.add_argument( + "--netbox-filter", + type=str, + default=None, + help="Filter NetBox instances by URL (substring match, e.g. 'primary' or 'secondary-1')", + ) return parser def take_action(self, parsed_args): @@ -95,13 +112,42 @@ def take_action(self, parsed_args): utils.check_task_lock_and_exit() wait = not parsed_args.no_wait + task_timeout = parsed_args.task_timeout + node_name = parsed_args.node + netbox_filter = parsed_args.netbox_filter - task = conductor.sync_netbox.delay() + task = conductor.sync_netbox.delay( + node_name=node_name, netbox_filter=netbox_filter + ) if wait: - logger.info( - f"Task {task.task_id} (sync netbox) is running. Wait. No more output." - ) - task.wait(timeout=None, interval=0.5) + if node_name: + logger.info( + f"Task {task.task_id} (sync netbox for node {node_name}) is running in background. Output comming soon." + ) + else: + logger.info( + f"Task {task.task_id} (sync netbox) is running in background. Output comming soon." + ) + try: + return utils.fetch_task_output(task.id, timeout=task_timeout) + except TimeoutError: + if node_name: + logger.error( + f"Timeout while waiting for further output of task {task.task_id} (sync netbox for node {node_name})" + ) + else: + logger.error( + f"Timeout while waiting for further output of task {task.task_id} (sync netbox)" + ) + else: + if node_name: + logger.info( + f"Task {task.task_id} (sync netbox for node {node_name}) is running in background. No more output." + ) + else: + logger.info( + f"Task {task.task_id} (sync netbox) is running in background. No more output." + ) class Manage(Command): diff --git a/osism/tasks/conductor/__init__.py b/osism/tasks/conductor/__init__.py index 534be95b..6a5f759d 100644 --- a/osism/tasks/conductor/__init__.py +++ b/osism/tasks/conductor/__init__.py @@ -3,7 +3,6 @@ import copy from celery import Celery from celery.signals import worker_process_init -from loguru import logger from osism import utils from osism.tasks import Config @@ -40,11 +39,13 @@ def get_ironic_parameters(self): @app.task(bind=True, name="osism.tasks.conductor.sync_netbox") -def sync_netbox(self, force_update=False): +def sync_netbox(self, node_name=None, netbox_filter=None): # Check if tasks are locked before execution utils.check_task_lock_and_exit() - logger.info("Not implemented") + from osism.tasks.conductor.ironic import sync_netbox_from_ironic + + sync_netbox_from_ironic(self.request.id, node_name, netbox_filter) @app.task(bind=True, name="osism.tasks.conductor.sync_ironic") diff --git a/osism/tasks/conductor/ironic.py b/osism/tasks/conductor/ironic.py index 8039f05d..eb5633e3 100644 --- a/osism/tasks/conductor/ironic.py +++ b/osism/tasks/conductor/ironic.py @@ -232,11 +232,6 @@ def sync_ironic(request_id, get_ironic_parameters, node_name=None, force_update= ) node = openstack.baremetal_node_create(device.name, node_attributes) else: - # NOTE: The listener service only reacts to changes in the baremetal node. Explicitly sync provision, power state and maintenance in case updates were missed by the listener. - # This sync is done unconditionally, because we do not know the state of secondary netboxes at this point - netbox.set_provision_state(device.name, node["provision_state"]) - netbox.set_power_state(device.name, node["power_state"]) - netbox.set_maintenance(device.name, state=node["is_maintenance"]) # NOTE: Check whether the baremetal node needs to be updated node_updates = {} deep_compare(node_attributes, node, node_updates) @@ -371,3 +366,70 @@ def sync_ironic(request_id, get_ironic_parameters, node_name=None, force_update= ) osism_utils.finish_task_output(request_id, rc=0) + + +def sync_netbox_from_ironic(request_id, node_name=None, netbox_filter=None): + """Sync Ironic node states to NetBox (including secondaries) + + This function synchronizes the state of Ironic baremetal nodes to NetBox. + It updates three custom fields in NetBox: + - provision_state: The current provision state of the node + - power_state: The current power state of the node + - maintenance: Whether the node is in maintenance mode + + The sync is performed for the primary NetBox instance and all configured + secondary NetBox instances. NetBox instances can be filtered by URL substring. + + Args: + request_id: The Celery task request ID for output tracking + node_name: Optional name of a specific node to sync. If None, all nodes are synced. + netbox_filter: Optional URL filter (substring match). If provided, only NetBox + instances whose base_url contains this substring will be updated. + Example: 'primary' matches 'https://primary-netbox.example.com' + """ + filter_msg = f" (NetBox filter: {netbox_filter})" if netbox_filter else "" + if node_name: + osism_utils.push_task_output( + request_id, + f"Starting Ironic to NetBox synchronisation for node {node_name}{filter_msg}\n", + ) + else: + osism_utils.push_task_output( + request_id, + f"Starting Ironic to NetBox synchronisation{filter_msg}\n", + ) + + # Get all Ironic nodes + nodes = openstack.baremetal_node_list() + + # Filter by node_name if specified + if node_name: + nodes = [n for n in nodes if n["name"] == node_name] + if not nodes: + osism_utils.push_task_output( + request_id, + f"Node {node_name} not found in Ironic\n", + ) + osism_utils.finish_task_output(request_id, rc=1) + return + + # Sync each node to NetBox (including secondaries) + for node in nodes: + osism_utils.push_task_output( + request_id, + f"Syncing state of {node['name']} to NetBox (including secondaries)\n", + ) + + # Update all three states (each function handles primary + secondary NetBox instances) + # Pass netbox_filter to only update matching NetBox instances + netbox.set_provision_state( + node["name"], node["provision_state"], netbox_filter=netbox_filter + ) + netbox.set_power_state( + node["name"], node["power_state"], netbox_filter=netbox_filter + ) + netbox.set_maintenance( + node["name"], state=node["is_maintenance"], netbox_filter=netbox_filter + ) + + osism_utils.finish_task_output(request_id, rc=0) diff --git a/osism/tasks/netbox.py b/osism/tasks/netbox.py index 4700a91e..acd1362b 100644 --- a/osism/tasks/netbox.py +++ b/osism/tasks/netbox.py @@ -29,8 +29,15 @@ def run(self, action, arguments): @app.task(bind=True, name="osism.tasks.netbox.set_maintenance") -def set_maintenance(self, device_name, state=True): - """Set the maintenance state for a device in the NetBox.""" +def set_maintenance(self, device_name, state=True, netbox_filter=None): + """Set the maintenance state for a device in the NetBox. + + Args: + device_name: Name of the device + state: Maintenance state (True/False) + netbox_filter: Optional URL filter (substring match, case-insensitive). + Only NetBox instances whose base_url contains this substring will be updated. + """ # Check if tasks are locked before execution utils.check_task_lock_and_exit() @@ -41,6 +48,13 @@ def set_maintenance(self, device_name, state=True): if lock.acquire(timeout=20): try: for nb in [utils.nb] + utils.secondary_nb_list: + # Apply filter if specified + if netbox_filter and netbox_filter.lower() not in nb.base_url.lower(): + logger.debug( + f"Skipping {nb.base_url} (does not match filter: {netbox_filter})" + ) + continue + logger.info( f"Set maintenance state of device {device_name} = {state} on {nb.base_url}" ) @@ -59,8 +73,15 @@ def set_maintenance(self, device_name, state=True): @app.task(bind=True, name="osism.tasks.netbox.set_provision_state") -def set_provision_state(self, device_name, state): - """Set the provision state for a device in the NetBox.""" +def set_provision_state(self, device_name, state, netbox_filter=None): + """Set the provision state for a device in the NetBox. + + Args: + device_name: Name of the device + state: Provision state value + netbox_filter: Optional URL filter (substring match, case-insensitive). + Only NetBox instances whose base_url contains this substring will be updated. + """ # Check if tasks are locked before execution utils.check_task_lock_and_exit() @@ -70,8 +91,14 @@ def set_provision_state(self, device_name, state): ) if lock.acquire(timeout=20): try: - for nb in [utils.nb] + utils.secondary_nb_list: + # Apply filter if specified + if netbox_filter and netbox_filter.lower() not in nb.base_url.lower(): + logger.debug( + f"Skipping {nb.base_url} (does not match filter: {netbox_filter})" + ) + continue + logger.info( f"Set provision state of device {device_name} = {state} on {nb.base_url}" ) @@ -90,8 +117,15 @@ def set_provision_state(self, device_name, state): @app.task(bind=True, name="osism.tasks.netbox.set_power_state") -def set_power_state(self, device_name, state): - """Set the provision state for a device in the NetBox.""" +def set_power_state(self, device_name, state, netbox_filter=None): + """Set the power state for a device in the NetBox. + + Args: + device_name: Name of the device + state: Power state value + netbox_filter: Optional URL filter (substring match, case-insensitive). + Only NetBox instances whose base_url contains this substring will be updated. + """ # Check if tasks are locked before execution utils.check_task_lock_and_exit() @@ -102,6 +136,13 @@ def set_power_state(self, device_name, state): if lock.acquire(timeout=20): try: for nb in [utils.nb] + utils.secondary_nb_list: + # Apply filter if specified + if netbox_filter and netbox_filter.lower() not in nb.base_url.lower(): + logger.debug( + f"Skipping {nb.base_url} (does not match filter: {netbox_filter})" + ) + continue + logger.info( f"Set power state of device {device_name} = {state} on {nb.base_url}" )