From 4c874cdf0fe703bc6cf8dd3dac972858c5b8fd09 Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Mon, 5 May 2025 16:56:15 -0500
Subject: [PATCH 01/20] wip: add node fixes

---
 cli/scripts/cluster.py           | 436 +++++++++++++------------------
 src/backrest/backrest.py         |  26 +-
 src/backrest/install-backrest.py |  88 +------
 3 files changed, 207 insertions(+), 343 deletions(-)

diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py
index 107ddce3..6348d568 100755
--- a/cli/scripts/cluster.py
+++ b/cli/scripts/cluster.py
@@ -1257,25 +1257,21 @@ def update_json(cluster_name, db_json):
     except Exception:
         util.exit_message("Unable to update JSON file", 1)
 
-def capture_backrest_config(cluster_name, verbose=False):
+def capture_backrest_config(node, verbose=False):
     """
-    Generate pgBackRest YAML on each node by delegating to
+    Generate pgBackRest YAML on a node by delegating to
     `./pgedge backrest write-config` (implemented in backrest.py).
 
-    The command is executed inside each node’s pgedge directory.
+    The command is executed inside the node’s pgedge directory.
     """
-    # Grab every node + its sub‑nodes from the cluster JSON
-    _, _, top_nodes = load_json(cluster_name)
-    nodes = [n for nd in top_nodes for n in (nd, *nd.get("sub_nodes", []))]
 
-    for nd in nodes:
-        cmd = f"cd {nd['path']}/pgedge && ./pgedge backrest write-config"
-        run_cmd(
-            cmd=cmd,
-            node=nd,
-            message="Generating pgBackrest YAML",
-            verbose=verbose,
-        )
+    cmd = f"cd {node['path']}/pgedge && ./pgedge backrest write-config"
+    run_cmd(
+        cmd=cmd,
+        node=node,
+        message="Generating pgBackrest YAML",
+        verbose=verbose,
+    )
 
 def init(cluster_name, install=True):
     """
@@ -1357,7 +1353,7 @@ def init(cluster_name, install=True):
     cluster_name_from_json = parsed_json["cluster_name"]
     
     for idx, node in enumerate(all_nodes, start=1):
-        backrest = node.get("backrest")
+        backrest = node.get("backrest", {})
         if backrest:
             util.message("## Integrating pgBackRest into the cluster", "info")
             util.message(f"### Configuring pgBackRest for node '{node['name']}'", "info")
@@ -1396,7 +1392,7 @@ def init(cluster_name, install=True):
             # -- Step 2: Configure postgresql.conf for pgBackRest (without --pg1-port)
             cmd_set_postgresqlconf = (
                 f"cd {node['path']}/pgedge && "
-                f"./pgedge backrest set_postgresqlconf "
+                f"./pgedge backrest set-postgresqlconf "
                 f"--stanza {stanza} "
                 f"--pg1-path {pg1_path} "
                 f"--repo1-path {repo1_path} "
@@ -1405,7 +1401,7 @@ def init(cluster_name, install=True):
             run_cmd(cmd_set_postgresqlconf, node=node, message="Modifying postgresql.conf for pgBackRest", verbose=verbose)
     
             # -- Step 3: Configure pg_hba.conf for pgBackRest (without --pg1-port)
-            cmd_set_hbaconf = f"cd {node['path']}/pgedge && ./pgedge backrest set_hbaconf"
+            cmd_set_hbaconf = f"cd {node['path']}/pgedge && ./pgedge backrest set-hbaconf"
             run_cmd(cmd_set_hbaconf, node=node, message="Modifying pg_hba.conf for pgBackRest", verbose=verbose)
     
             # -- Step 4: Reload PostgreSQL configuration to apply changes
@@ -1490,7 +1486,10 @@ def init(cluster_name, install=True):
                     # (f) Set BACKUP pg1-port to the node's port value
             cmd_set_pg1_port = f"cd {node['path']}/pgedge && ./pgedge set BACKUP repo1-path {repo1_path}"
             run_cmd(cmd_set_pg1_port, node=node, message=f"Setting BACKUP repo1-path to {repo1_path} on node '{node['name']}'", verbose=verbose)
-            capture_backrest_config(cluster_name, verbose=True)
+
+            for node in all_nodes:
+                if node.get("backrest", {}):
+                    capture_backrest_config(node, verbose=True)
     
     # 6. If it's an HA cluster, handle Patroni/etcd, etc.
     if is_ha_cluster:
@@ -1523,7 +1522,6 @@ def check_source_backrest_config(source_node_data):
             "info"
         )
     else:
-
         cmd = f"cd {source_node_data['path']}/pgedge && ./pgedge remove backrest"
         run_cmd(cmd, node=source_node_data, message="Removing pgBackRest configuration from source node", verbose=True)
 
@@ -1573,7 +1571,7 @@ def add_node(
     pg = db_settings["pg_version"]
     pgV = f"pg{pg}"
     verbose = cluster_data.get("log_level", "info")
-
+    
     # Load and validate the target node JSON
     target_node_file = f"{target_node}.json"
     if not os.path.isfile(target_node_file):
@@ -1589,7 +1587,7 @@ def add_node(
         )
 
     # NEW: Check if target JSON has backrest configuration
-    target_backrest_settings = target_node_data.get("backrest", {})
+    target_backrest_cfg = target_node_data.get("backrest", {})
     # Retrieve source node data
     source_node_data = next(
         (node for node in nodes if node["name"] == source_node), None
@@ -1598,8 +1596,8 @@ def add_node(
         util.exit_message(f"Source node '{source_node}' not found in cluster data.")
 
     # Extract backrest settings from source node (before using repo1_path flag)
-    backrest_settings = source_node_data.get("backrest", {})
-    source_repo1_path = backrest_settings.get("repo1_path")
+    source_backrest_cfg = source_node_data.get("backrest", {})
+    source_repo1_path = source_backrest_cfg.get("repo1_path")
 
     # Check: if source node JSON already provides repo1_path and the flag is given then exit
     if repo1_path and source_repo1_path:
@@ -1610,7 +1608,6 @@ def add_node(
 
     for group in target_node_data.get("node_groups", []):
         ssh_info = group.get("ssh")
-        backrest_info = group.get("backrest")
         os_user = ssh_info.get("os_user", "")
         ssh_key = ssh_info.get("private_key", "")
 
@@ -1626,9 +1623,8 @@ def add_node(
             "ssh_key": ssh_key,
         }
 
-    # If backrest settings are provided in the JSON, add them to new_node_data.
-    if backrest_info:
-        new_node_data["backrest"] = backrest_info
+
+
     if "public_ip" not in new_node_data and "private_ip" not in new_node_data:
         util.exit_message(
             "Both public_ip and private_ip are missing in target node data."
@@ -1648,10 +1644,11 @@ def add_node(
             "public_ip", new_node_data.get("private_ip")
         )
 
-    # Fetch backrest settings from the source node directory
-    backrest_settings = source_node_data.get("backrest", {})
-    # New check: if pgBackRest is not configured on the source node
-    if not backrest_settings:
+    # If backrest settings are provided in the JSON, add them to new_node_data.
+    if source_backrest_cfg:
+        new_node_data["backrest"] = source_backrest_cfg
+    else:
+        # Otherwise, we assume pgBackRest is not configured and should be installed.
         # Step 1: Install pgBackRest on the source node
         cmd_install_backrest = (
             f"cd {source_node_data['path']}/pgedge && ./pgedge install backrest"
@@ -1674,22 +1671,14 @@ def add_node(
         repo1_retention_full = "7"
         log_level_console = "info"
         repo1_cipher_type = "aes-256-cbc"
-        repo1_type = "posix"  # Could also be "s3"
+        repo1_type = "posix" 
+
         # Determine the repository path for the source node.
         if repo1_path:
             # Use the provided flag value as-is (trimmed of any trailing slash).
             repo1_path_source = repo1_path.rstrip("/")
         else:
-            # No repo1_path flag provided: check the JSON configuration or default.
-            json_repo1_path = backrest_settings.get("repo1_path")
-            if json_repo1_path:
-                repo1_path_source = json_repo1_path.rstrip("/")
-                if not repo1_path_source.endswith(source_node_data["name"]):
-                    repo1_path_source = (
-                        repo1_path_source + f"/{source_node_data['name']}"
-                    )
-            else:
-                repo1_path_source = f"/var/lib/pgbackrest/{source_node_data['name']}"
+            repo1_path_source = f"/var/lib/pgbackrest/{source_node_data['name']}"
 
         # Similarly, set restore_path to include node name
         restore_path_source = "/var/lib/pgbackrest_restore"
@@ -1705,7 +1694,7 @@ def add_node(
         # Step 2: Configure postgresql.conf for pgBackRest (without --pg1-port)
         cmd_set_postgresqlconf_source = (
             f"cd {source_node_data['path']}/pgedge && "
-            f"./pgedge backrest set_postgresqlconf "
+            f"./pgedge backrest set-postgresqlconf "
             f"--stanza {stanza_source} "
             f"--pg1-path {pg1_path_source} "
             f"--repo1-path {repo1_path_source} "
@@ -1720,7 +1709,7 @@ def add_node(
 
         # Step 3: Configure pg_hba.conf for pgBackRest (without --pg1-port)
         cmd_set_hbaconf_source = (
-            f"cd {source_node_data['path']}/pgedge && ./pgedge backrest set_hbaconf"
+            f"cd {source_node_data['path']}/pgedge && ./pgedge backrest set-hbaconf"
         )
         run_cmd(
             cmd_set_hbaconf_source,
@@ -1769,8 +1758,6 @@ def add_node(
             )
 
         # Step 6: Set all pgBackRest backup configuration values for the source node.
-        #
-        # Build one compound shell command
         compound_cmd = " && ".join(
             [
                 f"cd {source_node_data['path']}/pgedge",
@@ -1792,7 +1779,7 @@ def add_node(
             verbose=False,
         )
 
-        # Step 7:Create the pgBackRest stanza (this command uses --pg1-port because it connects to the DB)
+        # Step 7: Create the pgBackRest stanza (this command uses --pg1-port because it connects to the DB)
         cmd_create_stanza_source = (
             f"cd {source_node_data['path']}/pgedge && "
             f"./pgedge backrest command stanza-create "
@@ -1837,8 +1824,8 @@ def add_node(
             verbose=verbose,
         )
 
-        # Update pgbackrest_settings for further use downstream.
-        backrest_settings = {
+        # Update source backrest config for further use downstream.
+        source_backrest_cfg = {
             "stanza": stanza_source,
             "repo1_path": repo1_path_source,
             "repo1-retention-full": repo1_retention_full,
@@ -1847,16 +1834,12 @@ def add_node(
             "repo1_type": repo1_type,
         }
 
-    # NEW: Override pgbackrest settings with target JSON settings if present
-    if target_backrest_settings:
-        backrest_settings = target_backrest_settings
-
-    # For subsequent steps we extract settings from pgbackrest_settings.
-    stanza = backrest_settings.get("stanza", f"pg{pg}")
-    repo1_retention_full = backrest_settings.get("repo1-retention-full", "7")
-    log_level_console = backrest_settings.get("log-level-console", "info")
-    repo1_cipher_type = backrest_settings.get("repo1-cipher-type", "aes-256-cbc")
-    repo1_type = backrest_settings.get("repo1_type", "posix")
+    # For subsequent steps we extract pgbackrest settings from the source node configuration.
+    stanza = source_backrest_cfg.get("stanza", f"pg{pg}")
+    repo1_retention_full = source_backrest_cfg.get("repo1-retention-full", "7")
+    log_level_console = source_backrest_cfg.get("log-level-console", "info")
+    repo1_cipher_type = source_backrest_cfg.get("repo1-cipher-type", "aes-256-cbc")
+    repo1_type = source_backrest_cfg.get("repo1_type", "posix")
 
     rc = ssh_install_pgedge(
         cluster_name,
@@ -1876,16 +1859,16 @@ def add_node(
     if not repo1_path:
         # Do not install pgbackrest on source node; simply fetch the repo1_path from source's settings.
         repo1_path_default = f"/var/lib/pgbackrest/{source_node_data['name']}"
-        repo1_path = backrest_settings.get("repo1_path", f"{repo1_path_default}")
+        repo1_path = source_backrest_cfg.get("repo1_path", f"{repo1_path_default}")
     else:
         cmd = (
-            f"{source_node_data['path']}/pgedge/pgedge backrest set_postgresqlconf {stanza} "
+            f"{source_node_data['path']}/pgedge/pgedge backrest set-postgresqlconf {stanza} "
             f"{pg1_path} {repo1_path} {repo1_type}"
         )
         message = f"Modifying postgresql.conf file"
         run_cmd(cmd, source_node_data, message=message, verbose=verbose)
 
-        cmd = f"{source_node_data['path']}/pgedge/pgedge backrest set_hbaconf"
+        cmd = f"{source_node_data['path']}/pgedge/pgedge backrest set-hbaconf"
         message = f"Modifying pg_hba.conf file"
         run_cmd(cmd, source_node_data, message=message, verbose=verbose)
 
@@ -1940,7 +1923,7 @@ def add_node(
 
     cmd = (
         f'{new_node_data["path"]}/pgedge/pgedge backrest command restore '
-        f"--repo1-type={repo1_type} --stanza={stanza} "
+        f"--repo1-type={repo1_type} --stanza={stanza} --no-archive "
         f'--pg1-path={new_node_data["path"]}/pgedge/data/pg{pg} {args}'
     )
     message = f"Restoring backup"
@@ -1948,6 +1931,7 @@ def add_node(
 
     pgd = f'{new_node_data["path"]}/pgedge/data/pg{pg}'
     pgc = f"{pgd}/postgresql.conf"
+    log_directory = f'{new_node_data["path"]}/pgedge/data/logs/pg{pg}'
 
     cmd = f"echo \"ssl_cert_file='{pgd}/server.crt'\" >> {pgc}"
     message = f"Setting ssl_cert_file"
@@ -1957,19 +1941,12 @@ def add_node(
     message = f"Setting ssl_key_file"
     run_cmd(cmd, new_node_data, message=message, verbose=verbose)
 
-    cmd = f"echo \"log_directory='{pgd}/log'\" >> {pgc}"
+    cmd = f"echo \"log_directory='{log_directory}'\" >> {pgc}"
     message = f"Setting log_directory"
     run_cmd(cmd, new_node_data, message=message, verbose=verbose)
 
     cmd = (
-        f'echo "shared_preload_libraries = '
-        f"'pg_stat_statements, snowflake, spock'\" >> {pgc}"
-    )
-    message = f"Setting shared_preload_libraries"
-    run_cmd(cmd, new_node_data, message=message, verbose=verbose)
-
-    cmd = (
-        f'{new_node_data["path"]}/pgedge/pgedge backrest configure_replica {stanza} '
+        f'{new_node_data["path"]}/pgedge/pgedge backrest configure-replica {stanza} '
         f'{new_node_data["path"]}/pgedge/data/pg{pg} {source_node_data["ip_address"]} '
         f'{source_node_data["port"]} {source_node_data["os_user"]}'
     )
@@ -2001,7 +1978,7 @@ def add_node(
     cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}"
     message = f"Promoting standby to primary"
     run_cmd(cmd, new_node_data, message=message, verbose=verbose)
-
+    
     for mdb in db:
         sql_cmd = "SELECT sub_name FROM spock.subscription"
         cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {mdb['db_name']}"
@@ -2118,166 +2095,149 @@ def add_node(
         cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True
     )
     print(f"\n{result.stdout}")
+    
+    # A subsequent restart will be needed to apply the changes
+    # This will occur in the next section when pgBackrest is configured or removed
+    # Cleanup restore remnants by unsetting restore_command on target node
+    sql_cmd = (
+        'ALTER SYSTEM RESET restore_command'
+    )
+    cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}"
+    message = "Unsetting restore_command on target node"
+    run_cmd(cmd, node=new_node_data, message=message, verbose=verbose)
 
-    # Reload the complete target JSON file and fetch repo1_path and stanza from its pgbackrest settings.
-    try:
-        with open(target_node_file, "r") as f:
-            complete_target_json = json.load(f)
-
-        # Since pgbackrest settings are stored inside each node group, fetch from the first node group.
-        if (
-            "node_groups" in complete_target_json
-            and complete_target_json["node_groups"]
-        ):
-            repo1_path_target_file = (
-                complete_target_json["node_groups"][0]
-                .get("backrest", {})
-                .get("repo1_path")
-            )
-            target_stanza_target_file = (
-                complete_target_json["node_groups"][0].get("backrest", {}).get("stanza")
-            )
-
-            pgedge_dir = f"{new_node_data['path']}/pgedge"
-            restore_path = backrest_settings.get(
-                "restore_path", f"/var/lib/pgbackrest_restore/{new_node_data['name']}"
-            )
-            target_repo1_host_user = backrest_settings.get(
-                "repo1_host_user", new_node_data.get("os_user", "postgres")
-            )
-            target_pg1_path = backrest_settings.get(
-                "pg1_path", f"{pgedge_dir}/data/pg{pg}"
-            )
-            target_pg1_user = backrest_settings.get(
-                "pg1_user", new_node_data.get("os_user", "postgres")
-            )
-            target_pg1_port = backrest_settings.get(
-                "pg1_port", new_node_data.get("port", "6435")
-            )
-
-            # ---------------------------------------------------------------
-            # If the repo1_path or stanza is missing, remove backrest instead
-            # ---------------------------------------------------------------
-            if not repo1_path_target_file or not target_stanza_target_file:
-                # No valid repo1_path or stanza -> remove backrest
-                repo1_path_target_file = None
-                target_stanza_target_file = None
-
-                cmd_remove_backrest_target = (
-                    f"cd {pgedge_dir} && ./pgedge remove backrest"
-                )
-                run_cmd(
-                    cmd=cmd_remove_backrest_target,
-                    node=new_node_data,
-                    message="Removing backrest",
-                    verbose=verbose,
-                )
-            else:
-                # Both repo1_path_target_file and target_stanza_target_file exist
-                target_repo1_path = repo1_path_target_file
-                target_stanza = target_stanza_target_file
-
-                # Combined target node BACKUP configuration commands
-                combined_target_cmd = (
-                    f"sudo mkdir -p {restore_path} && "
-                    f"sudo chown -R {target_repo1_host_user}:{target_repo1_host_user} {restore_path} && "
-                    f"cd {pgedge_dir} && ./pgedge set BACKUP restore_path {restore_path} && "
-                    f"sudo mkdir -p {target_repo1_path} && "
-                    f"sudo chown -R {target_repo1_host_user}:{target_repo1_host_user} {target_repo1_path} && "
-                    f"cd {pgedge_dir} && ./pgedge set BACKUP repo1-path {target_repo1_path} && "
-                    f"cd {pgedge_dir} && ./pgedge set BACKUP repo1-host-user {target_repo1_host_user} && "
-                    f"cd {pgedge_dir} && ./pgedge set BACKUP pg1-path {target_pg1_path} && "
-                    f"cd {pgedge_dir} && ./pgedge set BACKUP pg1-user {target_pg1_user} && "
-                    f"cd {pgedge_dir} && ./pgedge set BACKUP pg1-port {target_pg1_port} && "
-                    f"cd {pgedge_dir} && ./pgedge set BACKUP stanza {target_stanza}"
-                )
-                run_cmd(
-                    cmd=combined_target_cmd,
-                    node=new_node_data,
-                    message="Setting all target node BACKUP configuration",
-                    verbose=False,
-                )
+    # Cleanup replica configuration
+    cmd = (
+        f"cd {new_node_data['path']}/pgedge && "
+        f"./pgedge backrest cleanup-replica "
+        f"--pg1-path {new_node_data['path']}/pgedge/data/pg{pg} "
+    )
+    run_cmd(cmd, node=new_node_data, message="Cleaning up replica configuration on target node", verbose=verbose)
 
-                # Append the BACKUP settings to the target's PostgreSQL configuration
-                cmd_set_postgresqlconf_target = (
-                    f"cd {pgedge_dir} && ./pgedge backrest set_postgresqlconf "
-                    f"{target_stanza} {target_pg1_path} {target_repo1_path} {repo1_type}"
-                )
-                run_cmd(
-                    cmd=cmd_set_postgresqlconf_target,
-                    node=new_node_data,
-                    message="Appending BACKUP settings to postgresql.conf for target node",
-                    verbose=verbose,
-                )
+    if target_backrest_cfg:
+        repo1_path_target_file = (
+            target_node_data["node_groups"][0]
+            .get("backrest", {})
+            .get("repo1_path")
+        )
+        target_stanza_target_file = (
+            target_node_data["node_groups"][0].get("backrest", {}).get("stanza")
+        )
 
-                # Restart PostgreSQL to apply the new configuration
-                cmd_restart_postgres = f"cd {pgedge_dir} && ./pgedge restart"
-                run_cmd(
-                    cmd=cmd_restart_postgres,
-                    node=new_node_data,
-                    message="Restarting PostgreSQL service",
-                    verbose=verbose,
-                )
+        pgedge_dir = f"{new_node_data['path']}/pgedge"
+        restore_path = target_backrest_cfg.get(
+            "restore_path", f"/var/lib/pgbackrest_restore/{new_node_data['name']}"
+        )
+        target_repo1_host_user = target_backrest_cfg.get(
+            "repo1_host_user", new_node_data.get("os_user", "postgres")
+        )
+        target_pg1_path = target_backrest_cfg.get(
+            "pg1_path", f"{pgedge_dir}/data/pg{pg}"
+        )
+        target_pg1_user = target_backrest_cfg.get(
+            "pg1_user", new_node_data.get("os_user", "postgres")
+        )
+        target_pg1_port = target_backrest_cfg.get(
+            "pg1_port", new_node_data.get("port", "6435")
+        )
 
-                # Now create the pgBackRest stanza on the target node
-                cmd_create_stanza_target = (
-                    f"cd {pgedge_dir} && "
-                    f"./pgedge backrest command stanza-create "
-                    f"--stanza '{target_stanza}' "
-                    f"--pg1-path '{target_pg1_path}' "
-                    f"--repo1-cipher-type {repo1_cipher_type} "
-                    f"--pg1-port {target_pg1_port} "
-                    f"--repo1-path {target_repo1_path}"
-                )
-                run_cmd(
-                    cmd=cmd_create_stanza_target,
-                    node=new_node_data,
-                    message=f"Creating pgBackRest stanza '{target_stanza}'",
-                    verbose=verbose,
-                )
+        # Both repo1_path_target_file and target_stanza_target_file exist
+        target_repo1_path = repo1_path_target_file
+        target_stanza = target_stanza_target_file
+
+        # Combined target node BACKUP configuration commands
+        combined_target_cmd = (
+            f"sudo mkdir -p {restore_path} && "
+            f"sudo chown -R {target_repo1_host_user}:{target_repo1_host_user} {restore_path} && "
+            f"cd {pgedge_dir} && ./pgedge set BACKUP restore_path {restore_path} && "
+            f"sudo mkdir -p {target_repo1_path} && "
+            f"sudo chown -R {target_repo1_host_user}:{target_repo1_host_user} {target_repo1_path} && "
+            f"cd {pgedge_dir} && ./pgedge set BACKUP repo1-path {target_repo1_path} && "
+            f"cd {pgedge_dir} && ./pgedge set BACKUP repo1-host-user {target_repo1_host_user} && "
+            f"cd {pgedge_dir} && ./pgedge set BACKUP pg1-path {target_pg1_path} && "
+            f"cd {pgedge_dir} && ./pgedge set BACKUP pg1-user {target_pg1_user} && "
+            f"cd {pgedge_dir} && ./pgedge set BACKUP pg1-port {target_pg1_port} && "
+            f"cd {pgedge_dir} && ./pgedge set BACKUP stanza {target_stanza}"
+        )
+        run_cmd(
+            cmd=combined_target_cmd,
+            node=new_node_data,
+            message="Setting all target node BACKUP configuration",
+            verbose=False,
+        )
 
-                # Create a full backup using pgBackRest
-                backrest_backup_args_target = (
-                    f"--repo1-path {target_repo1_path} "
-                    f"--stanza {target_stanza} "
-                    f"--pg1-path {target_pg1_path} "
-                    f"--repo1-type {repo1_type} "
-                    f"--log-level-console {log_level_console} "
-                    f"--pg1-port {target_pg1_port} "
-                    f"--db-socket-path /tmp "
-                    f"--repo1-cipher-type {repo1_cipher_type} "
-                    f"--repo1-retention-full {repo1_retention_full} "
-                    f"--type=full"
-                )
-                cmd_create_backup_target = (
-                    f"cd {pgedge_dir} && ./pgedge backrest command backup "
-                    f"'{backrest_backup_args_target}'"
-                )
-                run_cmd(
-                    cmd=cmd_create_backup_target,
-                    node=new_node_data,
-                    message="Creating full pgBackRest backup",
-                    verbose=verbose,
-                )
-            
-        else:
-            # If no node_groups exist at all, remove pgbackrest
-            repo1_path_target_file = None
-            target_stanza_target_file = None
+        # Append the BACKUP settings to the target's PostgreSQL configuration
+        cmd_set_postgresqlconf_target = (
+            f"cd {pgedge_dir} && ./pgedge backrest set-postgresqlconf "
+            f"{target_stanza} {target_pg1_path} {target_repo1_path} {repo1_type}"
+        )
+        run_cmd(
+            cmd=cmd_set_postgresqlconf_target,
+            node=new_node_data,
+            message="Appending BACKUP settings to postgresql.conf for target node",
+            verbose=verbose,
+        )
 
-            pgedge_dir = f"{new_node_data['path']}/pgedge"
-            cmd_remove_backrest_target = f"cd {pgedge_dir} && ./pgedge remove backrest"
-            run_cmd(
-                cmd=cmd_remove_backrest_target,
-                node=new_node_data,
-                message="Removing backrest",
-                verbose=verbose,
-            )
+        # Restart PostgreSQL to apply the new configuration
+        cmd_restart_postgres = f"cd {pgedge_dir} && ./pgedge restart"
+        run_cmd(
+            cmd=cmd_restart_postgres,
+            node=new_node_data,
+            message="Restarting PostgreSQL service",
+            verbose=verbose,
+        )
 
-    except Exception as e:
-        print(f"Error fetching values from target JSON file: {e}")
+        # Now create the pgBackRest stanza on the target node
+        cmd_create_stanza_target = (
+            f"cd {pgedge_dir} && "
+            f"./pgedge backrest command stanza-create "
+            f"--stanza '{target_stanza}' "
+            f"--pg1-path '{target_pg1_path}' "
+            f"--repo1-cipher-type {repo1_cipher_type} "
+            f"--pg1-port {target_pg1_port} "
+            f"--repo1-path {target_repo1_path}"
+        )
+        run_cmd(
+            cmd=cmd_create_stanza_target,
+            node=new_node_data,
+            message=f"Creating pgBackRest stanza '{target_stanza}'",
+            verbose=verbose,
+        )
 
-    # NEW: Check and display pgBackRest configuration status in the source node
+        # Create a full backup using pgBackRest
+        backrest_backup_args_target = (
+            f"--repo1-path {target_repo1_path} "
+            f"--stanza {target_stanza} "
+            f"--pg1-path {target_pg1_path} "
+            f"--repo1-type {repo1_type} "
+            f"--log-level-console {log_level_console} "
+            f"--pg1-port {target_pg1_port} "
+            f"--db-socket-path /tmp "
+            f"--repo1-cipher-type {repo1_cipher_type} "
+            f"--repo1-retention-full {repo1_retention_full} "
+            f"--type=full"
+        )
+        cmd_create_backup_target = (
+            f"cd {pgedge_dir} && ./pgedge backrest command backup "
+            f"'{backrest_backup_args_target}'"
+        )
+        run_cmd(
+            cmd=cmd_create_backup_target,
+            node=new_node_data,
+            message="Creating full pgBackRest backup",
+            verbose=verbose,
+        )
+    # else:
+    #     pgedge_dir = f"{new_node_data['path']}/pgedge"
+    #     cmd_remove_backrest_target = f"cd {pgedge_dir} && ./pgedge remove backrest"
+    #     run_cmd(
+    #         cmd=cmd_remove_backrest_target,
+    #         node=new_node_data,
+    #         message="Removing backrest from target node",
+    #         verbose=verbose,
+    #     )
+
+    # Check and display pgBackRest configuration status in the source node
     # Remove unnecessary keys before appending new node to the cluster data
     new_node_data.pop("ip_address", None)
     new_node_data.pop("os_user", None)
@@ -2288,32 +2248,10 @@ def add_node(
     cluster_data["update_date"] = datetime.datetime.now().astimezone().isoformat()
 
     write_cluster_json(cluster_name, cluster_data)
-    capture_backrest_config(cluster_name, verbose=True)
+    if target_backrest_cfg:
+        capture_backrest_config(new_node_data, verbose=True)
     check_source_backrest_config(source_node_data)
 
-
-def cleanup_backrest_from_cluster(cluster_json, target_json):
-    """
-    Compare the node groups in the target JSON with the cluster JSON.
-    For each node in cluster_json["node_groups"], if the corresponding node (by name)
-    is not present in target_json's node groups or does not contain a "backrest" key,
-    then remove the "backrest" key from that node in cluster_json.
-    
-    Args:
-        cluster_json (dict): The main cluster configuration.
-        target_json (dict): The target node configuration JSON.
-    """
-    # Create a mapping of node names to their configuration from the target JSON.
-    target_nodes = {group.get("name"): group for group in target_json.get("node_groups", [])}
-    
-    for node in cluster_json.get("node_groups", []):
-        node_name = node.get("name")
-        target_group = target_nodes.get(node_name)
-        # If the target group does not exist or does not contain a backrest key, delete backrest in the main config.
-        if not target_group or "backrest" not in target_group:
-            if "backrest" in node:
-                del node["backrest"]
-
 def json_validate_add_node(data):
     """
     Validate the structure of a node‑definition JSON file that will be fed to
diff --git a/src/backrest/backrest.py b/src/backrest/backrest.py
index 517b8908..52fcf823 100755
--- a/src/backrest/backrest.py
+++ b/src/backrest/backrest.py
@@ -7,8 +7,6 @@
 import sys
 from datetime import datetime
 from tabulate import tabulate
-import yaml
-from typing import Optional
 
 def pgV():
     """Return the first found PostgreSQL version (v14 thru v17)."""
@@ -183,6 +181,7 @@ def backup(stanza, type="full", verbose=True):
         )
     else:
         util.message(f"Successfully completed {type} backup for stanza '{stanza}'")
+
 def restore(stanza, data_dir=None, backup_label=None, recovery_target_time=None, verbose=True):
     """Restore a database cluster to a specified state."""
     config = fetch_config()
@@ -254,6 +253,19 @@ def pitr(stanza, data_dir=None, recovery_target_time=None, verbose=True):
     if restore(stanza, data_dir, None, recovery_target_time, verbose):
         _configure_pitr(stanza, data_dir, recovery_target_time)
 
+def cleanup_replica(pg1_path):
+    """Cleanup the replica configuration and restore remnants."""
+    conf_file = os.path.join(pg1_path, "postgresql.conf")
+    changes = {
+        "hot_standby": "off",
+        "primary_conninfo": "",
+        "archive_command": "",
+        "archive_mode": "off"
+    }
+    for key, value in changes.items():
+        change_pgconf_keyval(conf_file, key, value)
+
+
 def _configure_pitr(stanza, pg_data_dir=None, recovery_target_time=None):
     """Configure PostgreSQL for point-in-time recovery."""
     config = fetch_config()
@@ -264,7 +276,6 @@ def _configure_pitr(stanza, pg_data_dir=None, recovery_target_time=None):
     config_file = os.path.join(pg_data_dir, "postgresql.conf")
     changes = {
         "port": "5433",
-        "log_directory": os.path.join(pg_data_dir, "log"),
         "archive_command": "",
         "archive_mode": "off",
         "hot_standby": "on",
@@ -288,6 +299,7 @@ def change_pgconf_keyval(config_path, key, value):
                 file.write(line)
         if not key_found:
             file.write(f"{key} = '{value}'\n")
+
 def create_replica(stanza, data_dir=None, backup_label=None, verbose=True):
     """Create a replica by restoring from a backup."""
     if restore(stanza, data_dir, backup_label, verbose=verbose):
@@ -303,7 +315,6 @@ def configure_replica(stanza, pg1_path, pg1_host, pg1_port, pg1_user):
         "hot_standby": "on",
         "primary_conninfo": primary_conninfo,
         "port": pg1_port,
-        "log_directory": os.path.join(pg1_path, "log"),
         "archive_command": "cd .",
         "archive_mode": "on"
     }
@@ -518,12 +529,13 @@ def update_config(verbose=True):
         "pitr": pitr,
         "create-stanza": create_stanza,
         "create-replica": create_replica,
-        "configure_replica": configure_replica,
+        "configure-replica": configure_replica,
         "list-backups": list_backups,
         "show-config": show_config,
         "write-config": write_config,
         "update-config": update_config, 
-        "set_hbaconf": modify_hba_conf,
-        "set_postgresqlconf": modify_postgresql_conf,
+        "set-hbaconf": modify_hba_conf,
+        "set-postgresqlconf": modify_postgresql_conf,
+        "cleanup-replica": cleanup_replica,
         "command": run_external_command,
     })
diff --git a/src/backrest/install-backrest.py b/src/backrest/install-backrest.py
index c004d8ad..461168d1 100644
--- a/src/backrest/install-backrest.py
+++ b/src/backrest/install-backrest.py
@@ -2,12 +2,7 @@
 #     Copyright (c)  2022-2025 PGEDGE  #
 
 import os
-import subprocess
-import time
 import sys
-import getpass
-from crontab import CronTab
-import subprocess
 import util
 
 thisDir = os.path.dirname(os.path.realpath(__file__))
@@ -36,6 +31,7 @@ def osSys(p_input, p_display=False):
         util.message("# " + p_input)
     rc = os.system(p_input)
     return rc
+
 def configure_backup_settings():
     """Configure and return the pgBackRest settings."""
     stanza = "pg16"
@@ -93,94 +89,12 @@ def setup_pgbackrest_links():
     osSys("sudo mkdir -p -m 770 /var/lib/pgbackrest")
     osSys(f"sudo chown {usrUsr} -R /var/lib/pgbackrest")
 
-def modify_hba_conf():
-  new_rules = [
-      {
-            "type": "host",
-            "database": "replication",
-            "user": "all",
-            "address": "127.0.0.1/0",
-            "method": "trust"
-      }
-  ]
-  util.update_pg_hba_conf(pgV(), new_rules)
-
-def create_or_update_job(crontab_lines, job_comment, detailed_comment, new_job):
-    job_identifier = f"# {job_comment}"
-    detailed_comment_line = f"# {detailed_comment}\n"
-    job_exists = False
-
-    for i, line in enumerate(crontab_lines):
-        if job_identifier in line:
-            crontab_lines[i] = job_identifier + "\n"
-            if i + 1 < len(crontab_lines):
-                crontab_lines[i + 1] = detailed_comment_line
-                crontab_lines[i + 2] = new_job
-            job_exists = True
-            break
-
-    if not job_exists:
-        crontab_lines.extend([job_identifier + "\n", detailed_comment_line, new_job])
-
-def define_cron_job():
-    stanza = util.get_value("BACKUP", "stanza")
-    full_backup_command = f"pgbackrest --stanza={stanza} --type=full backup"
-    incr_backup_command = f"pgbackrest --stanza={stanza} --type=incr backup"
-    expire_backup_command = f"pgbackrest --stanza={stanza} expire"
-
-    run_as_user = 'root'
-
-    # Crontab entries with detailed comments
-    full_backup_cron = f"0 1 * * * {run_as_user} {full_backup_command}\n"
-    incr_backup_cron = f"0 * * * * {run_as_user} {incr_backup_command}\n"
-    expire_backup_cron = f"30 1 * * * {run_as_user} {expire_backup_command}\n"
-
-    # Detailed comments for each job
-    full_backup_comment = "Performs a full backup daily at 1 AM."
-    incr_backup_comment = "Performs an incremental backup every hour."
-    expire_backup_comment = "Manages backup retention, expiring old backups at 1:30 AM daily."
-
-    system_crontab_path = "/etc/crontab"
-    backrest_crontab_path = "backrest.crontab"
-
-    with open(system_crontab_path, 'r') as file:
-        existing_crontab = file.readlines()
-
-    create_or_update_job(existing_crontab, "FullBackup", full_backup_comment, full_backup_cron)
-    create_or_update_job(existing_crontab, "IncrementalBackup", incr_backup_comment, incr_backup_cron)
-    create_or_update_job(existing_crontab, "ExpireBackup", expire_backup_comment, expire_backup_cron)
-
-    with open(backrest_crontab_path, 'w') as file:
-        file.writelines(existing_crontab)
-
-    osSys(f"sudo cat {backrest_crontab_path} | sudo tee {system_crontab_path} > /dev/null", False)
-
-def fetch_backup_config():
-    """Fetch and return the pgBackRest configuration from system settings."""
-    config = {}
-    params = [
-        "stanza", "restore_path", "backup-type", "repo1-retention-full",
-        "repo1-retention-full-type", "repo1-path", "repo1-host-user",
-        "repo1-host", "repo1-cipher-type", "repo1-cipher-pass", "repo1-s3-bucket",
-        "repo1-s3-key-secret", "repo1-s3-key", "repo1-s3-region", "repo1-s3-endpoint",
-        "log-level-console", "repo1-type", "process-max", "compress-level",
-        "pg1-path", "pg1-user", "pg1-database", "db-socket-path", "pg1-port",
-        "pg1-host"
-    ]
-
-    # Fetch all parameters
-    for param in params:
-        config[param] = util.get_value("BACKUP", param)
-    
-    return config
-
 def print_header(header):
     bold_start = "\033[1m"
     bold_end = "\033[0m"
     print(bold_start + "##### " + header + " #####"+ bold_end)
 
 def main():
-    stanza = pgV()
     if os.path.isdir(f"/var/lib/pgbackrest/"):
         util.message("/var/lib/pgbackrest directory already exists")
     

From 94d644a025cf5c4969ab301dde3ed96de4d9e9fc Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Mon, 5 May 2025 20:57:40 -0500
Subject: [PATCH 02/20] wip: further changes

---
 cli/scripts/cluster.py   | 134 +++++++++------------------------------
 src/backrest/backrest.py |   6 +-
 2 files changed, 32 insertions(+), 108 deletions(-)

diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py
index 6348d568..f138a339 100755
--- a/cli/scripts/cluster.py
+++ b/cli/scripts/cluster.py
@@ -937,7 +937,7 @@ def get_cluster_info(cluster_name):
             "repo1_path": backrest_storage_path,
             "repo1_retention_full": "7",
             "log_level_console": "info",
-            "repo1_cipher-type": "aes-256-cbc",
+            "repo1_cipher_type": "aes-256-cbc",
             "archive_mode": backrest_archive_mode,
             "repo1_type": repo1_type,
         }
@@ -1364,7 +1364,7 @@ def init(cluster_name, install=True):
             # Load additional pgBackRest settings from JSON with defaults.
             repo1_retention_full = backrest.get("repo1_retention_full", "7")
             log_level_console = backrest.get("log_level_console", "info")
-            repo1_cipher_type = backrest.get("repo1_cipher-type", "aes-256-cbc")
+            repo1_cipher_type = backrest.get("repo1_cipher_type", "aes-256-cbc")
             repo1_type = backrest.get("repo1_type", "posix")  # Could also be "s3", etc.
     
             # Get repo1_path from JSON; if not provided, default to /var/lib/pgbackrest/{node_name}
@@ -1396,7 +1396,8 @@ def init(cluster_name, install=True):
                 f"--stanza {stanza} "
                 f"--pg1-path {pg1_path} "
                 f"--repo1-path {repo1_path} "
-                f"--repo1-type {repo1_type}"
+                f"--repo1-type {repo1_type} "
+                f"--repo1-cipher-type {repo1_cipher_type} "
             )
             run_cmd(cmd_set_postgresqlconf, node=node, message="Modifying postgresql.conf for pgBackRest", verbose=verbose)
     
@@ -1409,25 +1410,7 @@ def init(cluster_name, install=True):
             cmd_reload_conf = f"cd {node['path']}/pgedge && ./pgedge psql '{sql_reload_conf}' {db[0]['db_name']}"
             run_cmd(cmd_reload_conf, node=node, message="Reloading PostgreSQL configuration", verbose=verbose)
     
-            # -- Step 5: If using S3 as repository, export necessary environment variables
-            if repo1_type.lower() == "s3":
-                required_env_vars = [
-                    "PGBACKREST_REPO1_S3_KEY",
-                    "PGBACKREST_REPO1_S3_BUCKET",
-                    "PGBACKREST_REPO1_S3_KEY_SECRET",
-                    "PGBACKREST_REPO1_CIPHER_PASS",
-                ]
-                missing_env_vars = [var for var in required_env_vars if var not in os.environ]
-                if missing_env_vars:
-                    util.exit_message(
-                        f"Environment variables {', '.join(missing_env_vars)} must be set for S3 pgBackRest configuration.",
-                        1,
-                    )
-                s3_exports = " && ".join([f"export {var}={os.environ[var]}" for var in required_env_vars])
-                cmd_export_s3 = f"cd {node['path']}/pgedge && {s3_exports}"
-                run_cmd(cmd_export_s3, node=node, message="Setting S3 environment variables for pgBackRest", verbose=verbose)
-    
-            # -- Step 6: Set all pgBackRest backup configuration values
+            # -- Step 5: Set all pgBackRest backup configuration values
     
             # (a) Set the backup stanza
             cmd_set_backup_stanza = f"cd {node['path']}/pgedge && ./pgedge set BACKUP stanza {stanza}"
@@ -1456,7 +1439,7 @@ def init(cluster_name, install=True):
             cmd_set_pg1_port = f"cd {node['path']}/pgedge && ./pgedge set BACKUP pg1-port {port}"
             run_cmd(cmd_set_pg1_port, node=node, message=f"Setting BACKUP pg1-port to {port} on node '{node['name']}'", verbose=verbose)
     
-            # -- Step 7: Create the pgBackRest stanza (this command uses --pg1-port because it connects to the DB)
+            # -- Step 6: Create the pgBackRest stanza (this command uses --pg1-port because it connects to the DB)
             cmd_create_stanza = (
                 f"cd {node['path']}/pgedge && "
                 f"./pgedge backrest command stanza-create "
@@ -1468,7 +1451,7 @@ def init(cluster_name, install=True):
             )
             run_cmd(cmd_create_stanza, node=node, message=f"Creating pgBackRest stanza '{stanza}'", verbose=verbose)
     
-            # -- Step 8: Initiate a full backup using pgBackRest (again, passing the port)
+            # -- Step 7: Initiate a full backup using pgBackRest (again, passing the port)
             backrest_backup_args = (
                 f"--repo1-path {repo1_path} "
                 f"--stanza {stanza} "
@@ -1698,7 +1681,8 @@ def add_node(
             f"--stanza {stanza_source} "
             f"--pg1-path {pg1_path_source} "
             f"--repo1-path {repo1_path_source} "
-            f"--repo1-type {repo1_type}"
+            f"--repo1-type {repo1_type} "
+            f"--repo1-cipher-type {repo1_cipher_type} "
         )
         run_cmd(
             cmd_set_postgresqlconf_source,
@@ -1728,36 +1712,7 @@ def add_node(
             verbose=verbose,
         )
 
-        # Step 5: If using S3 as repository, export necessary environment variables
-        if repo1_type.lower() == "s3":
-            required_env_vars = [
-                "PGBACKREST_REPO1_S3_KEY",
-                "PGBACKREST_REPO1_S3_BUCKET",
-                "PGBACKREST_REPO1_S3_KEY_SECRET",
-                "PGBACKREST_REPO1_CIPHER_PASS",
-            ]
-            missing_env_vars = [
-                var for var in required_env_vars if var not in os.environ
-            ]
-            if missing_env_vars:
-                util.exit_message(
-                    f"Environment variables {', '.join(missing_env_vars)} must be set for S3 pgBackRest configuration.",
-                    1,
-                )
-            s3_exports = " && ".join(
-                [f"export {var}={os.environ[var]}" for var in required_env_vars]
-            )
-            cmd_export_s3_source = (
-                f"cd {source_node_data['path']}/pgedge && {s3_exports}"
-            )
-            run_cmd(
-                cmd_export_s3_source,
-                node=source_node_data,
-                message="Setting S3 environment variables for pgBackRest",
-                verbose=verbose,
-            )
-
-        # Step 6: Set all pgBackRest backup configuration values for the source node.
+        # Step 5: Set all pgBackRest backup configuration values for the source node.
         compound_cmd = " && ".join(
             [
                 f"cd {source_node_data['path']}/pgedge",
@@ -1779,7 +1734,7 @@ def add_node(
             verbose=False,
         )
 
-        # Step 7: Create the pgBackRest stanza (this command uses --pg1-port because it connects to the DB)
+        # Step 6: Create the pgBackRest stanza (this command uses --pg1-port because it connects to the DB)
         cmd_create_stanza_source = (
             f"cd {source_node_data['path']}/pgedge && "
             f"./pgedge backrest command stanza-create "
@@ -1795,7 +1750,7 @@ def add_node(
             message=f"Creating pgBackRest stanza '{stanza_source}'",
             verbose=verbose,
         )
-        # Step 8: Initiate a full backup using pgBackRest (again, passing the port)
+        # Step 7: Initiate a full backup using pgBackRest (again, passing the port)
         backrest_backup_args_source = (
             f"--repo1-path {repo1_path_source} "
             f"--stanza {stanza_source} "
@@ -1828,17 +1783,17 @@ def add_node(
         source_backrest_cfg = {
             "stanza": stanza_source,
             "repo1_path": repo1_path_source,
-            "repo1-retention-full": repo1_retention_full,
-            "log-level-console": log_level_console,
-            "repo1-cipher-type": repo1_cipher_type,
+            "repo1_retention_full": repo1_retention_full,
+            "log_level_console": log_level_console,
+            "repo1_cipher_type": repo1_cipher_type,
             "repo1_type": repo1_type,
         }
 
     # For subsequent steps we extract pgbackrest settings from the source node configuration.
     stanza = source_backrest_cfg.get("stanza", f"pg{pg}")
-    repo1_retention_full = source_backrest_cfg.get("repo1-retention-full", "7")
-    log_level_console = source_backrest_cfg.get("log-level-console", "info")
-    repo1_cipher_type = source_backrest_cfg.get("repo1-cipher-type", "aes-256-cbc")
+    repo1_retention_full = source_backrest_cfg.get("repo1_retention_full", "7")
+    log_level_console = source_backrest_cfg.get("log_level_console", "info")
+    repo1_cipher_type = source_backrest_cfg.get("repo1_cipher_type", "aes-256-cbc")
     repo1_type = source_backrest_cfg.get("repo1_type", "posix")
 
     rc = ssh_install_pgedge(
@@ -1863,7 +1818,7 @@ def add_node(
     else:
         cmd = (
             f"{source_node_data['path']}/pgedge/pgedge backrest set-postgresqlconf {stanza} "
-            f"{pg1_path} {repo1_path} {repo1_type}"
+            f"{pg1_path} {repo1_path} {repo1_type} {repo1_cipher_type} "
         )
         message = f"Modifying postgresql.conf file"
         run_cmd(cmd, source_node_data, message=message, verbose=verbose)
@@ -1877,37 +1832,6 @@ def add_node(
         message = f"Reload configuration pg_reload_conf()"
         run_cmd(cmd, source_node_data, message=message, verbose=verbose)
 
-        if repo1_type == "s3":
-            for env_var in [
-                "PGBACKREST_REPO1_S3_KEY",
-                "PGBACKREST_REPO1_S3_BUCKET",
-                "PGBACKREST_REPO1_S3_KEY_SECRET",
-                "PGBACKREST_REPO1_CIPHER_PASS",
-            ]:
-                if env_var not in os.environ:
-                    util.exit_message(f"Environment variable {env_var} not set.")
-            s3_export_cmds = [
-                f"export {env_var}={os.environ[env_var]}"
-                for env_var in [
-                    "PGBACKREST_REPO1_S3_KEY",
-                    "PGBACKREST_REPO1_S3_BUCKET",
-                    "PGBACKREST_REPO1_S3_KEY_SECRET",
-                    "PGBACKREST_REPO1_CIPHER_PASS",
-                ]
-            ]
-            run_cmd(
-                " && ".join(s3_export_cmds),
-                source_node_data,
-                message="Setting S3 environment variables on source node",
-                verbose=verbose,
-            )
-            run_cmd(
-                " && ".join(s3_export_cmds),
-                new_node_data,
-                message="Setting S3 environment variables on target node",
-                verbose=verbose,
-            )
-
     cmd = f"{new_node_data['path']}/pgedge/pgedge install backrest"
     message = f"Installing pgbackrest"
     run_cmd(cmd, new_node_data, message=message, verbose=verbose)
@@ -2169,7 +2093,7 @@ def add_node(
         # Append the BACKUP settings to the target's PostgreSQL configuration
         cmd_set_postgresqlconf_target = (
             f"cd {pgedge_dir} && ./pgedge backrest set-postgresqlconf "
-            f"{target_stanza} {target_pg1_path} {target_repo1_path} {repo1_type}"
+            f"{target_stanza} {target_pg1_path} {target_repo1_path} {repo1_type} {repo1_cipher_type} "
         )
         run_cmd(
             cmd=cmd_set_postgresqlconf_target,
@@ -2227,15 +2151,15 @@ def add_node(
             message="Creating full pgBackRest backup",
             verbose=verbose,
         )
-    # else:
-    #     pgedge_dir = f"{new_node_data['path']}/pgedge"
-    #     cmd_remove_backrest_target = f"cd {pgedge_dir} && ./pgedge remove backrest"
-    #     run_cmd(
-    #         cmd=cmd_remove_backrest_target,
-    #         node=new_node_data,
-    #         message="Removing backrest from target node",
-    #         verbose=verbose,
-    #     )
+    else:
+        pgedge_dir = f"{new_node_data['path']}/pgedge"
+        cmd_remove_backrest_target = f"cd {pgedge_dir} && ./pgedge remove backrest"
+        run_cmd(
+            cmd=cmd_remove_backrest_target,
+            node=new_node_data,
+            message="Removing backrest from target node",
+            verbose=verbose,
+        )
 
     # Check and display pgBackRest configuration status in the source node
     # Remove unnecessary keys before appending new node to the cluster data
diff --git a/src/backrest/backrest.py b/src/backrest/backrest.py
index 52fcf823..f2414cd5 100755
--- a/src/backrest/backrest.py
+++ b/src/backrest/backrest.py
@@ -92,7 +92,7 @@ def create_stanza(stanza, verbose=True):
 
     # Modify postgresql.conf to ensure archiving is on
     modify_postgresql_conf(
-        stanza, config['pg1-path'], config['repo1-path'], config['repo1-type']
+        stanza, config['pg1-path'], config['repo1-path'], config['repo1-type'], config['repo1-cipher-type']
     )
     # Modify pg_hba.conf for replication, if needed
     modify_hba_conf()
@@ -407,12 +407,12 @@ def modify_hba_conf():
     }]
     util.update_pg_hba_conf(pgV(), new_rules)
 
-def modify_postgresql_conf(stanza, pg1_path, repo1_path, repo1_type):
+def modify_postgresql_conf(stanza, pg1_path, repo1_path, repo1_type, repo1_cipher_type="aes-256-cbc"):
     """Modify 'postgresql.conf' to integrate with pgbackrest."""
     aCmd = (
         f"pgbackrest --stanza={stanza} --pg1-path={pg1_path} "
         f"--repo1-type={repo1_type} --repo1-path={repo1_path} "
-        f"--repo1-cipher-type=aes-256-cbc archive-push %p"
+        f"--repo1-cipher-type={repo1_cipher_type} archive-push %p"
     )
     util.change_pgconf_keyval(pgV(), "archive_command", aCmd, p_replace=True)
     util.change_pgconf_keyval(pgV(), "archive_mode", "on", p_replace=True)

From fccbbd4aa7c29a5534349df64a163e30749c6b6f Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Tue, 6 May 2025 15:51:43 -0500
Subject: [PATCH 03/20] variable renames / fixes between source and target

---
 cli/scripts/cluster.py | 379 +++++++++++++++++++++--------------------
 1 file changed, 194 insertions(+), 185 deletions(-)

diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py
index f138a339..b9a180d5 100755
--- a/cli/scripts/cluster.py
+++ b/cli/scripts/cluster.py
@@ -1547,6 +1547,9 @@ def add_node(
         install (bool, optional): Whether to install pgEdge on the target node. Defaults to True.
     """
     db, db_settings, nodes = load_json(cluster_name)
+    db_name = db[0]["db_name"]
+    db_user = db[0]["db_user"]
+    db_password = db[0]["db_password"]
 
     cluster_data = get_cluster_json(cluster_name)
     if cluster_data is None:
@@ -1562,15 +1565,13 @@ def add_node(
 
     try:
         with open(target_node_file, "r") as f:
-            target_node_data = json.load(f)
-            json_validate_add_node(target_node_data)
+            target_node_json = json.load(f)
+            json_validate_add_node(target_node_json)
     except Exception as e:
         util.exit_message(
             f"Unable to load new node json def file '{target_node_file}\n{e}"
         )
 
-    # NEW: Check if target JSON has backrest configuration
-    target_backrest_cfg = target_node_data.get("backrest", {})
     # Retrieve source node data
     source_node_data = next(
         (node for node in nodes if node["name"] == source_node), None
@@ -1589,13 +1590,14 @@ def add_node(
             "Do not provide the repo1_path flag when the source node has it configured."
         )
 
-    for group in target_node_data.get("node_groups", []):
+    for group in target_node_json.get("node_groups", []):
         ssh_info = group.get("ssh")
         os_user = ssh_info.get("os_user", "")
         ssh_key = ssh_info.get("private_key", "")
 
-        new_node_data = {
+        target_node_data = {
             "ssh": ssh_info,
+            "backrest": group.get("backrest", {}),
             "name": group.get("name", ""),
             "is_active": group.get("is_active", ""),
             "public_ip": group.get("public_ip", ""),
@@ -1607,8 +1609,7 @@ def add_node(
         }
 
 
-
-    if "public_ip" not in new_node_data and "private_ip" not in new_node_data:
+    if "public_ip" not in target_node_data and "private_ip" not in target_node_data:
         util.exit_message(
             "Both public_ip and private_ip are missing in target node data."
         )
@@ -1620,18 +1621,23 @@ def add_node(
             "public_ip", source_node_data.get("private_ip")
         )
 
-    if "public_ip" in new_node_data and "private_ip" in new_node_data:
-        new_node_data["ip_address"] = new_node_data["public_ip"]
+    if "public_ip" in target_node_data and "private_ip" in target_node_data:
+        target_node_data["ip_address"] = target_node_data["public_ip"]
     else:
-        new_node_data["ip_address"] = new_node_data.get(
-            "public_ip", new_node_data.get("private_ip")
+        target_node_data["ip_address"] = target_node_data.get(
+            "public_ip", target_node_data.get("private_ip")
         )
 
-    # If backrest settings are provided in the JSON, add them to new_node_data.
-    if source_backrest_cfg:
-        new_node_data["backrest"] = source_backrest_cfg
-    else:
-        # Otherwise, we assume pgBackRest is not configured and should be installed.
+    # Log source and target node data
+    util.message(
+        f"Source node data: {json.dumps(source_node_data, indent=2)}", "info"
+    )
+    util.message(
+        f"Target node data: {json.dumps(target_node_data, indent=2)}", "info"
+    )
+
+    # If backrest is not configured on the source node, install it
+    if not source_backrest_cfg:
         # Step 1: Install pgBackRest on the source node
         cmd_install_backrest = (
             f"cd {source_node_data['path']}/pgedge && ./pgedge install backrest"
@@ -1648,41 +1654,41 @@ def add_node(
             f"### Configuring pgBackRest for node '{source_node_data['name']}'", "info"
         )
         # Create a unique stanza name using the cluster name and node name
-        stanza_source = f"{cluster_name}_stanza_{source_node_data['name']}"
+        source_stanza = f"{cluster_name}_stanza_{source_node_data['name']}"
 
         # Load additional pgBackRest settings with defaults.
-        repo1_retention_full = "7"
-        log_level_console = "info"
-        repo1_cipher_type = "aes-256-cbc"
-        repo1_type = "posix" 
+        source_repo1_retention_full = "7"
+        source_log_level_console = "info"
+        source_repo1_cipher_type = "aes-256-cbc"
+        source_repo1_type = "posix" 
 
         # Determine the repository path for the source node.
         if repo1_path:
             # Use the provided flag value as-is (trimmed of any trailing slash).
-            repo1_path_source = repo1_path.rstrip("/")
+            source_repo1_path = repo1_path.rstrip("/")
         else:
-            repo1_path_source = f"/var/lib/pgbackrest/{source_node_data['name']}"
+            source_repo1_path = f"/var/lib/pgbackrest/{source_node_data['name']}"
 
         # Similarly, set restore_path to include node name
-        restore_path_source = "/var/lib/pgbackrest_restore"
-        if not restore_path_source.rstrip("/").endswith(source_node_data["name"]):
-            restore_path_source = (
-                restore_path_source.rstrip("/") + f"/{source_node_data['name']}"
+        source_restore_path = "/var/lib/pgbackrest_restore"
+        if not source_restore_path.rstrip("/").endswith(source_node_data["name"]):
+            source_restore_path = (
+                source_restore_path.rstrip("/") + f"/{source_node_data['name']}"
             )
 
         pg_version = db_settings["pg_version"]
-        pg1_path_source = f"{source_node_data['path']}/pgedge/data/pg{pg_version}"
-        port_source = source_node_data["port"]
+        source_pg1_path = f"{source_node_data['path']}/pgedge/data/pg{pg_version}"
+        source_port = source_node_data["port"]
 
         # Step 2: Configure postgresql.conf for pgBackRest (without --pg1-port)
         cmd_set_postgresqlconf_source = (
             f"cd {source_node_data['path']}/pgedge && "
             f"./pgedge backrest set-postgresqlconf "
-            f"--stanza {stanza_source} "
-            f"--pg1-path {pg1_path_source} "
-            f"--repo1-path {repo1_path_source} "
-            f"--repo1-type {repo1_type} "
-            f"--repo1-cipher-type {repo1_cipher_type} "
+            f"--stanza {source_stanza} "
+            f"--pg1-path {source_pg1_path} "
+            f"--repo1-path {source_repo1_path} "
+            f"--repo1-type {source_repo1_type} "
+            f"--repo1-cipher-type {source_repo1_cipher_type} "
         )
         run_cmd(
             cmd_set_postgresqlconf_source,
@@ -1704,7 +1710,7 @@ def add_node(
 
         # Step 4: Reload PostgreSQL configuration to apply changes
         sql_reload_conf = "select pg_reload_conf()"
-        cmd_reload_conf_source = f"cd {source_node_data['path']}/pgedge && ./pgedge psql '{sql_reload_conf}' {db[0]['db_name']}"
+        cmd_reload_conf_source = f"cd {source_node_data['path']}/pgedge && ./pgedge psql '{sql_reload_conf}' {db_name}"
         run_cmd(
             cmd_reload_conf_source,
             node=source_node_data,
@@ -1716,13 +1722,13 @@ def add_node(
         compound_cmd = " && ".join(
             [
                 f"cd {source_node_data['path']}/pgedge",
-                f"./pgedge set BACKUP stanza {stanza_source}",
-                f"sudo mkdir -p {restore_path_source}",
-                f"./pgedge set BACKUP restore_path {restore_path_source}",
+                f"./pgedge set BACKUP stanza {source_stanza}",
+                f"sudo mkdir -p {source_restore_path}",
+                f"./pgedge set BACKUP restore_path {source_restore_path}",
                 f"./pgedge set BACKUP repo1-host-user {source_node_data.get('os_user', 'postgres')}",
-                f"./pgedge set BACKUP pg1-path {pg1_path_source}",
+                f"./pgedge set BACKUP pg1-path {source_pg1_path}",
                 f"./pgedge set BACKUP pg1-user {source_node_data.get('os_user', 'postgres')}",
-                f"./pgedge set BACKUP pg1-port {port_source}",
+                f"./pgedge set BACKUP pg1-port {source_port}",
             ]
         )
 
@@ -1738,29 +1744,29 @@ def add_node(
         cmd_create_stanza_source = (
             f"cd {source_node_data['path']}/pgedge && "
             f"./pgedge backrest command stanza-create "
-            f"--stanza '{stanza_source}' "
-            f"--pg1-path '{pg1_path_source}' "
-            f"--repo1-cipher-type {repo1_cipher_type} "
-            f"--pg1-port {port_source} "
-            f"--repo1-path {repo1_path_source}"
+            f"--stanza '{source_stanza}' "
+            f"--pg1-path '{source_pg1_path}' "
+            f"--repo1-cipher-type {source_repo1_cipher_type} "
+            f"--pg1-port {source_port} "
+            f"--repo1-path {source_repo1_path}"
         )
         run_cmd(
             cmd_create_stanza_source,
             node=source_node_data,
-            message=f"Creating pgBackRest stanza '{stanza_source}'",
+            message=f"Creating pgBackRest stanza '{source_stanza}'",
             verbose=verbose,
         )
         # Step 7: Initiate a full backup using pgBackRest (again, passing the port)
         backrest_backup_args_source = (
-            f"--repo1-path {repo1_path_source} "
-            f"--stanza {stanza_source} "
-            f"--pg1-path {pg1_path_source} "
-            f"--repo1-type {repo1_type} "
-            f"--log-level-console {log_level_console} "
-            f"--pg1-port {port_source} "
+            f"--repo1-path {source_repo1_path} "
+            f"--stanza {source_stanza} "
+            f"--pg1-path {source_pg1_path} "
+            f"--repo1-type {source_repo1_type} "
+            f"--log-level-console {source_log_level_console} "
+            f"--pg1-port {source_port} "
             f"--db-socket-path /tmp "
-            f"--repo1-cipher-type {repo1_cipher_type} "
-            f"--repo1-retention-full {repo1_retention_full} "
+            f"--repo1-cipher-type {source_repo1_cipher_type} "
+            f"--repo1-retention-full {source_repo1_retention_full} "
             f"--type=full"
         )
         cmd_create_backup_source = f"cd {source_node_data['path']}/pgedge && ./pgedge backrest command backup '{backrest_backup_args_source}'"
@@ -1771,54 +1777,52 @@ def add_node(
             verbose=verbose,
         )
         # (i) (Optional) Reset BACKUP repo1-path if needed
-        cmd_set_repo1_path_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP repo1-path {repo1_path_source}"
+        cmd_set_repo1_path_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP repo1-path {source_repo1_path}"
         run_cmd(
             cmd_set_repo1_path_source,
             node=source_node_data,
-            message=f"Setting BACKUP repo1-path to {repo1_path_source} on node '{source_node_data['name']}'",
+            message=f"Setting BACKUP repo1-path to {source_repo1_path} on node '{source_node_data['name']}'",
             verbose=verbose,
         )
 
         # Update source backrest config for further use downstream.
         source_backrest_cfg = {
-            "stanza": stanza_source,
-            "repo1_path": repo1_path_source,
-            "repo1_retention_full": repo1_retention_full,
-            "log_level_console": log_level_console,
-            "repo1_cipher_type": repo1_cipher_type,
-            "repo1_type": repo1_type,
+            "stanza": source_stanza,
+            "repo1_path": source_repo1_path,
+            "repo1_retention_full": source_repo1_retention_full,
+            "log_level_console": source_log_level_console,
+            "repo1_cipher_type": source_repo1_cipher_type,
+            "repo1_type": source_repo1_type,
         }
 
     # For subsequent steps we extract pgbackrest settings from the source node configuration.
-    stanza = source_backrest_cfg.get("stanza", f"pg{pg}")
-    repo1_retention_full = source_backrest_cfg.get("repo1_retention_full", "7")
-    log_level_console = source_backrest_cfg.get("log_level_console", "info")
-    repo1_cipher_type = source_backrest_cfg.get("repo1_cipher_type", "aes-256-cbc")
-    repo1_type = source_backrest_cfg.get("repo1_type", "posix")
+    source_stanza = source_backrest_cfg.get("stanza", "")
+    source_repo1_retention_full = source_backrest_cfg.get("repo1_retention_full", "7")
+    source_log_level_console = source_backrest_cfg.get("log_level_console", "info")
+    source_repo1_cipher_type = source_backrest_cfg.get("repo1_cipher_type", "aes-256-cbc")
+    source_repo1_type = source_backrest_cfg.get("repo1_type", "posix")
 
     rc = ssh_install_pgedge(
         cluster_name,
-        db[0]["db_name"],
+        db_name,
         db_settings,
-        db[0]["db_user"],
-        db[0]["db_password"],
-        [new_node_data],
+        db_user,
+        db_password,
+        [target_node_data],
         install,
         verbose,
     )
 
-    os_user = new_node_data["os_user"]
-    port = source_node_data["port"]
-    pg1_path = f"{source_node_data['path']}/pgedge/data/pg{pg}"
-
     if not repo1_path:
         # Do not install pgbackrest on source node; simply fetch the repo1_path from source's settings.
         repo1_path_default = f"/var/lib/pgbackrest/{source_node_data['name']}"
         repo1_path = source_backrest_cfg.get("repo1_path", f"{repo1_path_default}")
     else:
+        pg1_path = f"{source_node_data['path']}/pgedge/data/{pgV}"
+
         cmd = (
-            f"{source_node_data['path']}/pgedge/pgedge backrest set-postgresqlconf {stanza} "
-            f"{pg1_path} {repo1_path} {repo1_type} {repo1_cipher_type} "
+            f"{source_node_data['path']}/pgedge/pgedge backrest set-postgresqlconf {source_stanza} "
+            f"{pg1_path} {repo1_path} {source_repo1_type} {source_repo1_cipher_type} "
         )
         message = f"Modifying postgresql.conf file"
         run_cmd(cmd, source_node_data, message=message, verbose=verbose)
@@ -1828,59 +1832,59 @@ def add_node(
         run_cmd(cmd, source_node_data, message=message, verbose=verbose)
 
         sql_cmd = "select pg_reload_conf()"
-        cmd = f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}"
+        cmd = f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db_name}"
         message = f"Reload configuration pg_reload_conf()"
         run_cmd(cmd, source_node_data, message=message, verbose=verbose)
 
-    cmd = f"{new_node_data['path']}/pgedge/pgedge install backrest"
+    cmd = f"{target_node_data['path']}/pgedge/pgedge install backrest"
     message = f"Installing pgbackrest"
-    run_cmd(cmd, new_node_data, message=message, verbose=verbose)
+    run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
-    manage_node(new_node_data, "stop", f"pg{pg}", verbose)
-    cmd = f'rm -rf {new_node_data["path"]}/pgedge/data/pg{pg}'
+    manage_node(target_node_data, "stop", f"{pgV}", verbose)
+    cmd = f'rm -rf {target_node_data["path"]}/pgedge/data/{pgV}'
     message = f"Removing old data directory"
-    run_cmd(cmd, new_node_data, message=message, verbose=verbose)
+    run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
-    args = f"--repo1-path {repo1_path} --repo1-cipher-type {repo1_cipher_type} "
+    args = f"--repo1-path {repo1_path} --repo1-cipher-type {source_repo1_cipher_type} "
     if backup_id:
         args += f"--set={backup_id} "
 
     cmd = (
-        f'{new_node_data["path"]}/pgedge/pgedge backrest command restore '
-        f"--repo1-type={repo1_type} --stanza={stanza} --no-archive "
-        f'--pg1-path={new_node_data["path"]}/pgedge/data/pg{pg} {args}'
+        f'{target_node_data["path"]}/pgedge/pgedge backrest command restore '
+        f"--repo1-type={source_repo1_type} --stanza={source_stanza} --no-archive "
+        f'--pg1-path={target_node_data["path"]}/pgedge/data/{pgV} {args}'
     )
     message = f"Restoring backup"
-    run_cmd(cmd, new_node_data, message=message, verbose=verbose)
+    run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
-    pgd = f'{new_node_data["path"]}/pgedge/data/pg{pg}'
+    pgd = f'{target_node_data["path"]}/pgedge/data/{pgV}'
     pgc = f"{pgd}/postgresql.conf"
-    log_directory = f'{new_node_data["path"]}/pgedge/data/logs/pg{pg}'
+    log_directory = f'{target_node_data["path"]}/pgedge/data/logs/{pgV}'
 
     cmd = f"echo \"ssl_cert_file='{pgd}/server.crt'\" >> {pgc}"
     message = f"Setting ssl_cert_file"
-    run_cmd(cmd, new_node_data, message=message, verbose=verbose)
+    run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
     cmd = f"echo \"ssl_key_file='{pgd}/server.key'\" >> {pgc}"
     message = f"Setting ssl_key_file"
-    run_cmd(cmd, new_node_data, message=message, verbose=verbose)
+    run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
     cmd = f"echo \"log_directory='{log_directory}'\" >> {pgc}"
     message = f"Setting log_directory"
-    run_cmd(cmd, new_node_data, message=message, verbose=verbose)
+    run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
     cmd = (
-        f'{new_node_data["path"]}/pgedge/pgedge backrest configure-replica {stanza} '
-        f'{new_node_data["path"]}/pgedge/data/pg{pg} {source_node_data["ip_address"]} '
+        f'{target_node_data["path"]}/pgedge/pgedge backrest configure-replica {source_stanza} '
+        f'{target_node_data["path"]}/pgedge/data/{pgV} {source_node_data["ip_address"]} '
         f'{source_node_data["port"]} {source_node_data["os_user"]}'
     )
     message = f"Configuring PITR on replica"
-    run_cmd(cmd, new_node_data, message=message, verbose=verbose)
+    run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
     if script.strip() and os.path.isfile(script):
         util.echo_cmd(f"{script}")
 
-    terminate_cluster_transactions(nodes, db[0]["db_name"], f"pg{pg}", verbose)
+    terminate_cluster_transactions(nodes, db[0]["db_name"], f"{pgV}", verbose)
 
     spock = db_settings["spock_version"]
     v4 = True
@@ -1888,28 +1892,27 @@ def add_node(
     if spock:
         ver = [int(x) for x in spock.split(".")]
         spock_maj = ver[0]
-        spock_min = ver[1]
         if spock_maj >= 4:
             v4 = True
 
-    set_cluster_readonly(nodes, True, db[0]["db_name"], f"pg{pg}", v4, verbose)
-    manage_node(new_node_data, "start", f"pg{pg}", verbose)
+    set_cluster_readonly(nodes, True, db_name, f"{pgV}", v4, verbose)
+    manage_node(target_node_data, "start", f"{pgV}", verbose)
     time.sleep(5)
 
-    check_cluster_lag(new_node_data, db[0]["db_name"], f"pg{pg}", verbose)
+    check_cluster_lag(target_node_data, db_name, f"{pgV}", verbose)
 
     sql_cmd = "SELECT pg_promote()"
-    cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}"
+    cmd = f"{target_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db_name}"
     message = f"Promoting standby to primary"
-    run_cmd(cmd, new_node_data, message=message, verbose=verbose)
+    run_cmd(cmd, target_node_data, message=message, verbose=verbose)
     
     for mdb in db:
         sql_cmd = "SELECT sub_name FROM spock.subscription"
-        cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {mdb['db_name']}"
+        cmd = f"{target_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {mdb['db_name']}"
         message = "Fetch existing subscriptions"
         result = run_cmd(
             cmd,
-            node=new_node_data,
+            node=target_node_data,
             message=message,
             verbose=verbose,
             capture_output=True,
@@ -1924,18 +1927,18 @@ def add_node(
 
         if subscriptions:
             for sub_name in subscriptions:
-                cmd = f"{new_node_data['path']}/pgedge/pgedge spock sub-drop {sub_name} {mdb['db_name']}"
+                cmd = f"{target_node_data['path']}/pgedge/pgedge spock sub-drop {sub_name} {mdb['db_name']}"
                 message = f"Dropping old subscription {sub_name}"
-                run_cmd(cmd, node=new_node_data, message=message, verbose=verbose)
+                run_cmd(cmd, node=target_node_data, message=message, verbose=verbose)
         else:
             print("No subscriptions to drop.")
 
         sql_cmd = "SELECT node_name FROM spock.node"
-        cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {mdb['db_name']}"
+        cmd = f"{target_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {mdb['db_name']}"
         message = "Check if there are nodes"
         result = run_cmd(
             cmd,
-            node=new_node_data,
+            node=target_node_data,
             message=message,
             verbose=verbose,
             capture_output=True,
@@ -1951,41 +1954,41 @@ def add_node(
 
         if nodes_list:
             for node_name in nodes_list:
-                cmd = f"{new_node_data['path']}/pgedge/pgedge spock node-drop {node_name} {mdb['db_name']}"
+                cmd = f"{target_node_data['path']}/pgedge/pgedge spock node-drop {node_name} {mdb['db_name']}"
                 message = f"Dropping node {node_name}"
-                run_cmd(cmd, node=new_node_data, message=message, verbose=verbose)
+                run_cmd(cmd, node=target_node_data, message=message, verbose=verbose)
         else:
             print("No nodes to drop.")
 
-        create_node(new_node_data, mdb["db_name"], verbose)
+        create_node(target_node_data, mdb["db_name"], verbose)
 
         if not v4:
-            set_cluster_readonly(nodes, False, mdb["db_name"], f"pg{pg}", v4, verbose)
+            set_cluster_readonly(nodes, False, mdb["db_name"], f"{pgV}", v4, verbose)
 
-        create_sub(nodes, new_node_data, mdb["db_name"], verbose)
-        create_sub_new(nodes, new_node_data, mdb["db_name"], verbose)
+        create_sub(nodes, target_node_data, mdb["db_name"], verbose)
+        create_sub_new(nodes, target_node_data, mdb["db_name"], verbose)
 
-        nc = os.path.join(new_node_data["path"], "pgedge", "pgedge ")
+        nc = os.path.join(target_node_data["path"], "pgedge", "pgedge ")
         cmd = f'{nc} spock repset-add-table default "*" {mdb["db_name"]}'
         message = f"Adding all tables to repset"
-        run_cmd(cmd, new_node_data, message=message, verbose=verbose)
+        run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
         cmd = f'{nc} spock repset-add-table default_insert_only "*" {mdb["db_name"]}'
-        run_cmd(cmd, new_node_data, message=message, verbose=verbose)
+        run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
     if v4:
-        set_cluster_readonly(nodes, False, db[0]["db_name"], f"pg{pg}", v4, verbose)
+        set_cluster_readonly(nodes, False, db_name, f"{pgV}", v4, verbose)
 
-    cmd = f'cd {new_node_data["path"]}/pgedge/; ./pgedge spock node-list {db[0]["db_name"]}'
+    cmd = f'cd {target_node_data["path"]}/pgedge/; ./pgedge spock node-list {db_name}'
     message = f"Listing spock nodes"
     result = run_cmd(
-        cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True
+        cmd, node=target_node_data, message=message, verbose=verbose, capture_output=True
     )
     print(f"\n{result.stdout}")
 
     sql_cmd = "select node_id,node_name from spock.node"
     cmd = (
-        f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}"
+        f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db_name}"
     )
     message = f"List nodes"
     result = run_cmd(
@@ -2002,7 +2005,7 @@ def add_node(
             "select sub_id,sub_name,sub_enabled,sub_slot_name,"
             "sub_replication_sets from spock.subscription"
         )
-        cmd = f"{node['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}"
+        cmd = f"{node['path']}/pgedge/pgedge psql '{sql_cmd}' {db_name}"
         message = f"List subscriptions"
         result = run_cmd(
             cmd, node=node, message=message, verbose=verbose, capture_output=True
@@ -2013,10 +2016,10 @@ def add_node(
         "select sub_id,sub_name,sub_enabled,sub_slot_name,"
         "sub_replication_sets from spock.subscription"
     )
-    cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}"
+    cmd = f"{target_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db_name}"
     message = f"List subscriptions"
     result = run_cmd(
-        cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True
+        cmd, node=target_node_data, message=message, verbose=verbose, capture_output=True
     )
     print(f"\n{result.stdout}")
     
@@ -2026,104 +2029,110 @@ def add_node(
     sql_cmd = (
         'ALTER SYSTEM RESET restore_command'
     )
-    cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}"
+    cmd = f"{target_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db_name}"
     message = "Unsetting restore_command on target node"
-    run_cmd(cmd, node=new_node_data, message=message, verbose=verbose)
+    run_cmd(cmd, node=target_node_data, message=message, verbose=verbose)
 
     # Cleanup replica configuration
     cmd = (
-        f"cd {new_node_data['path']}/pgedge && "
+        f"cd {target_node_data['path']}/pgedge && "
         f"./pgedge backrest cleanup-replica "
-        f"--pg1-path {new_node_data['path']}/pgedge/data/pg{pg} "
+        f"--pg1-path {target_node_data['path']}/pgedge/data/{pgV} "
     )
-    run_cmd(cmd, node=new_node_data, message="Cleaning up replica configuration on target node", verbose=verbose)
+    run_cmd(cmd, node=target_node_data, message="Cleaning up replica configuration on target node", verbose=verbose)
 
+    target_backrest_cfg = target_node_data.get("backrest", {})
     if target_backrest_cfg:
-        repo1_path_target_file = (
-            target_node_data["node_groups"][0]
+        target_repo1_path = (
+            target_node_data
             .get("backrest", {})
             .get("repo1_path")
         )
-        target_stanza_target_file = (
-            target_node_data["node_groups"][0].get("backrest", {}).get("stanza")
+        target_stanza = (
+            target_node_data.get("backrest", {}).get("stanza")
         )
 
-        pgedge_dir = f"{new_node_data['path']}/pgedge"
-        restore_path = target_backrest_cfg.get(
-            "restore_path", f"/var/lib/pgbackrest_restore/{new_node_data['name']}"
+        target_pgedge_dir = f"{target_node_data['path']}/pgedge"
+        target_restore_path = target_backrest_cfg.get(
+            "restore_path", f"/var/lib/pgbackrest_restore/{target_node_data['name']}"
+        )
+        target_repo1_type = target_backrest_cfg.get(
+            "repo1_type", "posix"
+        )
+        target_repo1_cipher_type = target_backrest_cfg.get(
+            "repo1_cipher_type", "aes-256-cbc"
         )
         target_repo1_host_user = target_backrest_cfg.get(
-            "repo1_host_user", new_node_data.get("os_user", "postgres")
+            "repo1_host_user", target_node_data.get("os_user", "postgres")
         )
         target_pg1_path = target_backrest_cfg.get(
-            "pg1_path", f"{pgedge_dir}/data/pg{pg}"
+            "pg1_path", f"{target_pgedge_dir}/data/{pgV}"
         )
         target_pg1_user = target_backrest_cfg.get(
-            "pg1_user", new_node_data.get("os_user", "postgres")
+            "pg1_user", target_node_data.get("os_user", "postgres")
         )
         target_pg1_port = target_backrest_cfg.get(
-            "pg1_port", new_node_data.get("port", "6435")
+            "pg1_port", target_node_data.get("port", "6435")
+        )
+        target_log_level_console = target_backrest_cfg.get(
+            "log_level_console", "info"
         )
-
-        # Both repo1_path_target_file and target_stanza_target_file exist
-        target_repo1_path = repo1_path_target_file
-        target_stanza = target_stanza_target_file
 
         # Combined target node BACKUP configuration commands
         combined_target_cmd = (
-            f"sudo mkdir -p {restore_path} && "
-            f"sudo chown -R {target_repo1_host_user}:{target_repo1_host_user} {restore_path} && "
-            f"cd {pgedge_dir} && ./pgedge set BACKUP restore_path {restore_path} && "
+            f"sudo mkdir -p {target_restore_path} && "
+            f"sudo chown -R {target_repo1_host_user}:{target_repo1_host_user} {target_restore_path} && "
+            f"cd {target_pgedge_dir} && ./pgedge set BACKUP restore_path {target_restore_path} && "
             f"sudo mkdir -p {target_repo1_path} && "
             f"sudo chown -R {target_repo1_host_user}:{target_repo1_host_user} {target_repo1_path} && "
-            f"cd {pgedge_dir} && ./pgedge set BACKUP repo1-path {target_repo1_path} && "
-            f"cd {pgedge_dir} && ./pgedge set BACKUP repo1-host-user {target_repo1_host_user} && "
-            f"cd {pgedge_dir} && ./pgedge set BACKUP pg1-path {target_pg1_path} && "
-            f"cd {pgedge_dir} && ./pgedge set BACKUP pg1-user {target_pg1_user} && "
-            f"cd {pgedge_dir} && ./pgedge set BACKUP pg1-port {target_pg1_port} && "
-            f"cd {pgedge_dir} && ./pgedge set BACKUP stanza {target_stanza}"
+            f"cd {target_pgedge_dir} && ./pgedge set BACKUP repo1-path {target_repo1_path} && "
+            f"cd {target_pgedge_dir} && ./pgedge set BACKUP repo1-host-user {target_repo1_host_user} && "
+            f"cd {target_pgedge_dir} && ./pgedge set BACKUP pg1-path {target_pg1_path} && "
+            f"cd {target_pgedge_dir} && ./pgedge set BACKUP pg1-user {target_pg1_user} && "
+            f"cd {target_pgedge_dir} && ./pgedge set BACKUP pg1-port {target_pg1_port} && "
+            f"cd {target_pgedge_dir} && ./pgedge set BACKUP stanza {target_stanza}"
         )
         run_cmd(
             cmd=combined_target_cmd,
-            node=new_node_data,
+            node=target_node_data,
             message="Setting all target node BACKUP configuration",
             verbose=False,
         )
 
         # Append the BACKUP settings to the target's PostgreSQL configuration
         cmd_set_postgresqlconf_target = (
-            f"cd {pgedge_dir} && ./pgedge backrest set-postgresqlconf "
-            f"{target_stanza} {target_pg1_path} {target_repo1_path} {repo1_type} {repo1_cipher_type} "
+            f"cd {target_pgedge_dir} && ./pgedge backrest set-postgresqlconf "
+            f"{target_stanza} {target_pg1_path} {target_repo1_path} {target_repo1_type} {target_repo1_cipher_type} "
         )
         run_cmd(
             cmd=cmd_set_postgresqlconf_target,
-            node=new_node_data,
+            node=target_node_data,
             message="Appending BACKUP settings to postgresql.conf for target node",
             verbose=verbose,
         )
 
         # Restart PostgreSQL to apply the new configuration
-        cmd_restart_postgres = f"cd {pgedge_dir} && ./pgedge restart"
+        cmd_restart_postgres = f"cd {target_pgedge_dir} && ./pgedge restart"
         run_cmd(
             cmd=cmd_restart_postgres,
-            node=new_node_data,
+            node=target_node_data,
             message="Restarting PostgreSQL service",
             verbose=verbose,
         )
 
         # Now create the pgBackRest stanza on the target node
         cmd_create_stanza_target = (
-            f"cd {pgedge_dir} && "
+            f"cd {target_pgedge_dir} && "
             f"./pgedge backrest command stanza-create "
             f"--stanza '{target_stanza}' "
             f"--pg1-path '{target_pg1_path}' "
-            f"--repo1-cipher-type {repo1_cipher_type} "
+            f"--repo1-cipher-type {target_repo1_cipher_type} "
             f"--pg1-port {target_pg1_port} "
             f"--repo1-path {target_repo1_path}"
         )
         run_cmd(
             cmd=cmd_create_stanza_target,
-            node=new_node_data,
+            node=target_node_data,
             message=f"Creating pgBackRest stanza '{target_stanza}'",
             verbose=verbose,
         )
@@ -2133,48 +2142,48 @@ def add_node(
             f"--repo1-path {target_repo1_path} "
             f"--stanza {target_stanza} "
             f"--pg1-path {target_pg1_path} "
-            f"--repo1-type {repo1_type} "
-            f"--log-level-console {log_level_console} "
+            f"--repo1-type {target_repo1_type} "
+            f"--log-level-console {target_log_level_console} "
             f"--pg1-port {target_pg1_port} "
             f"--db-socket-path /tmp "
-            f"--repo1-cipher-type {repo1_cipher_type} "
-            f"--repo1-retention-full {repo1_retention_full} "
+            f"--repo1-cipher-type {target_repo1_cipher_type} "
+            f"--repo1-retention-full {target_repo1_cipher_type} "
             f"--type=full"
         )
         cmd_create_backup_target = (
-            f"cd {pgedge_dir} && ./pgedge backrest command backup "
+            f"cd {target_pgedge_dir} && ./pgedge backrest command backup "
             f"'{backrest_backup_args_target}'"
         )
         run_cmd(
             cmd=cmd_create_backup_target,
-            node=new_node_data,
+            node=target_node_data,
             message="Creating full pgBackRest backup",
             verbose=verbose,
         )
-    else:
-        pgedge_dir = f"{new_node_data['path']}/pgedge"
-        cmd_remove_backrest_target = f"cd {pgedge_dir} && ./pgedge remove backrest"
-        run_cmd(
-            cmd=cmd_remove_backrest_target,
-            node=new_node_data,
-            message="Removing backrest from target node",
-            verbose=verbose,
-        )
+    # else:
+    #     target_pgedge_dir = f"{target_node_data['path']}/pgedge"
+    #     cmd_remove_backrest_target = f"cd {target_pgedge_dir} && ./pgedge remove backrest"
+    #     run_cmd(
+    #         cmd=cmd_remove_backrest_target,
+    #         node=target_node_data,
+    #         message="Removing backrest from target node",
+    #         verbose=verbose,
+    #     )
 
     # Check and display pgBackRest configuration status in the source node
     # Remove unnecessary keys before appending new node to the cluster data
-    new_node_data.pop("ip_address", None)
-    new_node_data.pop("os_user", None)
-    new_node_data.pop("ssh_key", None)
+    target_node_data.pop("ip_address", None)
+    target_node_data.pop("os_user", None)
+    target_node_data.pop("ssh_key", None)
 
     # Append new node data to the cluster JSON
-    cluster_data["node_groups"].append(new_node_data)
+    cluster_data["node_groups"].append(target_node_data)
     cluster_data["update_date"] = datetime.datetime.now().astimezone().isoformat()
 
     write_cluster_json(cluster_name, cluster_data)
     if target_backrest_cfg:
-        capture_backrest_config(new_node_data, verbose=True)
-    check_source_backrest_config(source_node_data)
+        capture_backrest_config(target_node_data, verbose=True)
+    # check_source_backrest_config(source_node_data)
 
 def json_validate_add_node(data):
     """

From f8c335a25c18827ff6f45e97c7be4c464c1f5b16 Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Fri, 9 May 2025 14:18:07 -0500
Subject: [PATCH 04/20] small fixes

---
 cli/scripts/cluster.py   | 250 ++++++++++++++++++++-------------------
 src/backrest/backrest.py |   1 -
 2 files changed, 128 insertions(+), 123 deletions(-)

diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py
index b9a180d5..e41f5cd7 100755
--- a/cli/scripts/cluster.py
+++ b/cli/scripts/cluster.py
@@ -1470,9 +1470,8 @@ def init(cluster_name, install=True):
             cmd_set_pg1_port = f"cd {node['path']}/pgedge && ./pgedge set BACKUP repo1-path {repo1_path}"
             run_cmd(cmd_set_pg1_port, node=node, message=f"Setting BACKUP repo1-path to {repo1_path} on node '{node['name']}'", verbose=verbose)
 
-            for node in all_nodes:
-                if node.get("backrest", {}):
-                    capture_backrest_config(node, verbose=True)
+            if node.get("backrest", {}):
+                capture_backrest_config(node, verbose=True)
     
     # 6. If it's an HA cluster, handle Patroni/etcd, etc.
     if is_ha_cluster:
@@ -2043,132 +2042,141 @@ def add_node(
 
     target_backrest_cfg = target_node_data.get("backrest", {})
     if target_backrest_cfg:
-        target_repo1_path = (
-            target_node_data
-            .get("backrest", {})
-            .get("repo1_path")
-        )
-        target_stanza = (
-            target_node_data.get("backrest", {}).get("stanza")
-        )
+        print("noop")
+    #     target_repo1_path = (
+    #         target_node_data
+    #         .get("backrest", {})
+    #         .get("repo1_path")
+    #     )
+    #     target_stanza = (
+    #         target_node_data.get("backrest", {}).get("stanza")
+    #     )
 
-        target_pgedge_dir = f"{target_node_data['path']}/pgedge"
-        target_restore_path = target_backrest_cfg.get(
-            "restore_path", f"/var/lib/pgbackrest_restore/{target_node_data['name']}"
-        )
-        target_repo1_type = target_backrest_cfg.get(
-            "repo1_type", "posix"
-        )
-        target_repo1_cipher_type = target_backrest_cfg.get(
-            "repo1_cipher_type", "aes-256-cbc"
-        )
-        target_repo1_host_user = target_backrest_cfg.get(
-            "repo1_host_user", target_node_data.get("os_user", "postgres")
-        )
-        target_pg1_path = target_backrest_cfg.get(
-            "pg1_path", f"{target_pgedge_dir}/data/{pgV}"
-        )
-        target_pg1_user = target_backrest_cfg.get(
-            "pg1_user", target_node_data.get("os_user", "postgres")
-        )
-        target_pg1_port = target_backrest_cfg.get(
-            "pg1_port", target_node_data.get("port", "6435")
-        )
-        target_log_level_console = target_backrest_cfg.get(
-            "log_level_console", "info"
-        )
+    #     target_pgedge_dir = f"{target_node_data['path']}/pgedge"
+    #     target_restore_path = target_backrest_cfg.get(
+    #         "restore_path", f"/var/lib/pgbackrest_restore/{target_node_data['name']}"
+    #     )
+    #     target_repo1_type = target_backrest_cfg.get(
+    #         "repo1_type", "posix"
+    #     )
+    #     target_repo1_cipher_type = target_backrest_cfg.get(
+    #         "repo1_cipher_type", "aes-256-cbc"
+    #     )
+    #     target_repo1_retention_full = target_backrest_cfg.get(
+    #         "repo1_retention_full", "7"
+    #     )
+    #     target_repo1_host_user = target_backrest_cfg.get(
+    #         "repo1_host_user", target_node_data.get("os_user", "postgres")
+    #     )
+    #     target_pg1_path = target_backrest_cfg.get(
+    #         "pg1_path", f"{target_pgedge_dir}/data/{pgV}"
+    #     )
+    #     target_pg1_user = target_backrest_cfg.get(
+    #         "pg1_user", target_node_data.get("os_user", "postgres")
+    #     )
+    #     target_pg1_port = target_backrest_cfg.get(
+    #         "pg1_port", target_node_data.get("port", "6435")
+    #     )
+    #     target_log_level_console = target_backrest_cfg.get(
+    #         "log_level_console", "info"
+    #     )
 
-        # Combined target node BACKUP configuration commands
-        combined_target_cmd = (
-            f"sudo mkdir -p {target_restore_path} && "
-            f"sudo chown -R {target_repo1_host_user}:{target_repo1_host_user} {target_restore_path} && "
-            f"cd {target_pgedge_dir} && ./pgedge set BACKUP restore_path {target_restore_path} && "
-            f"sudo mkdir -p {target_repo1_path} && "
-            f"sudo chown -R {target_repo1_host_user}:{target_repo1_host_user} {target_repo1_path} && "
-            f"cd {target_pgedge_dir} && ./pgedge set BACKUP repo1-path {target_repo1_path} && "
-            f"cd {target_pgedge_dir} && ./pgedge set BACKUP repo1-host-user {target_repo1_host_user} && "
-            f"cd {target_pgedge_dir} && ./pgedge set BACKUP pg1-path {target_pg1_path} && "
-            f"cd {target_pgedge_dir} && ./pgedge set BACKUP pg1-user {target_pg1_user} && "
-            f"cd {target_pgedge_dir} && ./pgedge set BACKUP pg1-port {target_pg1_port} && "
-            f"cd {target_pgedge_dir} && ./pgedge set BACKUP stanza {target_stanza}"
-        )
-        run_cmd(
-            cmd=combined_target_cmd,
-            node=target_node_data,
-            message="Setting all target node BACKUP configuration",
-            verbose=False,
-        )
+    #     # Combined target node BACKUP configuration commands
+    #     combined_target_cmd = (
+    #         f"sudo mkdir -p {target_restore_path} && "
+    #         f"sudo chown -R {target_repo1_host_user}:{target_repo1_host_user} {target_restore_path} && "
+    #         f"cd {target_pgedge_dir} && ./pgedge set BACKUP restore_path {target_restore_path} && "
+    #         f"sudo mkdir -p {target_repo1_path} && "
+    #         f"sudo chown -R {target_repo1_host_user}:{target_repo1_host_user} {target_repo1_path} && "
+    #         f"cd {target_pgedge_dir} && ./pgedge set BACKUP repo1-path {target_repo1_path} && "
+    #         f"cd {target_pgedge_dir} && ./pgedge set BACKUP repo1-host-user {target_repo1_host_user} && "
+    #         f"cd {target_pgedge_dir} && ./pgedge set BACKUP pg1-path {target_pg1_path} && "
+    #         f"cd {target_pgedge_dir} && ./pgedge set BACKUP pg1-user {target_pg1_user} && "
+    #         f"cd {target_pgedge_dir} && ./pgedge set BACKUP pg1-port {target_pg1_port} && "
+    #         f"cd {target_pgedge_dir} && ./pgedge set BACKUP stanza {target_stanza}"
+    #     )
+    #     run_cmd(
+    #         cmd=combined_target_cmd,
+    #         node=target_node_data,
+    #         message="Setting all target node BACKUP configuration",
+    #         verbose=False,
+    #     )
 
-        # Append the BACKUP settings to the target's PostgreSQL configuration
-        cmd_set_postgresqlconf_target = (
-            f"cd {target_pgedge_dir} && ./pgedge backrest set-postgresqlconf "
-            f"{target_stanza} {target_pg1_path} {target_repo1_path} {target_repo1_type} {target_repo1_cipher_type} "
-        )
-        run_cmd(
-            cmd=cmd_set_postgresqlconf_target,
-            node=target_node_data,
-            message="Appending BACKUP settings to postgresql.conf for target node",
-            verbose=verbose,
-        )
+    #     # Append the BACKUP settings to the target's PostgreSQL configuration
+    #     cmd_set_postgresqlconf_target = (
+    #         f"cd {target_pgedge_dir} && ./pgedge backrest set-postgresqlconf "
+    #         f"{target_stanza} {target_pg1_path} {target_repo1_path} {target_repo1_type} {target_repo1_cipher_type} "
+    #     )
+    #     run_cmd(
+    #         cmd=cmd_set_postgresqlconf_target,
+    #         node=target_node_data,
+    #         message="Appending BACKUP settings to postgresql.conf for target node",
+    #         verbose=verbose,
+    #     )
 
-        # Restart PostgreSQL to apply the new configuration
-        cmd_restart_postgres = f"cd {target_pgedge_dir} && ./pgedge restart"
-        run_cmd(
-            cmd=cmd_restart_postgres,
-            node=target_node_data,
-            message="Restarting PostgreSQL service",
-            verbose=verbose,
-        )
+    #     # Restart PostgreSQL to apply the new configuration
+    #     cmd_restart_postgres = f"cd {target_pgedge_dir} && ./pgedge restart"
+    #     run_cmd(
+    #         cmd=cmd_restart_postgres,
+    #         node=target_node_data,
+    #         message="Restarting PostgreSQL service",
+    #         verbose=verbose,
+    #     )
 
-        # Now create the pgBackRest stanza on the target node
-        cmd_create_stanza_target = (
-            f"cd {target_pgedge_dir} && "
-            f"./pgedge backrest command stanza-create "
-            f"--stanza '{target_stanza}' "
-            f"--pg1-path '{target_pg1_path}' "
-            f"--repo1-cipher-type {target_repo1_cipher_type} "
-            f"--pg1-port {target_pg1_port} "
-            f"--repo1-path {target_repo1_path}"
-        )
-        run_cmd(
-            cmd=cmd_create_stanza_target,
-            node=target_node_data,
-            message=f"Creating pgBackRest stanza '{target_stanza}'",
-            verbose=verbose,
-        )
+    #     # Now create the pgBackRest stanza on the target node
+    #     cmd_create_stanza_target = (
+    #         f"cd {target_pgedge_dir} && "
+    #         f"./pgedge backrest command stanza-create "
+    #         f"--stanza '{target_stanza}' "
+    #         f"--pg1-path '{target_pg1_path}' "
+    #         f"--repo1-cipher-type {target_repo1_cipher_type} "
+    #         f"--pg1-port {target_pg1_port} "
+    #         f"--repo1-path {target_repo1_path}"
+    #     )
+    #     run_cmd(
+    #         cmd=cmd_create_stanza_target,
+    #         node=target_node_data,
+    #         message=f"Creating pgBackRest stanza '{target_stanza}'",
+    #         verbose=verbose,
+    #     )
 
-        # Create a full backup using pgBackRest
-        backrest_backup_args_target = (
-            f"--repo1-path {target_repo1_path} "
-            f"--stanza {target_stanza} "
-            f"--pg1-path {target_pg1_path} "
-            f"--repo1-type {target_repo1_type} "
-            f"--log-level-console {target_log_level_console} "
-            f"--pg1-port {target_pg1_port} "
-            f"--db-socket-path /tmp "
-            f"--repo1-cipher-type {target_repo1_cipher_type} "
-            f"--repo1-retention-full {target_repo1_cipher_type} "
-            f"--type=full"
-        )
-        cmd_create_backup_target = (
-            f"cd {target_pgedge_dir} && ./pgedge backrest command backup "
-            f"'{backrest_backup_args_target}'"
-        )
-        run_cmd(
-            cmd=cmd_create_backup_target,
-            node=target_node_data,
-            message="Creating full pgBackRest backup",
-            verbose=verbose,
-        )
-    # else:
-    #     target_pgedge_dir = f"{target_node_data['path']}/pgedge"
-    #     cmd_remove_backrest_target = f"cd {target_pgedge_dir} && ./pgedge remove backrest"
+    #     # Create a full backup using pgBackRest
+    #     backrest_backup_args_target = (
+    #         f"--repo1-path {target_repo1_path} "
+    #         f"--stanza {target_stanza} "
+    #         f"--pg1-path {target_pg1_path} "
+    #         f"--repo1-type {target_repo1_type} "
+    #         f"--log-level-console {target_log_level_console} "
+    #         f"--pg1-port {target_pg1_port} "
+    #         f"--db-socket-path /tmp "
+    #         f"--repo1-cipher-type {target_repo1_cipher_type} "
+    #         f"--repo1-retention-full {target_repo1_retention_full} "
+    #         f"--type=full"
+    #     )
+    #     cmd_create_backup_target = (
+    #         f"cd {target_pgedge_dir} && ./pgedge backrest command backup "
+    #         f"'{backrest_backup_args_target}'"
+    #     )
     #     run_cmd(
-    #         cmd=cmd_remove_backrest_target,
+    #         cmd=cmd_create_backup_target,
     #         node=target_node_data,
-    #         message="Removing backrest from target node",
+    #         message="Creating full pgBackRest backup",
     #         verbose=verbose,
     #     )
+    else:
+        target_pgedge_dir = f"{target_node_data['path']}/pgedge"
+        cmd_remove_backrest_target = f"cd {target_pgedge_dir} && ./pgedge remove backrest"
+        run_cmd(
+            cmd=cmd_remove_backrest_target,
+            node=target_node_data,
+            message="Removing backrest from target node",
+            verbose=verbose,
+        )
+
+    if target_backrest_cfg:
+        capture_backrest_config(target_node_data, verbose=True)
+
+    check_source_backrest_config(source_node_data)
 
     # Check and display pgBackRest configuration status in the source node
     # Remove unnecessary keys before appending new node to the cluster data
@@ -2181,9 +2189,7 @@ def add_node(
     cluster_data["update_date"] = datetime.datetime.now().astimezone().isoformat()
 
     write_cluster_json(cluster_name, cluster_data)
-    if target_backrest_cfg:
-        capture_backrest_config(target_node_data, verbose=True)
-    # check_source_backrest_config(source_node_data)
+    
 
 def json_validate_add_node(data):
     """
diff --git a/src/backrest/backrest.py b/src/backrest/backrest.py
index f2414cd5..2a805509 100755
--- a/src/backrest/backrest.py
+++ b/src/backrest/backrest.py
@@ -314,7 +314,6 @@ def configure_replica(stanza, pg1_path, pg1_host, pg1_port, pg1_user):
     changes = {
         "hot_standby": "on",
         "primary_conninfo": primary_conninfo,
-        "port": pg1_port,
         "archive_command": "cd .",
         "archive_mode": "on"
     }

From 10bf3763312fac9b88773e9bb6243faea1aae129 Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Tue, 13 May 2025 15:43:38 -0500
Subject: [PATCH 05/20] use --type=standby to ensure only standby.signal is
 written

---
 cli/scripts/cluster.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py
index e41f5cd7..a32bd0be 100755
--- a/cli/scripts/cluster.py
+++ b/cli/scripts/cluster.py
@@ -1844,13 +1844,13 @@ def add_node(
     message = f"Removing old data directory"
     run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
-    args = f"--repo1-path {repo1_path} --repo1-cipher-type {source_repo1_cipher_type} "
+    args = f"--repo1-path {repo1_path} --repo1-cipher-type {source_repo1_cipher_type} --type standby "
     if backup_id:
         args += f"--set={backup_id} "
 
     cmd = (
         f'{target_node_data["path"]}/pgedge/pgedge backrest command restore '
-        f"--repo1-type={source_repo1_type} --stanza={source_stanza} --no-archive "
+        f"--repo1-type={source_repo1_type} --stanza={source_stanza} "
         f'--pg1-path={target_node_data["path"]}/pgedge/data/{pgV} {args}'
     )
     message = f"Restoring backup"

From 131e8d4fcc0df1df937d6144b76ae81fab74c7f4 Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Tue, 13 May 2025 15:53:05 -0500
Subject: [PATCH 06/20] readd backrest reconfiguration code

---
 cli/scripts/cluster.py | 264 ++++++++++++++++++++---------------------
 1 file changed, 129 insertions(+), 135 deletions(-)

diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py
index a32bd0be..377ff6f3 100755
--- a/cli/scripts/cluster.py
+++ b/cli/scripts/cluster.py
@@ -1491,21 +1491,15 @@ def init(cluster_name, install=True):
                 ha_patroni.configure_patroni(node, sub_nodes, db[0], db_settings)
 
 
-def check_source_backrest_config(source_node_data):
+def remove_backrest(node_data):
     """
-    Check the source node's JSON data for a pgBackRest configuration.
-    If a non‑empty 'backrest' block is found, display its configuration.
-    Otherwise, display a message that no pgBackRest configuration exists,
-    and remove any leftover pgBackRest configuration.
+    Remove pgBackRest YAML from a node delegating to
+    `./pgedge remove backrest` (implemented in backrest.py).
+
+    The command is executed inside the node’s pgedge directory.
     """
-    if "backrest" in source_node_data and source_node_data["backrest"]:
-        util.message(
-            f"Source node '{source_node_data['name']}' already has pgBackRest configuration: {source_node_data['backrest']}",
-            "info"
-        )
-    else:
-        cmd = f"cd {source_node_data['path']}/pgedge && ./pgedge remove backrest"
-        run_cmd(cmd, node=source_node_data, message="Removing pgBackRest configuration from source node", verbose=True)
+    cmd = f"cd {node_data['path']}/pgedge && ./pgedge remove backrest"
+    run_cmd(cmd, node=node_data, message="Removing pgBackRest", verbose=True)
 
 
 def add_node(
@@ -2042,127 +2036,126 @@ def add_node(
 
     target_backrest_cfg = target_node_data.get("backrest", {})
     if target_backrest_cfg:
-        print("noop")
-    #     target_repo1_path = (
-    #         target_node_data
-    #         .get("backrest", {})
-    #         .get("repo1_path")
-    #     )
-    #     target_stanza = (
-    #         target_node_data.get("backrest", {}).get("stanza")
-    #     )
-
-    #     target_pgedge_dir = f"{target_node_data['path']}/pgedge"
-    #     target_restore_path = target_backrest_cfg.get(
-    #         "restore_path", f"/var/lib/pgbackrest_restore/{target_node_data['name']}"
-    #     )
-    #     target_repo1_type = target_backrest_cfg.get(
-    #         "repo1_type", "posix"
-    #     )
-    #     target_repo1_cipher_type = target_backrest_cfg.get(
-    #         "repo1_cipher_type", "aes-256-cbc"
-    #     )
-    #     target_repo1_retention_full = target_backrest_cfg.get(
-    #         "repo1_retention_full", "7"
-    #     )
-    #     target_repo1_host_user = target_backrest_cfg.get(
-    #         "repo1_host_user", target_node_data.get("os_user", "postgres")
-    #     )
-    #     target_pg1_path = target_backrest_cfg.get(
-    #         "pg1_path", f"{target_pgedge_dir}/data/{pgV}"
-    #     )
-    #     target_pg1_user = target_backrest_cfg.get(
-    #         "pg1_user", target_node_data.get("os_user", "postgres")
-    #     )
-    #     target_pg1_port = target_backrest_cfg.get(
-    #         "pg1_port", target_node_data.get("port", "6435")
-    #     )
-    #     target_log_level_console = target_backrest_cfg.get(
-    #         "log_level_console", "info"
-    #     )
-
-    #     # Combined target node BACKUP configuration commands
-    #     combined_target_cmd = (
-    #         f"sudo mkdir -p {target_restore_path} && "
-    #         f"sudo chown -R {target_repo1_host_user}:{target_repo1_host_user} {target_restore_path} && "
-    #         f"cd {target_pgedge_dir} && ./pgedge set BACKUP restore_path {target_restore_path} && "
-    #         f"sudo mkdir -p {target_repo1_path} && "
-    #         f"sudo chown -R {target_repo1_host_user}:{target_repo1_host_user} {target_repo1_path} && "
-    #         f"cd {target_pgedge_dir} && ./pgedge set BACKUP repo1-path {target_repo1_path} && "
-    #         f"cd {target_pgedge_dir} && ./pgedge set BACKUP repo1-host-user {target_repo1_host_user} && "
-    #         f"cd {target_pgedge_dir} && ./pgedge set BACKUP pg1-path {target_pg1_path} && "
-    #         f"cd {target_pgedge_dir} && ./pgedge set BACKUP pg1-user {target_pg1_user} && "
-    #         f"cd {target_pgedge_dir} && ./pgedge set BACKUP pg1-port {target_pg1_port} && "
-    #         f"cd {target_pgedge_dir} && ./pgedge set BACKUP stanza {target_stanza}"
-    #     )
-    #     run_cmd(
-    #         cmd=combined_target_cmd,
-    #         node=target_node_data,
-    #         message="Setting all target node BACKUP configuration",
-    #         verbose=False,
-    #     )
-
-    #     # Append the BACKUP settings to the target's PostgreSQL configuration
-    #     cmd_set_postgresqlconf_target = (
-    #         f"cd {target_pgedge_dir} && ./pgedge backrest set-postgresqlconf "
-    #         f"{target_stanza} {target_pg1_path} {target_repo1_path} {target_repo1_type} {target_repo1_cipher_type} "
-    #     )
-    #     run_cmd(
-    #         cmd=cmd_set_postgresqlconf_target,
-    #         node=target_node_data,
-    #         message="Appending BACKUP settings to postgresql.conf for target node",
-    #         verbose=verbose,
-    #     )
-
-    #     # Restart PostgreSQL to apply the new configuration
-    #     cmd_restart_postgres = f"cd {target_pgedge_dir} && ./pgedge restart"
-    #     run_cmd(
-    #         cmd=cmd_restart_postgres,
-    #         node=target_node_data,
-    #         message="Restarting PostgreSQL service",
-    #         verbose=verbose,
-    #     )
-
-    #     # Now create the pgBackRest stanza on the target node
-    #     cmd_create_stanza_target = (
-    #         f"cd {target_pgedge_dir} && "
-    #         f"./pgedge backrest command stanza-create "
-    #         f"--stanza '{target_stanza}' "
-    #         f"--pg1-path '{target_pg1_path}' "
-    #         f"--repo1-cipher-type {target_repo1_cipher_type} "
-    #         f"--pg1-port {target_pg1_port} "
-    #         f"--repo1-path {target_repo1_path}"
-    #     )
-    #     run_cmd(
-    #         cmd=cmd_create_stanza_target,
-    #         node=target_node_data,
-    #         message=f"Creating pgBackRest stanza '{target_stanza}'",
-    #         verbose=verbose,
-    #     )
-
-    #     # Create a full backup using pgBackRest
-    #     backrest_backup_args_target = (
-    #         f"--repo1-path {target_repo1_path} "
-    #         f"--stanza {target_stanza} "
-    #         f"--pg1-path {target_pg1_path} "
-    #         f"--repo1-type {target_repo1_type} "
-    #         f"--log-level-console {target_log_level_console} "
-    #         f"--pg1-port {target_pg1_port} "
-    #         f"--db-socket-path /tmp "
-    #         f"--repo1-cipher-type {target_repo1_cipher_type} "
-    #         f"--repo1-retention-full {target_repo1_retention_full} "
-    #         f"--type=full"
-    #     )
-    #     cmd_create_backup_target = (
-    #         f"cd {target_pgedge_dir} && ./pgedge backrest command backup "
-    #         f"'{backrest_backup_args_target}'"
-    #     )
-    #     run_cmd(
-    #         cmd=cmd_create_backup_target,
-    #         node=target_node_data,
-    #         message="Creating full pgBackRest backup",
-    #         verbose=verbose,
-    #     )
+        target_repo1_path = (
+            target_node_data
+            .get("backrest", {})
+            .get("repo1_path")
+        )
+        target_stanza = (
+            target_node_data.get("backrest", {}).get("stanza")
+        )
+
+        target_pgedge_dir = f"{target_node_data['path']}/pgedge"
+        target_restore_path = target_backrest_cfg.get(
+            "restore_path", f"/var/lib/pgbackrest_restore/{target_node_data['name']}"
+        )
+        target_repo1_type = target_backrest_cfg.get(
+            "repo1_type", "posix"
+        )
+        target_repo1_cipher_type = target_backrest_cfg.get(
+            "repo1_cipher_type", "aes-256-cbc"
+        )
+        target_repo1_retention_full = target_backrest_cfg.get(
+            "repo1_retention_full", "7"
+        )
+        target_repo1_host_user = target_backrest_cfg.get(
+            "repo1_host_user", target_node_data.get("os_user", "postgres")
+        )
+        target_pg1_path = target_backrest_cfg.get(
+            "pg1_path", f"{target_pgedge_dir}/data/{pgV}"
+        )
+        target_pg1_user = target_backrest_cfg.get(
+            "pg1_user", target_node_data.get("os_user", "postgres")
+        )
+        target_pg1_port = target_backrest_cfg.get(
+            "pg1_port", target_node_data.get("port", "6435")
+        )
+        target_log_level_console = target_backrest_cfg.get(
+            "log_level_console", "info"
+        )
+
+        # Combined target node BACKUP configuration commands
+        combined_target_cmd = (
+            f"sudo mkdir -p {target_restore_path} && "
+            f"sudo chown -R {target_repo1_host_user}:{target_repo1_host_user} {target_restore_path} && "
+            f"cd {target_pgedge_dir} && ./pgedge set BACKUP restore_path {target_restore_path} && "
+            f"sudo mkdir -p {target_repo1_path} && "
+            f"sudo chown -R {target_repo1_host_user}:{target_repo1_host_user} {target_repo1_path} && "
+            f"cd {target_pgedge_dir} && ./pgedge set BACKUP repo1-path {target_repo1_path} && "
+            f"cd {target_pgedge_dir} && ./pgedge set BACKUP repo1-host-user {target_repo1_host_user} && "
+            f"cd {target_pgedge_dir} && ./pgedge set BACKUP pg1-path {target_pg1_path} && "
+            f"cd {target_pgedge_dir} && ./pgedge set BACKUP pg1-user {target_pg1_user} && "
+            f"cd {target_pgedge_dir} && ./pgedge set BACKUP pg1-port {target_pg1_port} && "
+            f"cd {target_pgedge_dir} && ./pgedge set BACKUP stanza {target_stanza}"
+        )
+        run_cmd(
+            cmd=combined_target_cmd,
+            node=target_node_data,
+            message="Setting all target node BACKUP configuration",
+            verbose=False,
+        )
+
+        # Append the BACKUP settings to the target's PostgreSQL configuration
+        cmd_set_postgresqlconf_target = (
+            f"cd {target_pgedge_dir} && ./pgedge backrest set-postgresqlconf "
+            f"{target_stanza} {target_pg1_path} {target_repo1_path} {target_repo1_type} {target_repo1_cipher_type} "
+        )
+        run_cmd(
+            cmd=cmd_set_postgresqlconf_target,
+            node=target_node_data,
+            message="Appending BACKUP settings to postgresql.conf for target node",
+            verbose=verbose,
+        )
+
+        # Restart PostgreSQL to apply the new configuration
+        cmd_restart_postgres = f"cd {target_pgedge_dir} && ./pgedge restart"
+        run_cmd(
+            cmd=cmd_restart_postgres,
+            node=target_node_data,
+            message="Restarting PostgreSQL service",
+            verbose=verbose,
+        )
+
+        # Now create the pgBackRest stanza on the target node
+        cmd_create_stanza_target = (
+            f"cd {target_pgedge_dir} && "
+            f"./pgedge backrest command stanza-create "
+            f"--stanza '{target_stanza}' "
+            f"--pg1-path '{target_pg1_path}' "
+            f"--repo1-cipher-type {target_repo1_cipher_type} "
+            f"--pg1-port {target_pg1_port} "
+            f"--repo1-path {target_repo1_path}"
+        )
+        run_cmd(
+            cmd=cmd_create_stanza_target,
+            node=target_node_data,
+            message=f"Creating pgBackRest stanza '{target_stanza}'",
+            verbose=verbose,
+        )
+
+        # Create a full backup using pgBackRest
+        backrest_backup_args_target = (
+            f"--repo1-path {target_repo1_path} "
+            f"--stanza {target_stanza} "
+            f"--pg1-path {target_pg1_path} "
+            f"--repo1-type {target_repo1_type} "
+            f"--log-level-console {target_log_level_console} "
+            f"--pg1-port {target_pg1_port} "
+            f"--db-socket-path /tmp "
+            f"--repo1-cipher-type {target_repo1_cipher_type} "
+            f"--repo1-retention-full {target_repo1_retention_full} "
+            f"--type=full"
+        )
+        cmd_create_backup_target = (
+            f"cd {target_pgedge_dir} && ./pgedge backrest command backup "
+            f"'{backrest_backup_args_target}'"
+        )
+        run_cmd(
+            cmd=cmd_create_backup_target,
+            node=target_node_data,
+            message="Creating full pgBackRest backup",
+            verbose=verbose,
+        )
     else:
         target_pgedge_dir = f"{target_node_data['path']}/pgedge"
         cmd_remove_backrest_target = f"cd {target_pgedge_dir} && ./pgedge remove backrest"
@@ -2176,7 +2169,8 @@ def add_node(
     if target_backrest_cfg:
         capture_backrest_config(target_node_data, verbose=True)
 
-    check_source_backrest_config(source_node_data)
+    if not source_backrest_cfg:
+        remove_backrest(source_node_data)
 
     # Check and display pgBackRest configuration status in the source node
     # Remove unnecessary keys before appending new node to the cluster data

From dfab95a90b3ea7914e3b63717c3068f623e6d741 Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Tue, 13 May 2025 15:53:15 -0500
Subject: [PATCH 07/20] remove temporary logging

---
 cli/scripts/cluster.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py
index 377ff6f3..45c74355 100755
--- a/cli/scripts/cluster.py
+++ b/cli/scripts/cluster.py
@@ -1621,14 +1621,6 @@ def add_node(
             "public_ip", target_node_data.get("private_ip")
         )
 
-    # Log source and target node data
-    util.message(
-        f"Source node data: {json.dumps(source_node_data, indent=2)}", "info"
-    )
-    util.message(
-        f"Target node data: {json.dumps(target_node_data, indent=2)}", "info"
-    )
-
     # If backrest is not configured on the source node, install it
     if not source_backrest_cfg:
         # Step 1: Install pgBackRest on the source node

From f6c9a0a1bb4ca80c7a903cd8a0e830e3c26e035d Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Tue, 13 May 2025 15:55:30 -0500
Subject: [PATCH 08/20] align pgbackrest secret in compose setup

---
 devel/setup/compose/Dockerfile.node.rocky95    | 2 +-
 devel/setup/compose/Dockerfile.node.ubuntu2204 | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/devel/setup/compose/Dockerfile.node.rocky95 b/devel/setup/compose/Dockerfile.node.rocky95
index 6d5abf00..7cef16b6 100644
--- a/devel/setup/compose/Dockerfile.node.rocky95
+++ b/devel/setup/compose/Dockerfile.node.rocky95
@@ -31,7 +31,7 @@ RUN chmod 700 /home/pgedge/.ssh/
 RUN sudo chown pgedge:pgedge /home/pgedge/.ssh/id_rsa
 RUN sudo chmod 600 /home/pgedge/.ssh/id_rsa
 
-RUN sed -i '1iexport PGBACKREST_REPO1_CIPHER_PASS=Really_s3cure_password' ~/.bashrc
+RUN sed -i '1iexport PGBACKREST_REPO1_CIPHER_PASS=supersecret' ~/.bashrc
 
 USER root
 EXPOSE 22
diff --git a/devel/setup/compose/Dockerfile.node.ubuntu2204 b/devel/setup/compose/Dockerfile.node.ubuntu2204
index 6cc7ba23..204679e4 100644
--- a/devel/setup/compose/Dockerfile.node.ubuntu2204
+++ b/devel/setup/compose/Dockerfile.node.ubuntu2204
@@ -31,7 +31,7 @@ RUN chmod 700 /home/pgedge/.ssh/
 RUN sudo chown pgedge:pgedge /home/pgedge/.ssh/id_rsa
 RUN sudo chmod 600 /home/pgedge/.ssh/id_rsa
 
-RUN echo 'export PGBACKREST_REPO1_CIPHER_PASS=Really_s3cure_password' >> ~/.bashrc
+RUN echo 'export PGBACKREST_REPO1_CIPHER_PASS=supersecret' >> ~/.bashrc
 
 USER root
 EXPOSE 22

From 31f17a7e50863884917ec3685541fd28b1b88205 Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Tue, 13 May 2025 15:55:45 -0500
Subject: [PATCH 09/20] add additional deps to build container

---
 devel/setup/Dockerfile.rocky95 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/devel/setup/Dockerfile.rocky95 b/devel/setup/Dockerfile.rocky95
index e31b5823..aa094562 100644
--- a/devel/setup/Dockerfile.rocky95
+++ b/devel/setup/Dockerfile.rocky95
@@ -8,7 +8,7 @@ USER root
 ENV install="dnf install -y --allowerasing"
 RUN $install dnf-plugins-core
 RUN $install python3 python3-pip git wget curl pigz which zip sqlite
-RUN $install openssh-server systemd sudo
+RUN $install openssh-server systemd sudo inotify-tools lsof
 
 RUN useradd build  -U -m -d /home/build  \
     && echo "build  ALL=(ALL)  NOPASSWD: ALL" >> /etc/sudoers

From e6da7fe153e17ba3fbffd9c07612f5005c19f375 Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Wed, 14 May 2025 14:06:49 -0500
Subject: [PATCH 10/20] adjustments to backrest args to resolve add node issues

---
 cli/scripts/cluster.py | 120 ++++++++++++++++++++++-------------------
 1 file changed, 64 insertions(+), 56 deletions(-)

diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py
index 45c74355..7aea611c 100755
--- a/cli/scripts/cluster.py
+++ b/cli/scripts/cluster.py
@@ -1443,25 +1443,25 @@ def init(cluster_name, install=True):
             cmd_create_stanza = (
                 f"cd {node['path']}/pgedge && "
                 f"./pgedge backrest command stanza-create "
-                f"--stanza '{stanza}' "
-                f"--pg1-path '{pg1_path}' "
-                f"--repo1-cipher-type {repo1_cipher_type} "
-                f"--pg1-port {port} "
-                f"--repo1-path {repo1_path}"
+                f"--stanza='{stanza}' "
+                f"--pg1-path='{pg1_path}' "
+                f"--repo1-cipher-type={repo1_cipher_type} "
+                f"--pg1-port={port} "
+                f"--repo1-path={repo1_path}"
             )
             run_cmd(cmd_create_stanza, node=node, message=f"Creating pgBackRest stanza '{stanza}'", verbose=verbose)
     
             # -- Step 7: Initiate a full backup using pgBackRest (again, passing the port)
             backrest_backup_args = (
-                f"--repo1-path {repo1_path} "
-                f"--stanza {stanza} "
-                f"--pg1-path {pg1_path} "
-                f"--repo1-type {repo1_type} "
-                f"--log-level-console {log_level_console} "
-                f"--pg1-port {port} "
-                f"--db-socket-path /tmp "
-                f"--repo1-cipher-type {repo1_cipher_type} "
-                f"--repo1-retention-full {repo1_retention_full} "
+                f"--repo1-path={repo1_path} "
+                f"--stanza={stanza} "
+                f"--pg1-path={pg1_path} "
+                f"--repo1-type={repo1_type} "
+                f"--log-level-console={log_level_console} "
+                f"--pg1-port={port} "
+                f"--db-socket-path=/tmp "
+                f"--repo1-cipher-type={repo1_cipher_type} "
+                f"--repo1-retention-full={repo1_retention_full} "
                 f"--type=full"
             )
             cmd_create_backup = f"cd {node['path']}/pgedge && ./pgedge backrest command backup '{backrest_backup_args}'"
@@ -1550,7 +1550,7 @@ def add_node(
     pg = db_settings["pg_version"]
     pgV = f"pg{pg}"
     verbose = cluster_data.get("log_level", "info")
-    
+
     # Load and validate the target node JSON
     target_node_file = f"{target_node}.json"
     if not os.path.isfile(target_node_file):
@@ -1601,7 +1601,6 @@ def add_node(
             "ssh_key": ssh_key,
         }
 
-
     if "public_ip" not in target_node_data and "private_ip" not in target_node_data:
         util.exit_message(
             "Both public_ip and private_ip are missing in target node data."
@@ -1729,11 +1728,11 @@ def add_node(
         cmd_create_stanza_source = (
             f"cd {source_node_data['path']}/pgedge && "
             f"./pgedge backrest command stanza-create "
-            f"--stanza '{source_stanza}' "
-            f"--pg1-path '{source_pg1_path}' "
-            f"--repo1-cipher-type {source_repo1_cipher_type} "
-            f"--pg1-port {source_port} "
-            f"--repo1-path {source_repo1_path}"
+            f"--stanza='{source_stanza}' "
+            f"--pg1-path='{source_pg1_path}' "
+            f"--repo1-cipher-type={source_repo1_cipher_type} "
+            f"--pg1-port={source_port} "
+            f"--repo1-path={source_repo1_path}"
         )
         run_cmd(
             cmd_create_stanza_source,
@@ -1743,15 +1742,15 @@ def add_node(
         )
         # Step 7: Initiate a full backup using pgBackRest (again, passing the port)
         backrest_backup_args_source = (
-            f"--repo1-path {source_repo1_path} "
-            f"--stanza {source_stanza} "
-            f"--pg1-path {source_pg1_path} "
-            f"--repo1-type {source_repo1_type} "
-            f"--log-level-console {source_log_level_console} "
-            f"--pg1-port {source_port} "
-            f"--db-socket-path /tmp "
-            f"--repo1-cipher-type {source_repo1_cipher_type} "
-            f"--repo1-retention-full {source_repo1_retention_full} "
+            f"--repo1-path={source_repo1_path} "
+            f"--stanza={source_stanza} "
+            f"--pg1-path={source_pg1_path} "
+            f"--repo1-type={source_repo1_type} "
+            f"--log-level-console={source_log_level_console} "
+            f"--pg1-port={source_port} "
+            f"--db-socket-path=/tmp "
+            f"--repo1-cipher-type={source_repo1_cipher_type} "
+            f"--repo1-retention-full={source_repo1_retention_full} "
             f"--type=full"
         )
         cmd_create_backup_source = f"cd {source_node_data['path']}/pgedge && ./pgedge backrest command backup '{backrest_backup_args_source}'"
@@ -1830,15 +1829,24 @@ def add_node(
     message = f"Removing old data directory"
     run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
-    args = f"--repo1-path {repo1_path} --repo1-cipher-type {source_repo1_cipher_type} --type standby "
+    restore_args = (
+        f'--cmd="pgbackrest --repo1-cipher-type={source_repo1_cipher_type}" '
+        f"--stanza={source_stanza} "
+        f"--pg1-path={target_node_data['path']}/pgedge/data/{pgV} "
+        f"--repo1-path={repo1_path} "
+        f"--repo1-cipher-type={source_repo1_cipher_type} "
+        f"--repo1-type={source_repo1_type} "
+        "--type=standby "
+    )
+
     if backup_id:
-        args += f"--set={backup_id} "
+        restore_args.append(f"--set={backup_id}")
 
     cmd = (
         f'{target_node_data["path"]}/pgedge/pgedge backrest command restore '
-        f"--repo1-type={source_repo1_type} --stanza={source_stanza} "
-        f'--pg1-path={target_node_data["path"]}/pgedge/data/{pgV} {args}'
+        f"'{restore_args}'"
     )
+
     message = f"Restoring backup"
     run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
@@ -1846,15 +1854,15 @@ def add_node(
     pgc = f"{pgd}/postgresql.conf"
     log_directory = f'{target_node_data["path"]}/pgedge/data/logs/{pgV}'
 
-    cmd = f"echo \"ssl_cert_file='{pgd}/server.crt'\" >> {pgc}"
+    cmd = f"sed -i \"/^ssl_cert_file/d\" {pgc} && echo \"ssl_cert_file = '{pgd}/server.crt'\" >> {pgc}"
     message = f"Setting ssl_cert_file"
     run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
-    cmd = f"echo \"ssl_key_file='{pgd}/server.key'\" >> {pgc}"
+    cmd = f"sed -i \"/^ssl_key_file/d\" {pgc} && echo \"ssl_key_file = '{pgd}/server.key'\" >> {pgc}"
     message = f"Setting ssl_key_file"
     run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
-    cmd = f"echo \"log_directory='{log_directory}'\" >> {pgc}"
+    cmd = f"sed -i \"/^log_directory/d\" {pgc} && echo \"log_directory = '{log_directory}'\" >> {pgc}"
     message = f"Setting log_directory"
     run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
@@ -1890,7 +1898,7 @@ def add_node(
     cmd = f"{target_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db_name}"
     message = f"Promoting standby to primary"
     run_cmd(cmd, target_node_data, message=message, verbose=verbose)
-    
+
     for mdb in db:
         sql_cmd = "SELECT sub_name FROM spock.subscription"
         cmd = f"{target_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {mdb['db_name']}"
@@ -2007,7 +2015,7 @@ def add_node(
         cmd, node=target_node_data, message=message, verbose=verbose, capture_output=True
     )
     print(f"\n{result.stdout}")
-    
+
     # A subsequent restart will be needed to apply the changes
     # This will occur in the next section when pgBackrest is configured or removed
     # Cleanup restore remnants by unsetting restore_command on target node
@@ -2015,7 +2023,7 @@ def add_node(
         'ALTER SYSTEM RESET restore_command'
     )
     cmd = f"{target_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db_name}"
-    message = "Unsetting restore_command on target node"
+    message = "Unsetting restore_command"
     run_cmd(cmd, node=target_node_data, message=message, verbose=verbose)
 
     # Cleanup replica configuration
@@ -2024,7 +2032,7 @@ def add_node(
         f"./pgedge backrest cleanup-replica "
         f"--pg1-path {target_node_data['path']}/pgedge/data/{pgV} "
     )
-    run_cmd(cmd, node=target_node_data, message="Cleaning up replica configuration on target node", verbose=verbose)
+    run_cmd(cmd, node=target_node_data, message="Cleaning up replica configuration", verbose=verbose)
 
     target_backrest_cfg = target_node_data.get("backrest", {})
     if target_backrest_cfg:
@@ -2112,11 +2120,11 @@ def add_node(
         cmd_create_stanza_target = (
             f"cd {target_pgedge_dir} && "
             f"./pgedge backrest command stanza-create "
-            f"--stanza '{target_stanza}' "
-            f"--pg1-path '{target_pg1_path}' "
-            f"--repo1-cipher-type {target_repo1_cipher_type} "
-            f"--pg1-port {target_pg1_port} "
-            f"--repo1-path {target_repo1_path}"
+            f"--stanza='{target_stanza}' "
+            f"--pg1-path='{target_pg1_path}' "
+            f"--repo1-cipher-type={target_repo1_cipher_type} "
+            f"--pg1-port={target_pg1_port} "
+            f"--repo1-path={target_repo1_path}"
         )
         run_cmd(
             cmd=cmd_create_stanza_target,
@@ -2127,15 +2135,15 @@ def add_node(
 
         # Create a full backup using pgBackRest
         backrest_backup_args_target = (
-            f"--repo1-path {target_repo1_path} "
-            f"--stanza {target_stanza} "
-            f"--pg1-path {target_pg1_path} "
-            f"--repo1-type {target_repo1_type} "
-            f"--log-level-console {target_log_level_console} "
-            f"--pg1-port {target_pg1_port} "
-            f"--db-socket-path /tmp "
-            f"--repo1-cipher-type {target_repo1_cipher_type} "
-            f"--repo1-retention-full {target_repo1_retention_full} "
+            f"--repo1-path={target_repo1_path} "
+            f"--stanza={target_stanza} "
+            f"--pg1-path={target_pg1_path} "
+            f"--repo1-type={target_repo1_type} "
+            f"--log-level-console={target_log_level_console} "
+            f"--pg1-port={target_pg1_port} "
+            f"--db-socket-path=/tmp "
+            f"--repo1-cipher-type={target_repo1_cipher_type} "
+            f"--repo1-retention-full={target_repo1_retention_full} "
             f"--type=full"
         )
         cmd_create_backup_target = (
@@ -2175,7 +2183,7 @@ def add_node(
     cluster_data["update_date"] = datetime.datetime.now().astimezone().isoformat()
 
     write_cluster_json(cluster_name, cluster_data)
-    
+
 
 def json_validate_add_node(data):
     """

From aca565890b8096347ae2409177b3be626b93f040 Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Wed, 14 May 2025 14:15:10 -0500
Subject: [PATCH 11/20] fix ascii chars

---
 cli/scripts/cluster.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py
index 7aea611c..e8990fc6 100755
--- a/cli/scripts/cluster.py
+++ b/cli/scripts/cluster.py
@@ -2187,21 +2187,21 @@ def add_node(
 
 def json_validate_add_node(data):
     """
-    Validate the structure of a node‑definition JSON file that will be fed to
-    the add‑node command.
+    Validate the structure of a node-definition JSON file that will be fed to
+    the add-node command.
 
     • The traditional checks (json_version, ssh, port, …) still apply.
     • A node_group is not required to have a “backrest” block.
     • If a “backrest” block is present, it must contain at least:
-         • stanza        – unique stanza name
-         • repo1_path    – absolute path to the repo directory
-         • repo1_type    – 'posix' or 's3'
-       and the values must be non‑empty and valid.
+         • stanza        - unique stanza name
+         • repo1_path    - absolute path to the repo directory
+         • repo1_type    - 'posix' or 's3'
+       and the values must be non-empty and valid.
     """
 
     required_top = {"json_version", "node_groups"}
     if not required_top.issubset(data):
-        util.exit_message("Invalid add‑node JSON: missing json_version or node_groups.")
+        util.exit_message("Invalid add-node JSON: missing json_version or node_groups.")
 
     if str(data.get("json_version")) != "1.0":
         util.exit_message("Invalid or unsupported json_version (must be '1.0').")
@@ -2227,7 +2227,7 @@ def json_validate_add_node(data):
         missing_basic = node_group_required - set(group.keys())
         if missing_basic:
             util.exit_message(
-                f"Node‑group '{gname}' missing keys: {', '.join(missing_basic)}"
+                f"Node-group '{gname}' missing keys: {', '.join(missing_basic)}"
             )
 
         # ssh block
@@ -2235,7 +2235,7 @@ def json_validate_add_node(data):
         missing_ssh = ssh_required - set(ssh_info.keys())
         if missing_ssh:
             util.exit_message(
-                f"SSH block in node‑group '{gname}' missing: {', '.join(missing_ssh)}"
+                f"SSH block in node-group '{gname}' missing: {', '.join(missing_ssh)}"
             )
 
         # backrest (optional but validated if present)
@@ -2246,24 +2246,24 @@ def json_validate_add_node(data):
             missing_br = backrest_required - set(br.keys())
             if missing_br:
                 util.exit_message(
-                    f"pgBackRest block in node‑group '{gname}' missing: {', '.join(missing_br)}"
+                    f"pgBackRest block in node-group '{gname}' missing: {', '.join(missing_br)}"
                 )
 
-            # ensure values are non‑empty
+            # ensure values are non-empty
             for k in backrest_required:
                 if not str(br[k]).strip():
                     util.exit_message(
-                        f"pgBackRest key '{k}' in node‑group '{gname}' cannot be empty."
+                        f"pgBackRest key '{k}' in node-group '{gname}' cannot be empty."
                     )
 
             # verify repo1_type is valid
             if br["repo1_type"] not in valid_repo1_types:
                 util.exit_message(
-                    f"Invalid repo1_type '{br['repo1_type']}' in node‑group '{gname}'. "
+                    f"Invalid repo1_type '{br['repo1_type']}' in node-group '{gname}'. "
                     f"Allowed: {', '.join(valid_repo1_types)}"
                 )
 
-    util.message("✔ add‑node JSON structure is valid.", "success")
+    util.message("✔ add-node JSON structure is valid.", "success")
 
 def remove_node(cluster_name, node_name):
     """

From 091b8e26c85204f5b66f466d678a3e87a52e997f Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Wed, 14 May 2025 14:18:48 -0500
Subject: [PATCH 12/20] formatting

---
 cli/scripts/cluster.py | 254 +++++++++++++++++++++++++++--------------
 1 file changed, 171 insertions(+), 83 deletions(-)

diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py
index e8990fc6..55499966 100755
--- a/cli/scripts/cluster.py
+++ b/cli/scripts/cluster.py
@@ -1273,12 +1273,13 @@ def capture_backrest_config(node, verbose=False):
         verbose=verbose,
     )
 
+
 def init(cluster_name, install=True):
     """
     Initialize a cluster via cluster configuration JSON file.
 
     Initialize a cluster via cluster configuration JSON file by performing the following steps:
-    
+
         1. Loads the cluster configuration.
         2. Checks SSH connectivity for all nodes.
         3. Installs pgEdge on all nodes.
@@ -1297,18 +1298,18 @@ def init(cluster_name, install=True):
         util.exit_message("Unable to load cluster JSON", 1)
     is_ha_cluster = parsed_json.get("is_ha_cluster", False)
     verbose = parsed_json.get("log_level", "info")
-    
+
     all_nodes = nodes.copy()
     for node in nodes:
         if "sub_nodes" in node and node["sub_nodes"]:
             all_nodes.extend(node["sub_nodes"])
-    
+
     # 2. Check SSH connectivity for all nodes
     util.message("## Checking SSH connectivity for all nodes", "info")
     for nd in all_nodes:
         message = f"Checking SSH connectivity on {nd['public_ip']}"
         run_cmd(cmd="hostname", node=nd, message=message, verbose=verbose)
-    
+
     # 3. Install pgEdge on all nodes
     util.message("## Installing pgEdge on all nodes", "info")
     ssh_install_pgedge(
@@ -1321,7 +1322,7 @@ def init(cluster_name, install=True):
         install,
         verbose,
     )
-    
+
     # 4. Configure Spock replication on all nodes (for the first DB)
     util.message("## Configuring Spock replication on all nodes", "info")
     ssh_cross_wire_pgedge(
@@ -1333,7 +1334,7 @@ def init(cluster_name, install=True):
         all_nodes,
         verbose,
     )
-    
+
     # If additional databases exist, configure them as well
     if len(db) > 1:
         util.message("## Configuring additional databases", "info")
@@ -1348,47 +1349,56 @@ def init(cluster_name, install=True):
                 all_nodes,
                 verbose,
             )
-    
+
     # 5. Integrate pgBackRest (if a "backrest" block is present) on each node
     cluster_name_from_json = parsed_json["cluster_name"]
-    
+
     for idx, node in enumerate(all_nodes, start=1):
         backrest = node.get("backrest", {})
         if backrest:
             util.message("## Integrating pgBackRest into the cluster", "info")
-            util.message(f"### Configuring pgBackRest for node '{node['name']}'", "info")
-    
+            util.message(
+                f"### Configuring pgBackRest for node '{node['name']}'", "info"
+            )
+
             # Create a unique stanza name: {cluster_name}_stanza_{node_name}
             stanza = f"{cluster_name_from_json}_stanza_{node['name']}"
-    
+
             # Load additional pgBackRest settings from JSON with defaults.
             repo1_retention_full = backrest.get("repo1_retention_full", "7")
             log_level_console = backrest.get("log_level_console", "info")
             repo1_cipher_type = backrest.get("repo1_cipher_type", "aes-256-cbc")
             repo1_type = backrest.get("repo1_type", "posix")  # Could also be "s3", etc.
-    
+
             # Get repo1_path from JSON; if not provided, default to /var/lib/pgbackrest/{node_name}
             json_repo1_path = backrest.get("repo1_path")
             if json_repo1_path:
-                repo1_path = json_repo1_path.rstrip('/')
+                repo1_path = json_repo1_path.rstrip("/")
                 if not repo1_path.endswith(node["name"]):
                     repo1_path = repo1_path + f"/{node['name']}"
             else:
                 repo1_path = f"/var/lib/pgbackrest/{node['name']}"
-    
+
             # Similarly, set restore_path to include node name (if needed)
             restore_path = "/var/lib/pgbackrest_restore"
-            if not restore_path.rstrip('/').endswith(node["name"]):
-                restore_path = restore_path.rstrip('/') + f"/{node['name']}"
-    
+            if not restore_path.rstrip("/").endswith(node["name"]):
+                restore_path = restore_path.rstrip("/") + f"/{node['name']}"
+
             pg_version = db_settings["pg_version"]
             pg1_path = f"{node['path']}/pgedge/data/pg{pg_version}"
             port = node["port"]  # Custom port from JSON
-    
+
             # -- Step 1: Install pgBackRest
-            cmd_install_backrest = f"cd {node['path']}/pgedge && ./pgedge install backrest"
-            run_cmd(cmd_install_backrest, node=node, message="Installing pgBackRest", verbose=verbose)
-    
+            cmd_install_backrest = (
+                f"cd {node['path']}/pgedge && ./pgedge install backrest"
+            )
+            run_cmd(
+                cmd_install_backrest,
+                node=node,
+                message="Installing pgBackRest",
+                verbose=verbose,
+            )
+
             # -- Step 2: Configure postgresql.conf for pgBackRest (without --pg1-port)
             cmd_set_postgresqlconf = (
                 f"cd {node['path']}/pgedge && "
@@ -1399,46 +1409,106 @@ def init(cluster_name, install=True):
                 f"--repo1-type {repo1_type} "
                 f"--repo1-cipher-type {repo1_cipher_type} "
             )
-            run_cmd(cmd_set_postgresqlconf, node=node, message="Modifying postgresql.conf for pgBackRest", verbose=verbose)
-    
+            run_cmd(
+                cmd_set_postgresqlconf,
+                node=node,
+                message="Modifying postgresql.conf for pgBackRest",
+                verbose=verbose,
+            )
+
             # -- Step 3: Configure pg_hba.conf for pgBackRest (without --pg1-port)
-            cmd_set_hbaconf = f"cd {node['path']}/pgedge && ./pgedge backrest set-hbaconf"
-            run_cmd(cmd_set_hbaconf, node=node, message="Modifying pg_hba.conf for pgBackRest", verbose=verbose)
-    
+            cmd_set_hbaconf = (
+                f"cd {node['path']}/pgedge && ./pgedge backrest set-hbaconf"
+            )
+            run_cmd(
+                cmd_set_hbaconf,
+                node=node,
+                message="Modifying pg_hba.conf for pgBackRest",
+                verbose=verbose,
+            )
+
             # -- Step 4: Reload PostgreSQL configuration to apply changes
             sql_reload_conf = "select pg_reload_conf()"
             cmd_reload_conf = f"cd {node['path']}/pgedge && ./pgedge psql '{sql_reload_conf}' {db[0]['db_name']}"
-            run_cmd(cmd_reload_conf, node=node, message="Reloading PostgreSQL configuration", verbose=verbose)
-    
+            run_cmd(
+                cmd_reload_conf,
+                node=node,
+                message="Reloading PostgreSQL configuration",
+                verbose=verbose,
+            )
+
             # -- Step 5: Set all pgBackRest backup configuration values
-    
+
             # (a) Set the backup stanza
-            cmd_set_backup_stanza = f"cd {node['path']}/pgedge && ./pgedge set BACKUP stanza {stanza}"
-            run_cmd(cmd_set_backup_stanza, node=node, message=f"Setting BACKUP stanza '{stanza}' on node '{node['name']}'", verbose=verbose)
-    
+            cmd_set_backup_stanza = (
+                f"cd {node['path']}/pgedge && ./pgedge set BACKUP stanza {stanza}"
+            )
+            run_cmd(
+                cmd_set_backup_stanza,
+                node=node,
+                message=f"Setting BACKUP stanza '{stanza}' on node '{node['name']}'",
+                verbose=verbose,
+            )
+
             # (b) Create restore directory and set restore_path for backups.
             cmd_create_restore_dir = f"sudo mkdir -p {restore_path}"
-            run_cmd(cmd_create_restore_dir, node=node, message=f"Creating restore directory {restore_path}", verbose=verbose)
+            run_cmd(
+                cmd_create_restore_dir,
+                node=node,
+                message=f"Creating restore directory {restore_path}",
+                verbose=verbose,
+            )
             cmd_set_restore_path = f"cd {node['path']}/pgedge && ./pgedge set BACKUP restore_path {restore_path}"
-            run_cmd(cmd_set_restore_path, node=node, message=f"Setting BACKUP restore_path to {restore_path}", verbose=verbose)
-    
+            run_cmd(
+                cmd_set_restore_path,
+                node=node,
+                message=f"Setting BACKUP restore_path to {restore_path}",
+                verbose=verbose,
+            )
+
             # (c) Set BACKUP repo1-host-user to the OS user (default: postgres)
             os_user = node.get("os_user", "postgres")
             cmd_set_repo1_host_user = f"cd {node['path']}/pgedge && ./pgedge set BACKUP repo1-host-user {os_user}"
-            run_cmd(cmd_set_repo1_host_user, node=node, message=f"Setting BACKUP repo1-host-user to {os_user} on node '{node['name']}'", verbose=verbose)
-    
+            run_cmd(
+                cmd_set_repo1_host_user,
+                node=node,
+                message=f"Setting BACKUP repo1-host-user to {os_user} on node '{node['name']}'",
+                verbose=verbose,
+            )
+
             # (d) Set BACKUP pg1-path to the PostgreSQL data directory
-            cmd_set_pg1_path = f"cd {node['path']}/pgedge && ./pgedge set BACKUP pg1-path {pg1_path}"
-            run_cmd(cmd_set_pg1_path, node=node, message=f"Setting BACKUP pg1-path to {pg1_path} on node '{node['name']}'", verbose=verbose)
-    
+            cmd_set_pg1_path = (
+                f"cd {node['path']}/pgedge && ./pgedge set BACKUP pg1-path {pg1_path}"
+            )
+            run_cmd(
+                cmd_set_pg1_path,
+                node=node,
+                message=f"Setting BACKUP pg1-path to {pg1_path} on node '{node['name']}'",
+                verbose=verbose,
+            )
+
             # (e) Set BACKUP pg1-user to the OS user
-            cmd_set_pg1_user = f"cd {node['path']}/pgedge && ./pgedge set BACKUP pg1-user {os_user}"
-            run_cmd(cmd_set_pg1_user, node=node, message=f"Setting BACKUP pg1-user to {os_user} on node '{node['name']}'", verbose=verbose)
-    
+            cmd_set_pg1_user = (
+                f"cd {node['path']}/pgedge && ./pgedge set BACKUP pg1-user {os_user}"
+            )
+            run_cmd(
+                cmd_set_pg1_user,
+                node=node,
+                message=f"Setting BACKUP pg1-user to {os_user} on node '{node['name']}'",
+                verbose=verbose,
+            )
+
             # (f) Set BACKUP pg1-port to the node's port value
-            cmd_set_pg1_port = f"cd {node['path']}/pgedge && ./pgedge set BACKUP pg1-port {port}"
-            run_cmd(cmd_set_pg1_port, node=node, message=f"Setting BACKUP pg1-port to {port} on node '{node['name']}'", verbose=verbose)
-    
+            cmd_set_pg1_port = (
+                f"cd {node['path']}/pgedge && ./pgedge set BACKUP pg1-port {port}"
+            )
+            run_cmd(
+                cmd_set_pg1_port,
+                node=node,
+                message=f"Setting BACKUP pg1-port to {port} on node '{node['name']}'",
+                verbose=verbose,
+            )
+
             # -- Step 6: Create the pgBackRest stanza (this command uses --pg1-port because it connects to the DB)
             cmd_create_stanza = (
                 f"cd {node['path']}/pgedge && "
@@ -1449,8 +1519,13 @@ def init(cluster_name, install=True):
                 f"--pg1-port={port} "
                 f"--repo1-path={repo1_path}"
             )
-            run_cmd(cmd_create_stanza, node=node, message=f"Creating pgBackRest stanza '{stanza}'", verbose=verbose)
-    
+            run_cmd(
+                cmd_create_stanza,
+                node=node,
+                message=f"Creating pgBackRest stanza '{stanza}'",
+                verbose=verbose,
+            )
+
             # -- Step 7: Initiate a full backup using pgBackRest (again, passing the port)
             backrest_backup_args = (
                 f"--repo1-path={repo1_path} "
@@ -1465,14 +1540,24 @@ def init(cluster_name, install=True):
                 f"--type=full"
             )
             cmd_create_backup = f"cd {node['path']}/pgedge && ./pgedge backrest command backup '{backrest_backup_args}'"
-            run_cmd(cmd_create_backup, node=node, message="Creating full pgBackRest backup", verbose=verbose)
-                    # (f) Set BACKUP pg1-port to the node's port value
+            run_cmd(
+                cmd_create_backup,
+                node=node,
+                message="Creating full pgBackRest backup",
+                verbose=verbose,
+            )
+            # (f) Set BACKUP pg1-port to the node's port value
             cmd_set_pg1_port = f"cd {node['path']}/pgedge && ./pgedge set BACKUP repo1-path {repo1_path}"
-            run_cmd(cmd_set_pg1_port, node=node, message=f"Setting BACKUP repo1-path to {repo1_path} on node '{node['name']}'", verbose=verbose)
+            run_cmd(
+                cmd_set_pg1_port,
+                node=node,
+                message=f"Setting BACKUP repo1-path to {repo1_path} on node '{node['name']}'",
+                verbose=verbose,
+            )
 
             if node.get("backrest", {}):
                 capture_backrest_config(node, verbose=True)
-    
+
     # 6. If it's an HA cluster, handle Patroni/etcd, etc.
     if is_ha_cluster:
         pg_ver = db_settings["pg_version"]
@@ -1532,9 +1617,9 @@ def add_node(
         cluster_name (str): The name of the cluster to which the node is being added.
         source_node (str): The name of the source node from which configurations and data are copied.
         target_node (str): The name of the new node being added.
-        repo1_path (str, optional): The repository path for pgBackRest. If not provided, 
+        repo1_path (str, optional): The repository path for pgBackRest. If not provided,
             the source node's configuration is used.
-        backup_id (str, optional): The ID of the backup to restore from. If not provided, 
+        backup_id (str, optional): The ID of the backup to restore from. If not provided,
             the latest backup is used.
         script (str, optional): A bash script to execute after the target node is added.
         install (bool, optional): Whether to install pgEdge on the target node. Defaults to True.
@@ -1644,7 +1729,7 @@ def add_node(
         source_repo1_retention_full = "7"
         source_log_level_console = "info"
         source_repo1_cipher_type = "aes-256-cbc"
-        source_repo1_type = "posix" 
+        source_repo1_type = "posix"
 
         # Determine the repository path for the source node.
         if repo1_path:
@@ -1783,7 +1868,9 @@ def add_node(
     source_stanza = source_backrest_cfg.get("stanza", "")
     source_repo1_retention_full = source_backrest_cfg.get("repo1_retention_full", "7")
     source_log_level_console = source_backrest_cfg.get("log_level_console", "info")
-    source_repo1_cipher_type = source_backrest_cfg.get("repo1_cipher_type", "aes-256-cbc")
+    source_repo1_cipher_type = source_backrest_cfg.get(
+        "repo1_cipher_type", "aes-256-cbc"
+    )
     source_repo1_type = source_backrest_cfg.get("repo1_type", "posix")
 
     rc = ssh_install_pgedge(
@@ -1854,15 +1941,15 @@ def add_node(
     pgc = f"{pgd}/postgresql.conf"
     log_directory = f'{target_node_data["path"]}/pgedge/data/logs/{pgV}'
 
-    cmd = f"sed -i \"/^ssl_cert_file/d\" {pgc} && echo \"ssl_cert_file = '{pgd}/server.crt'\" >> {pgc}"
+    cmd = f'sed -i "/^ssl_cert_file/d" {pgc} && echo "ssl_cert_file = \'{pgd}/server.crt\'" >> {pgc}'
     message = f"Setting ssl_cert_file"
     run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
-    cmd = f"sed -i \"/^ssl_key_file/d\" {pgc} && echo \"ssl_key_file = '{pgd}/server.key'\" >> {pgc}"
+    cmd = f'sed -i "/^ssl_key_file/d" {pgc} && echo "ssl_key_file = \'{pgd}/server.key\'" >> {pgc}'
     message = f"Setting ssl_key_file"
     run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
-    cmd = f"sed -i \"/^log_directory/d\" {pgc} && echo \"log_directory = '{log_directory}'\" >> {pgc}"
+    cmd = f'sed -i "/^log_directory/d" {pgc} && echo "log_directory = \'{log_directory}\'" >> {pgc}'
     message = f"Setting log_directory"
     run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
@@ -1975,14 +2062,16 @@ def add_node(
     cmd = f'cd {target_node_data["path"]}/pgedge/; ./pgedge spock node-list {db_name}'
     message = f"Listing spock nodes"
     result = run_cmd(
-        cmd, node=target_node_data, message=message, verbose=verbose, capture_output=True
+        cmd,
+        node=target_node_data,
+        message=message,
+        verbose=verbose,
+        capture_output=True,
     )
     print(f"\n{result.stdout}")
 
     sql_cmd = "select node_id,node_name from spock.node"
-    cmd = (
-        f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db_name}"
-    )
+    cmd = f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db_name}"
     message = f"List nodes"
     result = run_cmd(
         cmd,
@@ -2012,16 +2101,18 @@ def add_node(
     cmd = f"{target_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db_name}"
     message = f"List subscriptions"
     result = run_cmd(
-        cmd, node=target_node_data, message=message, verbose=verbose, capture_output=True
+        cmd,
+        node=target_node_data,
+        message=message,
+        verbose=verbose,
+        capture_output=True,
     )
     print(f"\n{result.stdout}")
 
     # A subsequent restart will be needed to apply the changes
     # This will occur in the next section when pgBackrest is configured or removed
     # Cleanup restore remnants by unsetting restore_command on target node
-    sql_cmd = (
-        'ALTER SYSTEM RESET restore_command'
-    )
+    sql_cmd = "ALTER SYSTEM RESET restore_command"
     cmd = f"{target_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db_name}"
     message = "Unsetting restore_command"
     run_cmd(cmd, node=target_node_data, message=message, verbose=verbose)
@@ -2032,26 +2123,23 @@ def add_node(
         f"./pgedge backrest cleanup-replica "
         f"--pg1-path {target_node_data['path']}/pgedge/data/{pgV} "
     )
-    run_cmd(cmd, node=target_node_data, message="Cleaning up replica configuration", verbose=verbose)
+    run_cmd(
+        cmd,
+        node=target_node_data,
+        message="Cleaning up replica configuration",
+        verbose=verbose,
+    )
 
     target_backrest_cfg = target_node_data.get("backrest", {})
     if target_backrest_cfg:
-        target_repo1_path = (
-            target_node_data
-            .get("backrest", {})
-            .get("repo1_path")
-        )
-        target_stanza = (
-            target_node_data.get("backrest", {}).get("stanza")
-        )
+        target_repo1_path = target_node_data.get("backrest", {}).get("repo1_path")
+        target_stanza = target_node_data.get("backrest", {}).get("stanza")
 
         target_pgedge_dir = f"{target_node_data['path']}/pgedge"
         target_restore_path = target_backrest_cfg.get(
             "restore_path", f"/var/lib/pgbackrest_restore/{target_node_data['name']}"
         )
-        target_repo1_type = target_backrest_cfg.get(
-            "repo1_type", "posix"
-        )
+        target_repo1_type = target_backrest_cfg.get("repo1_type", "posix")
         target_repo1_cipher_type = target_backrest_cfg.get(
             "repo1_cipher_type", "aes-256-cbc"
         )
@@ -2070,9 +2158,7 @@ def add_node(
         target_pg1_port = target_backrest_cfg.get(
             "pg1_port", target_node_data.get("port", "6435")
         )
-        target_log_level_console = target_backrest_cfg.get(
-            "log_level_console", "info"
-        )
+        target_log_level_console = target_backrest_cfg.get("log_level_console", "info")
 
         # Combined target node BACKUP configuration commands
         combined_target_cmd = (
@@ -2158,7 +2244,9 @@ def add_node(
         )
     else:
         target_pgedge_dir = f"{target_node_data['path']}/pgedge"
-        cmd_remove_backrest_target = f"cd {target_pgedge_dir} && ./pgedge remove backrest"
+        cmd_remove_backrest_target = (
+            f"cd {target_pgedge_dir} && ./pgedge remove backrest"
+        )
         run_cmd(
             cmd=cmd_remove_backrest_target,
             node=target_node_data,

From e6cbe73b9f0ab2df19ad97653864abe2a3e3fa28 Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Wed, 14 May 2025 14:29:52 -0500
Subject: [PATCH 13/20] add clarifying comment on restore_command behavior

---
 cli/scripts/cluster.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py
index 55499966..2e887489 100755
--- a/cli/scripts/cluster.py
+++ b/cli/scripts/cluster.py
@@ -1917,6 +1917,8 @@ def add_node(
     run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
     restore_args = (
+        # This line is required because pgbackrest will not include repo1-cipher-type
+        # by default when generating the restore_command
         f'--cmd="pgbackrest --repo1-cipher-type={source_repo1_cipher_type}" '
         f"--stanza={source_stanza} "
         f"--pg1-path={target_node_data['path']}/pgedge/data/{pgV} "

From 2a29d605d4663e549942b1c5e5e4571bb09b659d Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Wed, 14 May 2025 16:02:35 -0500
Subject: [PATCH 14/20] fixes for S3

---
 cli/scripts/cluster.py | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py
index 2e887489..f3f951db 100755
--- a/cli/scripts/cluster.py
+++ b/cli/scripts/cluster.py
@@ -1509,6 +1509,18 @@ def init(cluster_name, install=True):
                 verbose=verbose,
             )
 
+            # (g) Set BACKUP repo1-type to the node's repo1-type value
+            cmd_set_pg1_port = (
+                f"cd {node['path']}/pgedge && ./pgedge set BACKUP repo1-type {repo1_type}"
+            )
+            run_cmd(
+                cmd_set_pg1_port,
+                node=node,
+                message=f"Setting BACKUP repo1-type to {repo1_type} on node '{node['name']}'",
+                verbose=verbose,
+            )
+
+
             # -- Step 6: Create the pgBackRest stanza (this command uses --pg1-port because it connects to the DB)
             cmd_create_stanza = (
                 f"cd {node['path']}/pgedge && "
@@ -1517,7 +1529,8 @@ def init(cluster_name, install=True):
                 f"--pg1-path='{pg1_path}' "
                 f"--repo1-cipher-type={repo1_cipher_type} "
                 f"--pg1-port={port} "
-                f"--repo1-path={repo1_path}"
+                f"--repo1-path={repo1_path} "
+                f"--repo1-type={repo1_type}"
             )
             run_cmd(
                 cmd_create_stanza,
@@ -1795,6 +1808,7 @@ def add_node(
                 f"sudo mkdir -p {source_restore_path}",
                 f"./pgedge set BACKUP restore_path {source_restore_path}",
                 f"./pgedge set BACKUP repo1-host-user {source_node_data.get('os_user', 'postgres')}",
+                f"./pgedge set BACKUP repo1-type {source_repo1_type}",
                 f"./pgedge set BACKUP pg1-path {source_pg1_path}",
                 f"./pgedge set BACKUP pg1-user {source_node_data.get('os_user', 'postgres')}",
                 f"./pgedge set BACKUP pg1-port {source_port}",
@@ -1817,7 +1831,8 @@ def add_node(
             f"--pg1-path='{source_pg1_path}' "
             f"--repo1-cipher-type={source_repo1_cipher_type} "
             f"--pg1-port={source_port} "
-            f"--repo1-path={source_repo1_path}"
+            f"--repo1-path={source_repo1_path} "
+            f"--repo1-type={source_repo1_type}"
         )
         run_cmd(
             cmd_create_stanza_source,
@@ -2171,6 +2186,7 @@ def add_node(
             f"sudo chown -R {target_repo1_host_user}:{target_repo1_host_user} {target_repo1_path} && "
             f"cd {target_pgedge_dir} && ./pgedge set BACKUP repo1-path {target_repo1_path} && "
             f"cd {target_pgedge_dir} && ./pgedge set BACKUP repo1-host-user {target_repo1_host_user} && "
+            f"cd {target_pgedge_dir} && ./pgedge set BACKUP repo1-type {target_repo1_type} && "
             f"cd {target_pgedge_dir} && ./pgedge set BACKUP pg1-path {target_pg1_path} && "
             f"cd {target_pgedge_dir} && ./pgedge set BACKUP pg1-user {target_pg1_user} && "
             f"cd {target_pgedge_dir} && ./pgedge set BACKUP pg1-port {target_pg1_port} && "
@@ -2212,7 +2228,8 @@ def add_node(
             f"--pg1-path='{target_pg1_path}' "
             f"--repo1-cipher-type={target_repo1_cipher_type} "
             f"--pg1-port={target_pg1_port} "
-            f"--repo1-path={target_repo1_path}"
+            f"--repo1-path={target_repo1_path} "
+            f"--repo1-type={target_repo1_type}"
         )
         run_cmd(
             cmd=cmd_create_stanza_target,

From 555d2822e7413d11dd1d3b8d6002d870f0fa14c4 Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Wed, 14 May 2025 16:15:50 -0500
Subject: [PATCH 15/20] bump to alpha4

---
 cli/scripts/install.py | 2 +-
 cli/scripts/util.py    | 2 +-
 env.sh                 | 4 ++--
 src/conf/versions.sql  | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/cli/scripts/install.py b/cli/scripts/install.py
index c865722e..33ad6b5b 100644
--- a/cli/scripts/install.py
+++ b/cli/scripts/install.py
@@ -3,7 +3,7 @@
 
 import sys, os, tarfile, platform
 
-VER = "25.0.0-alpha3"
+VER = "25.0.0-alpha4"
 REPO = os.getenv("REPO", "https://pgedge-upstream.s3.amazonaws.com/REPO")
 
 if sys.version_info < (3, 9):
diff --git a/cli/scripts/util.py b/cli/scripts/util.py
index 704cb52a..18d529a0 100644
--- a/cli/scripts/util.py
+++ b/cli/scripts/util.py
@@ -4,7 +4,7 @@
 import os
 import time
 
-MY_VERSION = "25.0.0-alpha3"
+MY_VERSION = "25.0.0-alpha4"
 MY_CODENAME = ""
 
 DEFAULT_PG = "16"
diff --git a/env.sh b/env.sh
index 697dae8c..e42f52b4 100755
--- a/env.sh
+++ b/env.sh
@@ -1,5 +1,5 @@
-hubV=25.0.0-alpha3
-hubVV=25.0.0-alpha3
+hubV=25.0.0-alpha4
+hubVV=25.0.0-alpha4
 
 aceV=$hubV
 kirkV=$hubV
diff --git a/src/conf/versions.sql b/src/conf/versions.sql
index 22ffd721..38492e67 100644
--- a/src/conf/versions.sql
+++ b/src/conf/versions.sql
@@ -1,7 +1,7 @@
 
 DROP TABLE IF EXISTS hub;
 CREATE TABLE hub(v TEXT NOT NULL PRIMARY KEY, c TEXT NOT NULL, d TEXT NOT NULL);
-INSERT INTO hub VALUES ('25.0.0-alpha3', 'Constellation',  '20250421');
+INSERT INTO hub VALUES ('25.0.0-alpha4', 'Constellation',  '20250421');
 
 DROP VIEW  IF EXISTS v_versions;
 DROP VIEW  IF EXISTS v_products;

From 43143ac60f8791ac2e292753122d8577b7aedcce Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Wed, 14 May 2025 21:13:16 -0500
Subject: [PATCH 16/20] cleanup comments and unused imports

---
 cli/scripts/cluster.py | 102 +++++++++++++++++++----------------------
 1 file changed, 46 insertions(+), 56 deletions(-)

diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py
index f3f951db..9de72522 100755
--- a/cli/scripts/cluster.py
+++ b/cli/scripts/cluster.py
@@ -9,14 +9,9 @@
 import getpass
 from tabulate import tabulate # type: ignore
 from ipaddress import ip_address
-try:
-    import etcd 
-    import ha_patroni
-except Exception:
-    pass
 import os
 import re
-import yaml  
+
 BASE_DIR = "cluster"
 DEFAULT_REPO = "https://pgedge-download.s3.amazonaws.com/REPO"
 
@@ -1290,13 +1285,13 @@ def init(cluster_name, install=True):
         cluster_name (str): The name of the cluster to initialize.
         install (bool): Whether to install pgEdge on nodes. Defaults to True.
     """
-    # 1. Load cluster configuration
+    # Step 1. Load cluster configuration
     util.message(f"## Loading cluster '{cluster_name}' JSON definition file", "info")
     db, db_settings, nodes = load_json(cluster_name)
     parsed_json = get_cluster_json(cluster_name)
     if parsed_json is None:
         util.exit_message("Unable to load cluster JSON", 1)
-    is_ha_cluster = parsed_json.get("is_ha_cluster", False)
+
     verbose = parsed_json.get("log_level", "info")
 
     all_nodes = nodes.copy()
@@ -1304,13 +1299,13 @@ def init(cluster_name, install=True):
         if "sub_nodes" in node and node["sub_nodes"]:
             all_nodes.extend(node["sub_nodes"])
 
-    # 2. Check SSH connectivity for all nodes
+    # Step 2. Check SSH connectivity for all nodes
     util.message("## Checking SSH connectivity for all nodes", "info")
     for nd in all_nodes:
         message = f"Checking SSH connectivity on {nd['public_ip']}"
         run_cmd(cmd="hostname", node=nd, message=message, verbose=verbose)
 
-    # 3. Install pgEdge on all nodes
+    # Step 3. Install pgEdge on all nodes
     util.message("## Installing pgEdge on all nodes", "info")
     ssh_install_pgedge(
         cluster_name,
@@ -1323,7 +1318,7 @@ def init(cluster_name, install=True):
         verbose,
     )
 
-    # 4. Configure Spock replication on all nodes (for the first DB)
+    # Step 4. Configure Spock replication on all nodes (for the first DB)
     util.message("## Configuring Spock replication on all nodes", "info")
     ssh_cross_wire_pgedge(
         cluster_name,
@@ -1350,7 +1345,7 @@ def init(cluster_name, install=True):
                 verbose,
             )
 
-    # 5. Integrate pgBackRest (if a "backrest" block is present) on each node
+    # Step 5. Integrate pgBackRest (if a "backrest" block is present) on each node
     cluster_name_from_json = parsed_json["cluster_name"]
 
     for idx, node in enumerate(all_nodes, start=1):
@@ -1399,7 +1394,7 @@ def init(cluster_name, install=True):
                 verbose=verbose,
             )
 
-            # -- Step 2: Configure postgresql.conf for pgBackRest (without --pg1-port)
+            # Configure postgresql.conf for pgBackRest (without --pg1-port)
             cmd_set_postgresqlconf = (
                 f"cd {node['path']}/pgedge && "
                 f"./pgedge backrest set-postgresqlconf "
@@ -1416,7 +1411,7 @@ def init(cluster_name, install=True):
                 verbose=verbose,
             )
 
-            # -- Step 3: Configure pg_hba.conf for pgBackRest (without --pg1-port)
+            # Configure pg_hba.conf for pgBackRest (without --pg1-port)
             cmd_set_hbaconf = (
                 f"cd {node['path']}/pgedge && ./pgedge backrest set-hbaconf"
             )
@@ -1427,7 +1422,7 @@ def init(cluster_name, install=True):
                 verbose=verbose,
             )
 
-            # -- Step 4: Reload PostgreSQL configuration to apply changes
+            # Reload PostgreSQL configuration to apply changes
             sql_reload_conf = "select pg_reload_conf()"
             cmd_reload_conf = f"cd {node['path']}/pgedge && ./pgedge psql '{sql_reload_conf}' {db[0]['db_name']}"
             run_cmd(
@@ -1437,9 +1432,9 @@ def init(cluster_name, install=True):
                 verbose=verbose,
             )
 
-            # -- Step 5: Set all pgBackRest backup configuration values
+            # Set all pgBackRest backup configuration values
 
-            # (a) Set the backup stanza
+            # Set the backup stanza
             cmd_set_backup_stanza = (
                 f"cd {node['path']}/pgedge && ./pgedge set BACKUP stanza {stanza}"
             )
@@ -1450,7 +1445,7 @@ def init(cluster_name, install=True):
                 verbose=verbose,
             )
 
-            # (b) Create restore directory and set restore_path for backups.
+            # Create restore directory and set restore_path for backups.
             cmd_create_restore_dir = f"sudo mkdir -p {restore_path}"
             run_cmd(
                 cmd_create_restore_dir,
@@ -1466,7 +1461,7 @@ def init(cluster_name, install=True):
                 verbose=verbose,
             )
 
-            # (c) Set BACKUP repo1-host-user to the OS user (default: postgres)
+            # Set BACKUP repo1-host-user to the OS user (default: postgres)
             os_user = node.get("os_user", "postgres")
             cmd_set_repo1_host_user = f"cd {node['path']}/pgedge && ./pgedge set BACKUP repo1-host-user {os_user}"
             run_cmd(
@@ -1476,7 +1471,7 @@ def init(cluster_name, install=True):
                 verbose=verbose,
             )
 
-            # (d) Set BACKUP pg1-path to the PostgreSQL data directory
+            # Set BACKUP pg1-path to the PostgreSQL data directory
             cmd_set_pg1_path = (
                 f"cd {node['path']}/pgedge && ./pgedge set BACKUP pg1-path {pg1_path}"
             )
@@ -1487,7 +1482,7 @@ def init(cluster_name, install=True):
                 verbose=verbose,
             )
 
-            # (e) Set BACKUP pg1-user to the OS user
+            # Set BACKUP pg1-user to the OS user
             cmd_set_pg1_user = (
                 f"cd {node['path']}/pgedge && ./pgedge set BACKUP pg1-user {os_user}"
             )
@@ -1498,7 +1493,7 @@ def init(cluster_name, install=True):
                 verbose=verbose,
             )
 
-            # (f) Set BACKUP pg1-port to the node's port value
+            # Set BACKUP pg1-port to the node's port value
             cmd_set_pg1_port = (
                 f"cd {node['path']}/pgedge && ./pgedge set BACKUP pg1-port {port}"
             )
@@ -1509,7 +1504,7 @@ def init(cluster_name, install=True):
                 verbose=verbose,
             )
 
-            # (g) Set BACKUP repo1-type to the node's repo1-type value
+            # Set BACKUP repo1-type to the node's repo1-type value
             cmd_set_pg1_port = (
                 f"cd {node['path']}/pgedge && ./pgedge set BACKUP repo1-type {repo1_type}"
             )
@@ -1521,7 +1516,7 @@ def init(cluster_name, install=True):
             )
 
 
-            # -- Step 6: Create the pgBackRest stanza (this command uses --pg1-port because it connects to the DB)
+            # Create the pgBackRest stanza (this command uses --pg1-port because it connects to the DB)
             cmd_create_stanza = (
                 f"cd {node['path']}/pgedge && "
                 f"./pgedge backrest command stanza-create "
@@ -1539,7 +1534,7 @@ def init(cluster_name, install=True):
                 verbose=verbose,
             )
 
-            # -- Step 7: Initiate a full backup using pgBackRest (again, passing the port)
+            # Initiate a full backup using pgBackRest (again, passing the port)
             backrest_backup_args = (
                 f"--repo1-path={repo1_path} "
                 f"--stanza={stanza} "
@@ -1559,7 +1554,7 @@ def init(cluster_name, install=True):
                 message="Creating full pgBackRest backup",
                 verbose=verbose,
             )
-            # (f) Set BACKUP pg1-port to the node's port value
+            # Set BACKUP pg1-port to the node's port value
             cmd_set_pg1_port = f"cd {node['path']}/pgedge && ./pgedge set BACKUP repo1-path {repo1_path}"
             run_cmd(
                 cmd_set_pg1_port,
@@ -1571,23 +1566,6 @@ def init(cluster_name, install=True):
             if node.get("backrest", {}):
                 capture_backrest_config(node, verbose=True)
 
-    # 6. If it's an HA cluster, handle Patroni/etcd, etc.
-    if is_ha_cluster:
-        pg_ver = db_settings["pg_version"]
-        for node in nodes:
-            if "sub_nodes" in node and node["sub_nodes"]:
-                sub_nodes = node["sub_nodes"]
-                # Stop and clean sub-nodes
-                for n in sub_nodes:
-                    manage_node(n, "stop", f"pg{pg_ver}", verbose)
-                    pgdata = f"{n['path']}/pgedge/data/pg{pg_ver}"
-                    cmd = f"rm -rf {pgdata}"
-                    message = f"Removing old data directory on {n['name']}"
-                    run_cmd(cmd, n, message=message, verbose=verbose)
-                # Configure etcd and Patroni
-                etcd.configure_etcd(node, sub_nodes)
-                ha_patroni.configure_patroni(node, sub_nodes, db[0], db_settings)
-
 
 def remove_backrest(node_data):
     """
@@ -1615,13 +1593,14 @@ def add_node(
     Add a new node to a cluster by performing the following steps:
 
         1. Validate the cluster and target node JSON configurations.
-        2. Install pgEdge on the target node, if required.
-        3. Configure pgBackRest on the source node, if not already configured.
+        2. Configure pgBackRest on the source node, if not already configured.
+        3. Install pgEdge on the target node, if required.
         4. Restore the target node from a backup of the source node using pgBackRest.
         5. Configure the target node as a standby replica of the source node.
         6. Promote the target node to a primary once it catches up to the source node.
         7. Configure replication and subscriptions for the new node across the cluster.
-        8. Update the cluster JSON configuration with the new node.
+        8. Reconfigure pgBackrest on the source and target nodes, if required.
+        9. Update the cluster JSON configuration with the new node.
 
         A target node JSON configuration file must be provided in the same directory from which
         this command is invoked, named '<node_name>.json'.
@@ -1637,6 +1616,7 @@ def add_node(
         script (str, optional): A bash script to execute after the target node is added.
         install (bool, optional): Whether to install pgEdge on the target node. Defaults to True.
     """
+    # Step 1. Validate the cluster and target node JSON configurations.
     db, db_settings, nodes = load_json(cluster_name)
     db_name = db[0]["db_name"]
     db_user = db[0]["db_user"]
@@ -1718,9 +1698,9 @@ def add_node(
             "public_ip", target_node_data.get("private_ip")
         )
 
-    # If backrest is not configured on the source node, install it
+    # Step 2. Configure pgBackRest on the source node, if not already configured.
     if not source_backrest_cfg:
-        # Step 1: Install pgBackRest on the source node
+        # Install pgBackRest on the source node
         cmd_install_backrest = (
             f"cd {source_node_data['path']}/pgedge && ./pgedge install backrest"
         )
@@ -1762,7 +1742,7 @@ def add_node(
         source_pg1_path = f"{source_node_data['path']}/pgedge/data/pg{pg_version}"
         source_port = source_node_data["port"]
 
-        # Step 2: Configure postgresql.conf for pgBackRest (without --pg1-port)
+        # Configure postgresql.conf for pgBackRest (without --pg1-port)
         cmd_set_postgresqlconf_source = (
             f"cd {source_node_data['path']}/pgedge && "
             f"./pgedge backrest set-postgresqlconf "
@@ -1779,7 +1759,7 @@ def add_node(
             verbose=verbose,
         )
 
-        # Step 3: Configure pg_hba.conf for pgBackRest (without --pg1-port)
+        # Configure pg_hba.conf for pgBackRest (without --pg1-port)
         cmd_set_hbaconf_source = (
             f"cd {source_node_data['path']}/pgedge && ./pgedge backrest set-hbaconf"
         )
@@ -1790,7 +1770,7 @@ def add_node(
             verbose=verbose,
         )
 
-        # Step 4: Reload PostgreSQL configuration to apply changes
+        # Reload PostgreSQL configuration to apply changes
         sql_reload_conf = "select pg_reload_conf()"
         cmd_reload_conf_source = f"cd {source_node_data['path']}/pgedge && ./pgedge psql '{sql_reload_conf}' {db_name}"
         run_cmd(
@@ -1800,7 +1780,7 @@ def add_node(
             verbose=verbose,
         )
 
-        # Step 5: Set all pgBackRest backup configuration values for the source node.
+        # Set all pgBackRest backup configuration values for the source node.
         compound_cmd = " && ".join(
             [
                 f"cd {source_node_data['path']}/pgedge",
@@ -1823,7 +1803,7 @@ def add_node(
             verbose=False,
         )
 
-        # Step 6: Create the pgBackRest stanza (this command uses --pg1-port because it connects to the DB)
+        # Create the pgBackRest stanza (this command uses --pg1-port because it connects to the DB)
         cmd_create_stanza_source = (
             f"cd {source_node_data['path']}/pgedge && "
             f"./pgedge backrest command stanza-create "
@@ -1840,7 +1820,7 @@ def add_node(
             message=f"Creating pgBackRest stanza '{source_stanza}'",
             verbose=verbose,
         )
-        # Step 7: Initiate a full backup using pgBackRest (again, passing the port)
+        # Initiate a full backup using pgBackRest (again, passing the port)
         backrest_backup_args_source = (
             f"--repo1-path={source_repo1_path} "
             f"--stanza={source_stanza} "
@@ -1860,7 +1840,7 @@ def add_node(
             message="Creating full pgBackRest backup",
             verbose=verbose,
         )
-        # (i) (Optional) Reset BACKUP repo1-path if needed
+        # Reset BACKUP repo1-path if needed
         cmd_set_repo1_path_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP repo1-path {source_repo1_path}"
         run_cmd(
             cmd_set_repo1_path_source,
@@ -1888,6 +1868,7 @@ def add_node(
     )
     source_repo1_type = source_backrest_cfg.get("repo1_type", "posix")
 
+    # Step 3. Install pgEdge on the target node, if required.
     rc = ssh_install_pgedge(
         cluster_name,
         db_name,
@@ -1899,6 +1880,7 @@ def add_node(
         verbose,
     )
 
+    # Step 4. Restore the target node from a backup of the source node using pgBackRest.
     if not repo1_path:
         # Do not install pgbackrest on source node; simply fetch the repo1_path from source's settings.
         repo1_path_default = f"/var/lib/pgbackrest/{source_node_data['name']}"
@@ -1970,6 +1952,7 @@ def add_node(
     message = f"Setting log_directory"
     run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
+    # Step 5. Configure the target node as a standby replica of the source node.
     cmd = (
         f'{target_node_data["path"]}/pgedge/pgedge backrest configure-replica {source_stanza} '
         f'{target_node_data["path"]}/pgedge/data/{pgV} {source_node_data["ip_address"]} '
@@ -1992,17 +1975,21 @@ def add_node(
         if spock_maj >= 4:
             v4 = True
 
+    
     set_cluster_readonly(nodes, True, db_name, f"{pgV}", v4, verbose)
     manage_node(target_node_data, "start", f"{pgV}", verbose)
     time.sleep(5)
 
-    check_cluster_lag(target_node_data, db_name, f"{pgV}", verbose)
+    # Step 6. Promote the target node to a primary once it catches up to the source node.
 
+    check_cluster_lag(target_node_data, db_name, f"{pgV}", verbose)
+    
     sql_cmd = "SELECT pg_promote()"
     cmd = f"{target_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db_name}"
     message = f"Promoting standby to primary"
     run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
+    # Step 7. Configure replication and subscriptions for the new node across the cluster.
     for mdb in db:
         sql_cmd = "SELECT sub_name FROM spock.subscription"
         cmd = f"{target_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {mdb['db_name']}"
@@ -2126,6 +2113,8 @@ def add_node(
     )
     print(f"\n{result.stdout}")
 
+    # Step 8. Reconfigure pgBackrest on the source and target nodes, if required.
+
     # A subsequent restart will be needed to apply the changes
     # This will occur in the next section when pgBackrest is configured or removed
     # Cleanup restore remnants by unsetting restore_command on target node
@@ -2279,6 +2268,7 @@ def add_node(
     if not source_backrest_cfg:
         remove_backrest(source_node_data)
 
+    # Step 9. Update the cluster JSON configuration with the new node.
     # Check and display pgBackRest configuration status in the source node
     # Remove unnecessary keys before appending new node to the cluster data
     target_node_data.pop("ip_address", None)

From af8f1dd555bc83726b3565efed3d21e9ba250dfc Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Thu, 15 May 2025 10:02:30 -0500
Subject: [PATCH 17/20] check pg_is_in_recovery before proceeding to cluster
 rewiring

---
 cli/scripts/cluster.py | 36 +++++++++++++++++++++++++++++++-----
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py
index 9de72522..d00663cb 100755
--- a/cli/scripts/cluster.py
+++ b/cli/scripts/cluster.py
@@ -1977,18 +1977,23 @@ def add_node(
 
     
     set_cluster_readonly(nodes, True, db_name, f"{pgV}", v4, verbose)
+
     manage_node(target_node_data, "start", f"{pgV}", verbose)
     time.sleep(5)
 
-    # Step 6. Promote the target node to a primary once it catches up to the source node.
-
-    check_cluster_lag(target_node_data, db_name, f"{pgV}", verbose)
+    # Wait until the target node is in sync with the source node
+    check_cluster_lag(target_node_data, db_name, verbose)
     
+    # Promote the target node to primary
     sql_cmd = "SELECT pg_promote()"
     cmd = f"{target_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db_name}"
     message = f"Promoting standby to primary"
     run_cmd(cmd, target_node_data, message=message, verbose=verbose)
 
+    # Wait for the target node to finish recovery
+    # This ensures we can start writes on the new node
+    check_recovery(target_node_data, db_name, verbose)
+
     # Step 7. Configure replication and subscriptions for the new node across the cluster.
     for mdb in db:
         sql_cmd = "SELECT sub_name FROM spock.subscription"
@@ -2663,7 +2668,7 @@ def set_cluster_readonly(nodes, readonly, dbname, stanza, v4, verbose):
             run_cmd(cmd, node=node, message=message, verbose=verbose, important=True)
 
 
-def check_cluster_lag(n, dbname, stanza, verbose, timeout=600, interval=1):
+def check_cluster_lag(n, dbname, verbose, timeout=600, interval=2):
     sql_cmd = """
     SELECT COALESCE(
     (SELECT pg_wal_lsn_diff(pg_last_wal_receive_lsn(), pg_last_wal_replay_lsn())),
@@ -2678,7 +2683,7 @@ def check_cluster_lag(n, dbname, stanza, verbose, timeout=600, interval=1):
         if time.time() - start_time > timeout:
             return
 
-        time.sleep(2)
+        time.sleep(interval)
         cmd = f"{n['path']}/pgedge/pgedge psql '{sql_cmd}' {dbname}"
         message = f"Checking lag time of new cluster"
         result = run_cmd(
@@ -2687,6 +2692,27 @@ def check_cluster_lag(n, dbname, stanza, verbose, timeout=600, interval=1):
         print(result.stdout)
         lag_bytes = int(extract_psql_value(result.stdout, "lag_bytes"))
 
+def check_recovery(n, dbname, verbose, timeout=600, interval=5):
+    sql_cmd = """
+    SELECT pg_is_in_recovery() AS in_recovery
+    """
+
+    start_time = time.time()
+    in_recovery = True
+
+    while in_recovery:
+        if time.time() - start_time > timeout:
+            return
+
+        time.sleep(interval)
+        cmd = f"{n['path']}/pgedge/pgedge psql '{sql_cmd}' {dbname}"
+        message = f"Checking status of recovery"
+        result = run_cmd(
+            cmd=cmd, node=n, message=message, verbose=verbose, capture_output=True
+        )
+        print(result.stdout)
+        in_recovery = extract_psql_value(result.stdout, "in_recovery") == "t"
+
 
 def check_wal_rec(n, dbname, stanza, verbose, timeout=600, interval=1):
     sql_cmd = """

From e22da9732d388406d9cd639a87422d8ba9c464f1 Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Thu, 15 May 2025 10:22:33 -0500
Subject: [PATCH 18/20] update comment

---
 cli/scripts/cluster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py
index d00663cb..8e3a4862 100755
--- a/cli/scripts/cluster.py
+++ b/cli/scripts/cluster.py
@@ -1383,7 +1383,7 @@ def init(cluster_name, install=True):
             pg1_path = f"{node['path']}/pgedge/data/pg{pg_version}"
             port = node["port"]  # Custom port from JSON
 
-            # -- Step 1: Install pgBackRest
+            # Install pgBackRest
             cmd_install_backrest = (
                 f"cd {node['path']}/pgedge && ./pgedge install backrest"
             )

From 847a0f43e44324de2142f0005c85c4dc4b2ab047 Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Thu, 15 May 2025 10:49:14 -0500
Subject: [PATCH 19/20] fix for build container

---
 devel/setup/Dockerfile.rocky810 | 3 ++-
 devel/setup/Dockerfile.rocky95  | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/devel/setup/Dockerfile.rocky810 b/devel/setup/Dockerfile.rocky810
index 06b7ccbc..5b75f4da 100644
--- a/devel/setup/Dockerfile.rocky810
+++ b/devel/setup/Dockerfile.rocky810
@@ -7,8 +7,9 @@ USER root
 
 ENV install="dnf install -y --allowerasing"
 RUN $install dnf-plugins-core
+RUN $install epel-release
 RUN $install python3 python3-pip git wget curl pigz which zip sqlite
-RUN $install openssh-server systemd sudo
+RUN $install openssh-server systemd sudo inotify-tools lsof
 
 RUN useradd build  -U -m -d /home/build  \
     && echo "build  ALL=(ALL)  NOPASSWD: ALL" >> /etc/sudoers
diff --git a/devel/setup/Dockerfile.rocky95 b/devel/setup/Dockerfile.rocky95
index aa094562..5298262d 100644
--- a/devel/setup/Dockerfile.rocky95
+++ b/devel/setup/Dockerfile.rocky95
@@ -7,6 +7,7 @@ USER root
 
 ENV install="dnf install -y --allowerasing"
 RUN $install dnf-plugins-core
+RUN $install epel-release
 RUN $install python3 python3-pip git wget curl pigz which zip sqlite
 RUN $install openssh-server systemd sudo inotify-tools lsof
 

From e87f100285c183aecc1af57ed5da3ccac7f7ed0e Mon Sep 17 00:00:00 2001
From: Matthew Mols <matt@pgedge.com>
Date: Thu, 15 May 2025 13:49:31 -0500
Subject: [PATCH 20/20] use private_ip in dsn for add-node when cross wiring

---
 cli/scripts/cluster.py | 28 +++++-----------------------
 1 file changed, 5 insertions(+), 23 deletions(-)

diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py
index 8e3a4862..987ad353 100755
--- a/cli/scripts/cluster.py
+++ b/cli/scripts/cluster.py
@@ -1684,20 +1684,6 @@ def add_node(
             "Both public_ip and private_ip are missing in target node data."
         )
 
-    if "public_ip" in source_node_data and "private_ip" in source_node_data:
-        source_node_data["ip_address"] = source_node_data["public_ip"]
-    else:
-        source_node_data["ip_address"] = source_node_data.get(
-            "public_ip", source_node_data.get("private_ip")
-        )
-
-    if "public_ip" in target_node_data and "private_ip" in target_node_data:
-        target_node_data["ip_address"] = target_node_data["public_ip"]
-    else:
-        target_node_data["ip_address"] = target_node_data.get(
-            "public_ip", target_node_data.get("private_ip")
-        )
-
     # Step 2. Configure pgBackRest on the source node, if not already configured.
     if not source_backrest_cfg:
         # Install pgBackRest on the source node
@@ -1955,7 +1941,7 @@ def add_node(
     # Step 5. Configure the target node as a standby replica of the source node.
     cmd = (
         f'{target_node_data["path"]}/pgedge/pgedge backrest configure-replica {source_stanza} '
-        f'{target_node_data["path"]}/pgedge/data/{pgV} {source_node_data["ip_address"]} '
+        f'{target_node_data["path"]}/pgedge/data/{pgV} {source_node_data.get("private_ip", source_node_data.get("public_ip"))} '
         f'{source_node_data["port"]} {source_node_data["os_user"]}'
     )
     message = f"Configuring PITR on replica"
@@ -1975,7 +1961,6 @@ def add_node(
         if spock_maj >= 4:
             v4 = True
 
-    
     set_cluster_readonly(nodes, True, db_name, f"{pgV}", v4, verbose)
 
     manage_node(target_node_data, "start", f"{pgV}", verbose)
@@ -1983,7 +1968,7 @@ def add_node(
 
     # Wait until the target node is in sync with the source node
     check_cluster_lag(target_node_data, db_name, verbose)
-    
+
     # Promote the target node to primary
     sql_cmd = "SELECT pg_promote()"
     cmd = f"{target_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db_name}"
@@ -2276,7 +2261,6 @@ def add_node(
     # Step 9. Update the cluster JSON configuration with the new node.
     # Check and display pgBackRest configuration status in the source node
     # Remove unnecessary keys before appending new node to the cluster data
-    target_node_data.pop("ip_address", None)
     target_node_data.pop("os_user", None)
     target_node_data.pop("ssh_key", None)
 
@@ -2539,7 +2523,7 @@ def create_node(node, dbname, verbose):
     """
     Creates a new node in the database cluster.
     """
-    ip = node["public_ip"] if "public_ip" in node else node["private_ip"]
+    ip = node["private_ip"] if node["private_ip"] else node["public_ip"]
     if not ip:
         util.exit_message(f"Node '{node['name']}' does not have a valid IP address.")
 
@@ -2559,7 +2543,7 @@ def create_sub_new(nodes, n, dbname, verbose):
     """
     for node in nodes:
         sub_name = f"sub_{n['name']}{node['name']}"
-        ip = node["public_ip"] if "public_ip" in node else node["private_ip"]
+        ip = node["private_ip"] if node["private_ip"] else node["public_ip"]
         if not ip:
             util.exit_message(
                 f"Node '{node['name']}' does not have a valid IP address."
@@ -2580,9 +2564,7 @@ def create_sub(nodes, new_node, dbname, verbose):
     """
     for n in nodes:
         sub_name = f"sub_{n['name']}{new_node['name']}"
-        ip = (
-            new_node["public_ip"] if "public_ip" in new_node else new_node["private_ip"]
-        )
+        ip = new_node["private_ip"] if new_node["private_ip"] else new_node["public_ip"]
         if not ip:
             util.exit_message(
                 f"Node '{new_node['name']}' does not have a valid IP address."