From d1a9b8fa3d926e1558e4b0b2959192eff9e5e105 Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Wed, 2 Apr 2025 03:17:32 +0500 Subject: [PATCH 01/19] yaml file created for backrest json in backrest dir --- cli/scripts/cluster.py | 459 +---------------------------------------- 1 file changed, 1 insertion(+), 458 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index 6ec1ddfb..23ff4ccf 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -11,7 +11,7 @@ import re from tabulate import tabulate # type: ignore from ipaddress import ip_address - +import yaml try: import etcd import ha_patroni @@ -1563,463 +1563,6 @@ def init(cluster_name, install=True): etcd.configure_etcd(node, sub_nodes) ha_patroni.configure_patroni(node, sub_nodes, db[0], db_settings) -def add_node( - cluster_name, - source_node, - target_node, - repo1_path=None, - backup_id=None, - script=" ", - stanza=" ", - install=True, -): - """ - Adds a new node to a cluster, copying configurations from a specified - source node. - - Args: - cluster_name (str): The name of the cluster to which the node is being - added. - source_node (str): The node from which configurations are copied. - target_node (str): The new node. - repo1_path (str): The repo1 path to use. - backup_id (str): Backup ID. - stanza (str): Stanza name. - script (str): Bash script. - """ - if (repo1_path and not backup_id) or (backup_id and not repo1_path): - util.exit_message("Both repo1_path and backup_id must be supplied together.") - json_validate(cluster_name) - db, db_settings, nodes = load_json(cluster_name) - - cluster_data = get_cluster_json(cluster_name) - if cluster_data is None: - util.exit_message("Cluster data is missing.") - pg = db_settings["pg_version"] - pgV = f"pg{pg}" - verbose = cluster_data.get("log_level", "info") - - # Load and validate the target node JSON - target_node_file = f"{target_node}.json" - if not os.path.isfile(target_node_file): - util.exit_message(f"New node json file '{target_node_file}' not found") - - try: - with open(target_node_file, "r") as f: - target_node_data = json.load(f) - json_validate_add_node(target_node_data) - except Exception as e: - util.exit_message( - f"Unable to load new node json def file '{target_node_file}\n{e}" - ) - - # Retrieve source node data - source_node_data = next( - (node for node in nodes if node["name"] == source_node), None - ) - if source_node_data is None: - util.exit_message(f"Source node '{source_node}' not found in cluster data.") - - for group in target_node_data.get("node_groups", []): - ssh_info = group.get("ssh") - backrest_info = group.get("backrest") - os_user = ssh_info.get("os_user", "") - ssh_key = ssh_info.get("private_key", "") - - new_node_data = { - "ssh": ssh_info, - "name": group.get("name", ""), - "is_active": group.get("is_active", ""), - "public_ip": group.get("public_ip", ""), - "private_ip": group.get("private_ip", ""), - "port": group.get("port", ""), - "path": group.get("path", ""), - "os_user": os_user, - "ssh_key": ssh_key, - } - - if "public_ip" not in new_node_data and "private_ip" not in new_node_data: - util.exit_message( - "Both public_ip and private_ip are missing in target node data." - ) - - if "public_ip" in source_node_data and "private_ip" in source_node_data: - source_node_data["ip_address"] = source_node_data["public_ip"] - else: - source_node_data["ip_address"] = source_node_data.get( - "public_ip", source_node_data.get("private_ip") - ) - - if "public_ip" in new_node_data and "private_ip" in new_node_data: - new_node_data["ip_address"] = new_node_data["public_ip"] - else: - new_node_data["ip_address"] = new_node_data.get( - "public_ip", new_node_data.get("private_ip") - ) - - # Fetch backrest settings from cluster JSON - backrest_settings = source_node_data.get("backrest", {}) - - stanza = backrest_settings.get("stanza", f"pg{pg}") - repo1_retention_full = backrest_settings.get("repo1-retention-full", "7") - log_level_console = backrest_settings.get("log-level-console", "info") - repo1_cipher_type = backrest_settings.get("repo1-cipher-type", "aes-256-cbc") - - rc = ssh_install_pgedge( - cluster_name, - db[0]["db_name"], - db_settings, - db[0]["db_user"], - db[0]["db_password"], - [new_node_data], - install, - verbose, - ) - - os_user = new_node_data["os_user"] - repo1_type = new_node_data.get("repo1_type", "posix") - port = source_node_data["port"] - pg1_path = f"{source_node_data['path']}/pgedge/data/pg{pg}" - - if not repo1_path: - cmd = f"{source_node_data['path']}/pgedge/pgedge install backrest" - message = f"Installing backrest" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - - repo1_path_default = f"/var/lib/pgbackrest/{source_node_data['name']}" - - repo1_path = backrest_settings.get("repo1_path", f"{repo1_path_default}") - - args = ( - f"--repo1-path {repo1_path} --stanza {stanza} " - f"--pg1-path {pg1_path} --repo1-type {repo1_type} " - f"--log-level-console {log_level_console} --pg1-port {port} " - f"--db-socket-path /tmp --repo1-cipher-type {repo1_cipher_type}" - ) - - cmd = f"{source_node_data['path']}/pgedge/pgedge backrest command stanza-create '{args}'" - message = f"Creating stanza {stanza}" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - - cmd = ( - f"{source_node_data['path']}/pgedge/pgedge backrest set_postgresqlconf {stanza} " - f"{pg1_path} {repo1_path} {repo1_type}" - ) - message = f"Modifying postgresql.conf file" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - - cmd = f"{source_node_data['path']}/pgedge/pgedge backrest set_hbaconf" - message = f"Modifying pg_hba.conf file" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - - sql_cmd = "select pg_reload_conf()" - cmd = f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" - message = f"Reload configuration pg_reload_conf()" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - - args = args + f" --repo1-retention-full={repo1_retention_full} --type=full" - cmd = ( - f"{source_node_data['path']}/pgedge/pgedge backrest command backup '{args}'" - ) - message = f"Creating full backup" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - else: - cmd = ( - f"{source_node_data['path']}/pgedge/pgedge backrest set_postgresqlconf {stanza} " - f"{pg1_path} {repo1_path} {repo1_type}" - ) - message = f"Modifying postgresql.conf file" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - - cmd = f"{source_node_data['path']}/pgedge/pgedge backrest set_hbaconf" - message = f"Modifying pg_hba.conf file" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - - sql_cmd = "select pg_reload_conf()" - cmd = f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" - message = f"Reload configuration pg_reload_conf()" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - - repo1_type = new_node_data.get("repo1_type", "posix") - if repo1_type == "s3": - for env_var in [ - "PGBACKREST_REPO1_S3_KEY", - "PGBACKREST_REPO1_S3_BUCKET", - "PGBACKREST_REPO1_S3_KEY_SECRET", - "PGBACKREST_REPO1_CIPHER_PASS", - ]: - if env_var not in os.environ: - util.exit_message(f"Environment variable {env_var} not set.") - s3_export_cmds = [ - f"export {env_var}={os.environ[env_var]}" - for env_var in [ - "PGBACKREST_REPO1_S3_KEY", - "PGBACKREST_REPO1_S3_BUCKET", - "PGBACKREST_REPO1_S3_KEY_SECRET", - "PGBACKREST_REPO1_CIPHER_PASS", - ] - ] - run_cmd( - " && ".join(s3_export_cmds), - source_node_data, - message="Setting S3 environment variables on source node", - verbose=verbose, - ) - run_cmd( - " && ".join(s3_export_cmds), - new_node_data, - message="Setting S3 environment variables on target node", - verbose=verbose, - ) - - cmd = f"{new_node_data['path']}/pgedge/pgedge install backrest" - message = f"Installing backrest" - run_cmd(cmd, new_node_data, message=message, verbose=verbose) - - manage_node(new_node_data, "stop", f"pg{pg}", verbose) - cmd = f'rm -rf {new_node_data["path"]}/pgedge/data/pg{pg}' - message = f"Removing old data directory" - run_cmd(cmd, new_node_data, message=message, verbose=verbose) - - args = f"--repo1-path {repo1_path} --repo1-cipher-type {repo1_cipher_type} " - - if backup_id: - args += f"--set={backup_id} " - - cmd = ( - f'{new_node_data["path"]}/pgedge/pgedge backrest command restore ' - f"--repo1-type={repo1_type} --stanza={stanza} " - f'--pg1-path={new_node_data["path"]}/pgedge/data/pg{pg} {args}' - ) - - message = f"Restoring backup" - run_cmd(cmd, new_node_data, message=message, verbose=verbose) - - pgd = f'{new_node_data["path"]}/pgedge/data/pg{pg}' - pgc = f"{pgd}/postgresql.conf" - - cmd = f"echo \"ssl_cert_file='{pgd}/server.crt'\" >> {pgc}" - message = f"Setting ssl_cert_file" - run_cmd(cmd, new_node_data, message=message, verbose=verbose) - - cmd = f"echo \"ssl_key_file='{pgd}/server.key'\" >> {pgc}" - message = f"Setting ssl_key_file" - run_cmd(cmd, new_node_data, message=message, verbose=verbose) - - cmd = f"echo \"log_directory='{pgd}/log'\" >> {pgc}" - message = f"Setting log_directory" - run_cmd(cmd, new_node_data, message=message, verbose=verbose) - - cmd = ( - f'echo "shared_preload_libraries = ' - f"'pg_stat_statements, snowflake, spock'\" >> {pgc}" - ) - message = f"Setting shared_preload_libraries" - run_cmd(cmd, new_node_data, message=message, verbose=verbose) - - cmd = ( - f'{new_node_data["path"]}/pgedge/pgedge backrest configure_replica {stanza} ' - f'{new_node_data["path"]}/pgedge/data/pg{pg} {source_node_data["ip_address"]} ' - f'{source_node_data["port"]} {source_node_data["os_user"]}' - ) - message = f"Configuring PITR on replica" - run_cmd(cmd, new_node_data, message=message, verbose=verbose) - - if script.strip() and os.path.isfile(script): - util.echo_cmd(f"{script}") - - terminate_cluster_transactions(nodes, db[0]["db_name"], f"pg{pg}", verbose) - - spock = db_settings["spock_version"] - v4 = True - spock_maj = 4 - if spock: - ver = [int(x) for x in spock.split(".")] - spock_maj = ver[0] - spock_min = ver[1] - if spock_maj >= 4: - v4 = True - - set_cluster_readonly(nodes, True, db[0]["db_name"], f"pg{pg}", v4, verbose) - manage_node(new_node_data, "start", f"pg{pg}", verbose) - time.sleep(5) - - check_cluster_lag(new_node_data, db[0]["db_name"], f"pg{pg}", verbose) - - sql_cmd = "SELECT pg_promote()" - cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" - message = f"Promoting standby to primary" - run_cmd(cmd, new_node_data, message=message, verbose=verbose) - - for mdb in db: - # Fetch all subscription names directly - sql_cmd = "SELECT sub_name FROM spock.subscription" - cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {mdb['db_name']}" - message = "Fetch existing subscriptions" - result = run_cmd( - cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True - ) - - subscriptions = [ - re.sub(r"\x1b\[[0-9;]*m", "", line.strip()) # Remove escape sequences - for line in result.stdout.splitlines()[2:] # Skip header lines - if line.strip() and not line.strip().startswith("(") # Exclude metadata lines - ] - - # Remove any remaining blank or invalid entries - subscriptions = [sub for sub in subscriptions if sub] - - # Drop each subscription if any exist - if subscriptions: - for sub_name in subscriptions: - cmd = f"{new_node_data['path']}/pgedge/pgedge spock sub-drop {sub_name} {mdb['db_name']}" - message = f"Dropping old subscription {sub_name}" - run_cmd(cmd, node=new_node_data, message=message, verbose=verbose) - else: - print("No subscriptions to drop.") - - # Check the number of nodes - sql_cmd = "SELECT node_name FROM spock.node" - cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {mdb['db_name']}" - message = "Check if there are nodes" - result = run_cmd( - cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True - ) - - # Parse node names from the output - print(f"\nRaw output:\n{result.stdout}") - nodes_list = [ - re.sub(r"\x1b\[[0-9;]*m", "", line.strip()) # Remove escape sequences - for line in result.stdout.splitlines()[2:] # Skip header lines - if line.strip() and not line.strip().startswith("(") # Exclude metadata lines - ] - - # Remove any remaining blank or invalid entries - nodes_list = [node for node in nodes_list if node] - - # Drop each node if any exist - if nodes_list: - for node_name in nodes_list: - cmd = f"{new_node_data['path']}/pgedge/pgedge spock node-drop {node_name} {mdb['db_name']}" - message = f"Dropping node {node_name}" - run_cmd(cmd, node=new_node_data, message=message, verbose=verbose) - else: - print("No nodes to drop.") - - create_node(new_node_data, mdb["db_name"], verbose) - - if not v4: - set_cluster_readonly(nodes, False, mdb["db_name"], f"pg{pg}", v4, verbose) - - create_sub(nodes, new_node_data, mdb["db_name"], verbose) - create_sub_new(nodes, new_node_data, mdb["db_name"], verbose) - - nc = os.path.join(new_node_data['path'], "pgedge", "pgedge ") - cmd = f'{nc} spock repset-add-table default "*" {mdb["db_name"]}' - - message = f"Adding all tables to repset" - run_cmd(cmd, new_node_data, message=message, verbose=verbose) - - cmd = f'{nc} spock repset-add-table default_insert_only "*" {mdb["db_name"]}' - run_cmd(cmd, new_node_data, message=message, verbose=verbose) - - if v4: - set_cluster_readonly(nodes, False, db[0]["db_name"], f"pg{pg}", v4, verbose) - - cmd = f'cd {new_node_data["path"]}/pgedge/; ./pgedge spock node-list {db[0]["db_name"]}' - message = f"Listing spock nodes" - result = run_cmd( - cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True - ) - print(f"\n{result.stdout}") - - sql_cmd = "select node_id,node_name from spock.node" - cmd = ( - f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" - ) - message = f"List nodes" - result = run_cmd( - cmd, - node=source_node_data, - message=message, - verbose=verbose, - capture_output=True, - ) - print(f"\n{result.stdout}") - - for node in nodes: - sql_cmd = ( - "select sub_id,sub_name,sub_enabled,sub_slot_name," - "sub_replication_sets from spock.subscription" - ) - cmd = f"{node['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" - message = f"List subscriptions" - result = run_cmd( - cmd, node=node, message=message, verbose=verbose, capture_output=True - ) - print(f"\n{result.stdout}") - - sql_cmd = ( - "select sub_id,sub_name,sub_enabled,sub_slot_name," - "sub_replication_sets from spock.subscription" - ) - cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" - message = f"List subscriptions" - result = run_cmd( - cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True - ) - print(f"\n{result.stdout}") - - # Remove unnecessary keys before appending new node to the cluster data - new_node_data.pop("repo1_type", None) - new_node_data.pop("os_user", None) - new_node_data.pop("ssh_key", None) - - # Append new node data to the cluster JSON - node_group = target_node_data.get - cluster_data["node_groups"].append(new_node_data) - cluster_data["update_date"] = datetime.datetime.now().astimezone().isoformat() - - write_cluster_json(cluster_name, cluster_data) - - -def json_validate_add_node(data): - """Validate the structure of a node configuration JSON file.""" - required_keys = ["json_version", "node_groups"] - node_group_keys = [ - "ssh", - "name", - "is_active", - "public_ip", - "private_ip", - "port", - "path", - ] - ssh_keys = ["os_user", "private_key"] - if "json_version" not in data or data["json_version"] == "1.0": - util.exit_message("Invalid or missing JSON version.") - - for key in required_keys: - if key not in data: - util.exit_message(f"Key '{key}' missing from JSON data.") - - for group in data["node_groups"]: - for node_group_key in node_group_keys: - if node_group_key not in group: - util.exit_message(f"Key '{node_group_key}' missing from node group.") - - ssh_info = group.get("ssh", {}) - for ssh_key in ssh_keys: - if ssh_key not in ssh_info: - util.exit_message(f"Key '{ssh_key}' missing from ssh configuration.") - - if "public_ip" not in group and "private_ip" not in group: - util.exit_message( - "Both 'public_ip' and 'private_ip' are missing from node group." - ) - - util.message(f"New node json file structure is valid.", "success") - def remove_node(cluster_name, node_name): """Remove a node from the cluster configuration. From d5ad11afa0dadc9c81b6f2c1143a4cb7a5042a4f Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Wed, 2 Apr 2025 03:19:15 +0500 Subject: [PATCH 02/19] yaml file created for backrest json in backrest dir --- cli/scripts/cluster.py | 536 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 536 insertions(+) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index 23ff4ccf..3312b8b1 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -1563,6 +1563,542 @@ def init(cluster_name, install=True): etcd.configure_etcd(node, sub_nodes) ha_patroni.configure_patroni(node, sub_nodes, db[0], db_settings) + +def add_node( + cluster_name, + source_node, + target_node, + repo1_path=None, + backup_id=None, + script=" ", + stanza=" ", + install=True, +): + """ + Adds a new node to a cluster, copying configurations from a specified + source node. + + Args: + cluster_name (str): The name of the cluster to which the node is being added. + source_node (str): The node from which configurations are copied. + target_node (str): The new node. + repo1_path (str): The repo1 path to use. + backup_id (str): Backup ID. + stanza (str): Stanza name. + script (str): Bash script. + install (bool): Whether to install pgEdge on the new node. + """ + + json_validate(cluster_name) + db, db_settings, nodes = load_json(cluster_name) + + cluster_data = get_cluster_json(cluster_name) + if cluster_data is None: + util.exit_message("Cluster data is missing.") + pg = db_settings["pg_version"] + pgV = f"pg{pg}" + verbose = cluster_data.get("log_level", "info") + + # Load and validate the target node JSON + target_node_file = f"{target_node}.json" + if not os.path.isfile(target_node_file): + util.exit_message(f"New node json file '{target_node_file}' not found") + try: + with open(target_node_file, "r") as f: + target_node_data = json.load(f) + json_validate_add_node(target_node_data) + except Exception as e: + util.exit_message(f"Unable to load new node json def file '{target_node_file}\n{e}") + + # Retrieve source node data + source_node_data = next((node for node in nodes if node["name"] == source_node), None) + if source_node_data is None: + util.exit_message(f"Source node '{source_node}' not found in cluster data.") + + # Process target node configuration(s) and update backrest paths. + for group in target_node_data.get("node_groups", []): + ssh_info = group.get("ssh") + # Capture backrest info only if provided in the target JSON. + backrest_info = group.get("backrest") + os_user = ssh_info.get("os_user", "") + ssh_key = ssh_info.get("private_key", "") + new_node_data = { + "ssh": ssh_info, + "name": group.get("name", ""), + "is_active": group.get("is_active", ""), + "public_ip": group.get("public_ip", ""), + "private_ip": group.get("private_ip", ""), + "port": group.get("port", ""), + "path": group.get("path", ""), + "os_user": os_user, + "ssh_key": ssh_key, + } + if backrest_info: + new_node_data["backrest"] = backrest_info.copy() + # Force the repo1_path to be exactly "/var/lib/pgbackrest/{node_name}" + new_node_data["backrest"]["repo1_path"] = f"/var/lib/pgbackrest/{new_node_data['name']}" + mkdir_cmd = f"sudo mkdir -p {new_node_data['backrest']['repo1_path']}" + run_cmd( + cmd=mkdir_cmd, + node=new_node_data, + message=f"Creating backrest directory {new_node_data['backrest']['repo1_path']} on node '{new_node_data['name']}'", + verbose=verbose + ) + + # Ensure valid IP addresses. + if "public_ip" not in new_node_data and "private_ip" not in new_node_data: + util.exit_message("Both public_ip and private_ip are missing in target node data.") + if "public_ip" in source_node_data and "private_ip" in source_node_data: + source_node_data["ip_address"] = source_node_data["public_ip"] + else: + source_node_data["ip_address"] = source_node_data.get("public_ip", source_node_data.get("private_ip")) + if "public_ip" in new_node_data and "private_ip" in new_node_data: + new_node_data["ip_address"] = new_node_data["public_ip"] + else: + new_node_data["ip_address"] = new_node_data.get("public_ip", new_node_data.get("private_ip")) + + # Process backrest configuration only if new_node_data contains a backrest block. + if "backrest" in new_node_data and new_node_data["backrest"]: + # Fetch backrest settings from source node. + backrest_settings = source_node_data.get("backrest", {}) + stanza = backrest_settings.get("stanza", f"pg{pg}") + repo1_retention_full = backrest_settings.get("repo1-retention-full", "7") + log_level_console = backrest_settings.get("log-level-console", "info") + repo1_cipher_type = backrest_settings.get("repo1-cipher-type", "aes-256-cbc") + + os_user = new_node_data["os_user"] + repo1_type = new_node_data.get("repo1_type", "posix") + port = source_node_data["port"] + pg1_path = f"{source_node_data['path']}/pgedge/data/pg{pg}" + + if not repo1_path: + cmd = f"{source_node_data['path']}/pgedge/pgedge install backrest" + message = f"Installing backrest" + run_cmd(cmd, source_node_data, message=message, verbose=verbose) + repo1_path_default = f"/var/lib/pgbackrest/{source_node_data['name']}" + repo1_path = backrest_settings.get("repo1_path", f"{repo1_path_default}") + args = ( + f"--repo1-path {repo1_path} --stanza {stanza} " + f"--pg1-path {pg1_path} --repo1-type {repo1_type} " + f"--log-level-console {log_level_console} --pg1-port {port} " + f"--db-socket-path /tmp --repo1-cipher-type {repo1_cipher_type}" + ) + cmd = f"{source_node_data['path']}/pgedge/pgedge backrest command stanza-create '{args}'" + message = f"Creating stanza {stanza}" + run_cmd(cmd, source_node_data, message=message, verbose=verbose) + cmd = (f"{source_node_data['path']}/pgedge/pgedge backrest set_postgresqlconf {stanza} " + f"{pg1_path} {repo1_path} {repo1_type}") + message = f"Modifying postgresql.conf file" + run_cmd(cmd, source_node_data, message=message, verbose=verbose) + cmd = f"{source_node_data['path']}/pgedge/pgedge backrest set_hbaconf" + message = f"Modifying pg_hba.conf file" + run_cmd(cmd, source_node_data, message=message, verbose=verbose) + sql_cmd = "select pg_reload_conf()" + cmd = f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" + message = f"Reload configuration pg_reload_conf()" + run_cmd(cmd, source_node_data, message=message, verbose=verbose) + args = args + f" --repo1-retention-full={repo1_retention_full} --type=full" + cmd = f"{source_node_data['path']}/pgedge/pgedge backrest command backup '{args}'" + message = f"Creating full backup" + run_cmd(cmd, source_node_data, message=message, verbose=verbose) + else: + cmd = (f"{source_node_data['path']}/pgedge/pgedge backrest set_postgresqlconf {stanza} " + f"{pg1_path} {repo1_path} {repo1_type}") + message = f"Modifying postgresql.conf file" + run_cmd(cmd, source_node_data, message=message, verbose=verbose) + cmd = f"{source_node_data['path']}/pgedge/pgedge backrest set_hbaconf" + message = f"Modifying pg_hba.conf file" + run_cmd(cmd, source_node_data, message=message, verbose=verbose) + sql_cmd = "select pg_reload_conf()" + cmd = f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" + message = f"Reload configuration pg_reload_conf()" + run_cmd(cmd, source_node_data, message=message, verbose=verbose) + repo1_type = new_node_data.get("repo1_type", "posix") + if repo1_type == "s3": + for env_var in [ + "PGBACKREST_REPO1_S3_KEY", + "PGBACKREST_REPO1_S3_BUCKET", + "PGBACKREST_REPO1_S3_KEY_SECRET", + "PGBACKREST_REPO1_CIPHER_PASS", + ]: + if env_var not in os.environ: + util.exit_message(f"Environment variable {env_var} not set.") + s3_export_cmds = [ + f"export {env_var}={os.environ[env_var]}" + for env_var in [ + "PGBACKREST_REPO1_S3_KEY", + "PGBACKREST_REPO1_S3_BUCKET", + "PGBACKREST_REPO1_S3_KEY_SECRET", + "PGBACKREST_REPO1_CIPHER_PASS", + ] + ] + run_cmd(" && ".join(s3_export_cmds), + source_node_data, + message="Setting S3 environment variables on source node", + verbose=verbose) + run_cmd(" && ".join(s3_export_cmds), + new_node_data, + message="Setting S3 environment variables on target node", + verbose=verbose) + # End of backrest configuration block + + rc = ssh_install_pgedge( + cluster_name, + db[0]["db_name"], + db_settings, + db[0]["db_user"], + db[0]["db_password"], + [new_node_data], + install, + verbose, + ) + + os_user = new_node_data["os_user"] + repo1_type = new_node_data.get("repo1_type", "posix") + port = source_node_data["port"] + pg1_path = f"{source_node_data['path']}/pgedge/data/pg{pg}" + + if not repo1_path: + cmd = f"{source_node_data['path']}/pgedge/pgedge install backrest" + message = f"Installing backrest" + run_cmd(cmd, source_node_data, message=message, verbose=verbose) + repo1_path_default = f"/var/lib/pgbackrest/{source_node_data['name']}" + repo1_path = backrest_settings.get("repo1_path", f"{repo1_path_default}") + args = ( + f"--repo1-path {repo1_path} --stanza {stanza} " + f"--pg1-path {pg1_path} --repo1-type {repo1_type} " + f"--log-level-console {log_level_console} --pg1-port {port} " + f"--db-socket-path /tmp --repo1-cipher-type {repo1_cipher_type}" + ) + cmd = f"{source_node_data['path']}/pgedge/pgedge backrest command stanza-create '{args}'" + message = f"Creating stanza {stanza}" + run_cmd(cmd, source_node_data, message=message, verbose=verbose) + cmd = (f"{source_node_data['path']}/pgedge/pgedge backrest set_postgresqlconf {stanza} " + f"{pg1_path} {repo1_path} {repo1_type}") + message = f"Modifying postgresql.conf file" + run_cmd(cmd, source_node_data, message=message, verbose=verbose) + cmd = f"{source_node_data['path']}/pgedge/pgedge backrest set_hbaconf" + message = f"Modifying pg_hba.conf file" + run_cmd(cmd, source_node_data, message=message, verbose=verbose) + sql_cmd = "select pg_reload_conf()" + cmd = f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" + message = f"Reload configuration pg_reload_conf()" + run_cmd(cmd, source_node_data, message=message, verbose=verbose) + args = args + f" --repo1-retention-full={repo1_retention_full} --type=full" + cmd = f"{source_node_data['path']}/pgedge/pgedge backrest command backup '{args}'" + message = f"Creating full backup" + run_cmd(cmd, source_node_data, message=message, verbose=verbose) + else: + cmd = (f"{source_node_data['path']}/pgedge/pgedge backrest set_postgresqlconf {stanza} " + f"{pg1_path} {repo1_path} {repo1_type}") + message = f"Modifying postgresql.conf file" + run_cmd(cmd, source_node_data, message=message, verbose=verbose) + cmd = f"{source_node_data['path']}/pgedge/pgedge backrest set_hbaconf" + message = f"Modifying pg_hba.conf file" + run_cmd(cmd, source_node_data, message=message, verbose=verbose) + sql_cmd = "select pg_reload_conf()" + cmd = f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" + message = f"Reload configuration pg_reload_conf()" + run_cmd(cmd, source_node_data, message=message, verbose=verbose) + repo1_type = new_node_data.get("repo1_type", "posix") + if repo1_type == "s3": + for env_var in [ + "PGBACKREST_REPO1_S3_KEY", + "PGBACKREST_REPO1_S3_BUCKET", + "PGBACKREST_REPO1_S3_KEY_SECRET", + "PGBACKREST_REPO1_CIPHER_PASS", + ]: + if env_var not in os.environ: + util.exit_message(f"Environment variable {env_var} not set.") + s3_export_cmds = [ + f"export {env_var}={os.environ[env_var]}" + for env_var in [ + "PGBACKREST_REPO1_S3_KEY", + "PGBACKREST_REPO1_S3_BUCKET", + "PGBACKREST_REPO1_S3_KEY_SECRET", + "PGBACKREST_REPO1_CIPHER_PASS", + ] + ] + run_cmd(" && ".join(s3_export_cmds), + source_node_data, + message="Setting S3 environment variables on source node", + verbose=verbose) + run_cmd(" && ".join(s3_export_cmds), + new_node_data, + message="Setting S3 environment variables on target node", + verbose=verbose) + + cmd = f"{new_node_data['path']}/pgedge/pgedge install backrest" + message = f"Installing backrest" + run_cmd(cmd, new_node_data, message=message, verbose=verbose) + + manage_node(new_node_data, "stop", f"pg{pg}", verbose) + cmd = f'rm -rf {new_node_data["path"]}/pgedge/data/pg{pg}' + message = f"Removing old data directory" + run_cmd(cmd, new_node_data, message=message, verbose=verbose) + + args = f"--repo1-path {repo1_path} --repo1-cipher-type {repo1_cipher_type} " + if backup_id: + args += f"--set={backup_id} " + cmd = (f'{new_node_data["path"]}/pgedge/pgedge backrest command restore ' + f"--repo1-type={repo1_type} --stanza={stanza} " + f'--pg1-path={new_node_data["path"]}/pgedge/data/pg{pg} {args}') + message = f"Restoring backup" + run_cmd(cmd, new_node_data, message=message, verbose=verbose) + + pgd = f'{new_node_data["path"]}/pgedge/data/pg{pg}' + pgc = f"{pgd}/postgresql.conf" + cmd = f"echo \"ssl_cert_file='{pgd}/server.crt'\" >> {pgc}" + message = f"Setting ssl_cert_file" + run_cmd(cmd, new_node_data, message=message, verbose=verbose) + cmd = f"echo \"ssl_key_file='{pgd}/server.key'\" >> {pgc}" + message = f"Setting ssl_key_file" + run_cmd(cmd, new_node_data, message=message, verbose=verbose) + cmd = f"echo \"log_directory='{pgd}/log'\" >> {pgc}" + message = f"Setting log_directory" + run_cmd(cmd, new_node_data, message=message, verbose=verbose) + cmd = (f'echo "shared_preload_libraries = ' + f"'pg_stat_statements, snowflake, spock'\" >> {pgc}") + message = f"Setting shared_preload_libraries" + run_cmd(cmd, new_node_data, message=message, verbose=verbose) + cmd = (f'{new_node_data["path"]}/pgedge/pgedge backrest configure_replica {stanza} ' + f'{new_node_data["path"]}/pgedge/data/pg{pg} {source_node_data["ip_address"]} ' + f'{source_node_data["port"]} {source_node_data["os_user"]}') + message = f"Configuring PITR on replica" + run_cmd(cmd, new_node_data, message=message, verbose=verbose) + if script.strip() and os.path.isfile(script): + util.echo_cmd(f"{script}") + terminate_cluster_transactions(nodes, db[0]["db_name"], f"pg{pg}", verbose) + spock = db_settings["spock_version"] + v4 = True + spock_maj = 4 + if spock: + ver = [int(x) for x in spock.split(".")] + spock_maj = ver[0] + spock_min = ver[1] + if spock_maj >= 4: + v4 = True + set_cluster_readonly(nodes, True, db[0]["db_name"], f"pg{pg}", v4, verbose) + manage_node(new_node_data, "start", f"pg{pg}", verbose) + time.sleep(5) + check_cluster_lag(new_node_data, db[0]["db_name"], f"pg{pg}", verbose) + sql_cmd = "SELECT pg_promote()" + cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" + message = f"Promoting standby to primary" + run_cmd(cmd, new_node_data, message=message, verbose=verbose) + for mdb in db: + sql_cmd = "SELECT sub_name FROM spock.subscription" + cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {mdb['db_name']}" + message = "Fetch existing subscriptions" + result = run_cmd(cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True) + subscriptions = [re.sub(r"\x1b\[[0-9;]*m", "", line.strip()) + for line in result.stdout.splitlines()[2:] + if line.strip() and not line.strip().startswith("(")] + subscriptions = [sub for sub in subscriptions if sub] + if subscriptions: + for sub_name in subscriptions: + cmd = f"{new_node_data['path']}/pgedge/pgedge spock sub-drop {sub_name} {mdb['db_name']}" + message = f"Dropping old subscription {sub_name}" + run_cmd(cmd, node=new_node_data, message=message, verbose=verbose) + else: + print("No subscriptions to drop.") + sql_cmd = "SELECT node_name FROM spock.node" + cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {mdb['db_name']}" + message = "Check if there are nodes" + result = run_cmd(cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True) + print(f"\nRaw output:\n{result.stdout}") + nodes_list = [re.sub(r"\x1b\[[0-9;]*m", "", line.strip()) + for line in result.stdout.splitlines()[2:] + if line.strip() and not line.strip().startswith("(")] + nodes_list = [node for node in nodes_list if node] + if nodes_list: + for node_name in nodes_list: + cmd = f"{new_node_data['path']}/pgedge/pgedge spock node-drop {node_name} {mdb['db_name']}" + message = f"Dropping node {node_name}" + run_cmd(cmd, node=new_node_data, message=message, verbose=verbose) + else: + print("No nodes to drop.") + create_node(new_node_data, mdb["db_name"], verbose) + if not v4: + set_cluster_readonly(nodes, False, mdb["db_name"], f"pg{pg}", v4, verbose) + create_sub(nodes, new_node_data, mdb["db_name"], verbose) + create_sub_new(nodes, new_node_data, mdb["db_name"], verbose) + nc = os.path.join(new_node_data['path'], "pgedge", "pgedge ") + cmd = f'{nc} spock repset-add-table default "*" {mdb["db_name"]}' + message = f"Adding all tables to repset" + run_cmd(cmd, new_node_data, message=message, verbose=verbose) + cmd = f'{nc} spock repset-add-table default_insert_only "*" {mdb["db_name"]}' + run_cmd(cmd, new_node_data, message=message, verbose=verbose) + if v4: + set_cluster_readonly(nodes, False, db[0]["db_name"], f"pg{pg}", v4, verbose) + cmd = f'cd {new_node_data["path"]}/pgedge/; ./pgedge spock node-list {db[0]["db_name"]}' + message = f"Listing spock nodes" + result = run_cmd(cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True) + print(f"\n{result.stdout}") + sql_cmd = "select node_id,node_name from spock.node" + cmd = f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" + message = f"List nodes" + result = run_cmd(cmd, node=source_node_data, message=message, verbose=verbose, capture_output=True) + print(f"\n{result.stdout}") + for node in nodes: + sql_cmd = ("select sub_id,sub_name,sub_enabled,sub_slot_name," + "sub_replication_sets from spock.subscription") + cmd = f"{node['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" + message = f"List subscriptions" + result = run_cmd(cmd, node=node, message=message, verbose=verbose, capture_output=True) + print(f"\n{result.stdout}") + sql_cmd = ("select sub_id,sub_name,sub_enabled,sub_slot_name," + "sub_replication_sets from spock.subscription") + cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" + message = f"List subscriptions" + result = run_cmd(cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True) + print(f"\n{result.stdout}") + cluster_data["node_groups"].append(new_node_data) + cluster_data["update_date"] = datetime.datetime.now().astimezone().isoformat() + write_cluster_json(cluster_name, cluster_data) + + # Combine target node backup configuration commands into one command with verbose disabled. + pgedge_dir = f"{new_node_data['path']}/pgedge" + combined_target_cmd = ( + f"sudo mkdir -p /var/lib/pgbackrest_restore/{new_node_data['name']} && " + f"cd {pgedge_dir} && ./pgedge set BACKUP restore_path /var/lib/pgbackrest_restore/{new_node_data['name']} && " + f"sudo mkdir -p /var/lib/pgbackrest/{new_node_data['name']} && " + f"cd {pgedge_dir} && ./pgedge set BACKUP repo1-path /var/lib/pgbackrest/{new_node_data['name']} && " + f"cd {pgedge_dir} && ./pgedge set BACKUP repo1-host-user {new_node_data.get('os_user', 'postgres')} && " + f"cd {pgedge_dir} && ./pgedge set BACKUP pg1-path {pgedge_dir}/data/pg{pg} && " + f"cd {pgedge_dir} && ./pgedge set BACKUP pg1-user {new_node_data.get('os_user', 'postgres')} && " + f"cd {pgedge_dir} && ./pgedge set BACKUP pg1-port {new_node_data.get('port', '6435')}" + ) + run_cmd( + cmd=combined_target_cmd, + node=new_node_data, + message="Setting all target node BACKUP configuration", + verbose=False + ) + + # --- Begin Backup Section for Target Node: Ensure repo1_path is writable. + node_os_user = new_node_data.get("os_user", "postgres") + combined_chown_cmd = f"sudo chown -R {node_os_user}:{node_os_user} {repo1_path} && sudo chmod -R 775 {repo1_path}" + run_cmd( + cmd=combined_chown_cmd, + node=new_node_data, + message=f"Setting ownership and permissions on {repo1_path}", + verbose=False + ) + capture_backrest_config(cluster_name) +# Cleanup backrest settings if target JSON did not include them + cleanup_backrest_from_cluster(cluster_data, target_node_data) +def capture_backrest_config(cluster_name, verbose=False): + """ + Capture and clean BackRest configuration for all nodes (and sub-nodes) in the cluster + that have BackRest enabled. + + For each node with a non-empty 'backrest' configuration in the cluster JSON, + this function will: + 1. Change directory to the node's pgedge directory. + 2. Run "./pgedge backrest show-config" and capture the output. + 3. Clean the output by removing extraneous lines, ANSI escape codes, and literal "^[[0m" strings, + then parse it into a dictionary. + 4. Write the cleaned configuration as YAML to a file named + "backrest_{node_name}.yaml" in the node's "pgedge/backrest" directory. + """ + # Load the cluster configuration + db, db_settings, nodes = load_json(cluster_name) + + # Create a combined list of all nodes (including sub-nodes) + all_nodes = [] + for node in nodes: + all_nodes.append(node) + if "sub_nodes" in node and isinstance(node["sub_nodes"], list): + all_nodes.extend(node["sub_nodes"]) + + # Regex to remove ANSI escape sequences + ansi_escape = re.compile(r'\x1B[@-_][0-?]*[ -/]*[@-~]') + + for node in all_nodes: + if node.get("backrest"): + cmd = f"cd {node['path']}/pgedge && ./pgedge backrest show-config" + result = run_cmd( + cmd, + node, + message=f"Capturing BackRest configuration for node {node['name']}", + verbose=verbose, + capture_output=True + ) + # Debug: Check raw command output + util.message(f"[DEBUG] Raw output for node {node['name']}:\n{result.stdout}", "debug") + + # Remove ANSI escape sequences and literal "^[[0m" + output = ansi_escape.sub('', result.stdout) + output = output.replace("^[[0m", "") + + # Debug: Print cleaned output + util.message(f"[DEBUG] Cleaned output for node {node['name']}:\n{output}", "debug") + + # Parse the cleaned output into a dictionary + config_dict = {} + for line in output.splitlines(): + line = line.strip() + # Skip empty lines, header/footer lines, or lines without '=' + if not line or line.startswith('#') or ('=' not in line): + continue + parts = line.split('=', 1) + if len(parts) != 2: + continue + key = parts[0].strip() + value = parts[1].strip() + if value.isdigit(): + value = int(value) + config_dict[key] = value + + # Debug: Check the resulting dictionary + util.message(f"[DEBUG] Parsed config for node {node['name']}: {config_dict}", "debug") + + if not config_dict: + util.message(f"[WARNING] No configuration parsed for node {node['name']}.", "warning") + + # Ensure the "pgedge/backrest" directory exists on the node + backrest_dir = os.path.join(node["path"], "pgedge", "backrest") + os.makedirs(backrest_dir, exist_ok=True) + file_path = os.path.join(backrest_dir, f"backrest_{node['name']}.yaml") + + try: + with open(file_path, "w") as yaml_file: + yaml.dump(config_dict, yaml_file, default_flow_style=False) + util.message( + f"Cleaned BackRest configuration for node '{node['name']}' written to {file_path}", + "info" + ) + except Exception as e: + util.exit_message( + f"Failed to write cleaned BackRest configuration for node '{node['name']}': {e}" + ) + else: + util.message( + f"Node '{node['name']}' does not have BackRest enabled; skipping.", + "info" + ) +def cleanup_backrest_from_cluster(cluster_json, target_json): + """ + Compare the node groups in the target JSON with the cluster JSON. + For each node in cluster_json["node_groups"], if the corresponding node (by name) + is not present in target_json's node groups or does not contain a "backrest" key, + then remove the "backrest" key from that node in cluster_json. + + Args: + cluster_json (dict): The main cluster configuration. + target_json (dict): The target node configuration JSON. + """ + # Create a mapping of node names to their configuration from the target JSON. + target_nodes = {group.get("name"): group for group in target_json.get("node_groups", [])} + + for node in cluster_json.get("node_groups", []): + node_name = node.get("name") + target_group = target_nodes.get(node_name) + # If the target group does not exist or does not contain a backrest key, delete backrest in the main config. + if not target_group or "backrest" not in target_group: + if "backrest" in node: + del node["backrest"] def remove_node(cluster_name, node_name): """Remove a node from the cluster configuration. From 3515dd34a1638c14425faa59b01c22dbe4d3a525 Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Wed, 2 Apr 2025 19:23:36 +0500 Subject: [PATCH 03/19] Update cluster.py json validate add --- cli/scripts/cluster.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index 3312b8b1..875a70a8 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -2099,6 +2099,42 @@ def cleanup_backrest_from_cluster(cluster_json, target_json): if not target_group or "backrest" not in target_group: if "backrest" in node: del node["backrest"] +def json_validate_add_node(data): + """Validate the structure of a node configuration JSON file.""" + required_keys = ["json_version", "node_groups"] + node_group_keys = [ + "ssh", + "name", + "is_active", + "public_ip", + "private_ip", + "port", + "path", + ] + ssh_keys = ["os_user", "private_key"] + if "json_version" not in data or data["json_version"] == "1.0": + util.exit_message("Invalid or missing JSON version.") + + for key in required_keys: + if key not in data: + util.exit_message(f"Key '{key}' missing from JSON data.") + + for group in data["node_groups"]: + for node_group_key in node_group_keys: + if node_group_key not in group: + util.exit_message(f"Key '{node_group_key}' missing from node group.") + + ssh_info = group.get("ssh", {}) + for ssh_key in ssh_keys: + if ssh_key not in ssh_info: + util.exit_message(f"Key '{ssh_key}' missing from ssh configuration.") + + if "public_ip" not in group and "private_ip" not in group: + util.exit_message( + "Both 'public_ip' and 'private_ip' are missing from node group." + ) + + util.message(f"New node json file structure is valid.", "success") def remove_node(cluster_name, node_name): """Remove a node from the cluster configuration. From 895ee3cf6797268adbd91f1a277701b0908bd146 Mon Sep 17 00:00:00 2001 From: Matthew Mols Date: Wed, 2 Apr 2025 23:04:58 -0500 Subject: [PATCH 04/19] add build_cli.sh script --- build_cli.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 build_cli.sh diff --git a/build_cli.sh b/build_cli.sh new file mode 100755 index 00000000..328ac1ad --- /dev/null +++ b/build_cli.sh @@ -0,0 +1,6 @@ +source ./env.sh +rm -f $OUT/hub-$hubV* +rm -f $OUT/$bundle-cli-$hubV* +./build.sh -X posix -c $bundle-cli -N $hubV + +exit 0 \ No newline at end of file From ee4a264c8e0dfb6922d50921d570f1d763c9ba38 Mon Sep 17 00:00:00 2001 From: Matthew Mols Date: Wed, 2 Apr 2025 23:05:21 -0500 Subject: [PATCH 05/19] mount host github repo in container --- devel/setup/compose/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/devel/setup/compose/docker-compose.yaml b/devel/setup/compose/docker-compose.yaml index 5a3545c8..bfe63f91 100644 --- a/devel/setup/compose/docker-compose.yaml +++ b/devel/setup/compose/docker-compose.yaml @@ -37,6 +37,7 @@ services: - mynetwork volumes: - ./repo:/home/build/dev/out + - ${GITHUB_REPO_ROOT:-../../../}:/home/build/dev/cli stdin_open: true tty: true From f6aa5d128d40fa4ee420f715fb220d41bba48a73 Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Fri, 4 Apr 2025 03:41:24 +0500 Subject: [PATCH 06/19] backrest installing on none backrest configured source node creating backup , backrest installing on target if mentioned in target json and configuring backrest/taking backup else cleanup of backrest --- cli/scripts/cluster.py | 618 ++++++++++++++++++++++++++--------------- 1 file changed, 396 insertions(+), 222 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index 875a70a8..d960d939 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -716,7 +716,6 @@ def json_template(cluster_name, db, num_nodes, usr, passwd, pg, port): json_create(cluster_name, num_nodes, db, usr, passwd, pg, port, True) - def json_create( cluster_name, num_nodes, db, usr, passwd, pg_ver=None, port=None, force=False ): @@ -748,6 +747,8 @@ def message(message, level="info"): print(f"{message}") util = Util() + + # Assuming meta is a module that provides default PostgreSQL and Spock versions pg_default, pgs = meta.get_default_pg() if not pg_ver: spock_default, spocks = meta.get_default_spock(str(pg_default)) @@ -929,8 +930,9 @@ def get_cluster_info(cluster_name): util.exit_message( "Invalid pgBackRest repository type. Allowed values are 'posix' or 's3'." ) + # Create base BackRest configuration backrest_json = { - "stanza": "demo_stanza", + "stanza": f"{cluster_name}_stanza_", # base stanza; node name will be appended later "repo1_path": backrest_storage_path, "repo1_retention_full": "7", "log_level_console": "info", @@ -1010,6 +1012,8 @@ def get_cluster_info(cluster_name): node_backrest = backrest_json.copy() # Append the node name to the repo1_path so each node gets a unique backup directory. node_backrest["repo1_path"] = f"{node_backrest['repo1_path'].rstrip('/')}/{node_json['name']}" + # Update the stanza value to include the node name. + node_backrest["stanza"] = f"{cluster_name}_stanza_{node_json['name']}" node_json["backrest"] = node_backrest if is_ha_cluster: @@ -1080,16 +1084,15 @@ def get_cluster_info(cluster_name): if backrest_enabled: sub_node_backrest = backrest_json.copy() sub_node_backrest["repo1_path"] = f"{sub_node_backrest['repo1_path'].rstrip('/')}/{sub_node_json['name']}" + sub_node_backrest["stanza"] = f"{cluster_name}_stanza_{sub_node_json['name']}" sub_node_json["backrest"] = sub_node_backrest - node_json["sub_nodes"].append(sub_node_json) + node_json.setdefault("sub_nodes", []).append(sub_node_json) node_groups.append(node_json) cluster_json["node_groups"] = node_groups - # ... [validation and file saving code remains unchanged] - # Validate configuration validation_errors = [] @@ -1579,16 +1582,17 @@ def add_node( source node. Args: - cluster_name (str): The name of the cluster to which the node is being added. + cluster_name (str): The name of the cluster to which the node is being + added. source_node (str): The node from which configurations are copied. target_node (str): The new node. repo1_path (str): The repo1 path to use. backup_id (str): Backup ID. stanza (str): Stanza name. script (str): Bash script. - install (bool): Whether to install pgEdge on the new node. """ - + if (repo1_path and not backup_id) or (backup_id and not repo1_path): + util.exit_message("Both repo1_path and backup_id must be supplied together.") json_validate(cluster_name) db, db_settings, nodes = load_json(cluster_name) @@ -1603,25 +1607,32 @@ def add_node( target_node_file = f"{target_node}.json" if not os.path.isfile(target_node_file): util.exit_message(f"New node json file '{target_node_file}' not found") + try: with open(target_node_file, "r") as f: target_node_data = json.load(f) json_validate_add_node(target_node_data) except Exception as e: - util.exit_message(f"Unable to load new node json def file '{target_node_file}\n{e}") + util.exit_message( + f"Unable to load new node json def file '{target_node_file}\n{e}" + ) + + # NEW: Check if target JSON has backrest configuration + target_backrest_settings = target_node_data.get("backrest", {}) # Retrieve source node data - source_node_data = next((node for node in nodes if node["name"] == source_node), None) + source_node_data = next( + (node for node in nodes if node["name"] == source_node), None + ) if source_node_data is None: util.exit_message(f"Source node '{source_node}' not found in cluster data.") - # Process target node configuration(s) and update backrest paths. for group in target_node_data.get("node_groups", []): ssh_info = group.get("ssh") - # Capture backrest info only if provided in the target JSON. backrest_info = group.get("backrest") os_user = ssh_info.get("os_user", "") ssh_key = ssh_info.get("private_key", "") + new_node_data = { "ssh": ssh_info, "name": group.get("name", ""), @@ -1633,114 +1644,172 @@ def add_node( "os_user": os_user, "ssh_key": ssh_key, } - if backrest_info: - new_node_data["backrest"] = backrest_info.copy() - # Force the repo1_path to be exactly "/var/lib/pgbackrest/{node_name}" - new_node_data["backrest"]["repo1_path"] = f"/var/lib/pgbackrest/{new_node_data['name']}" - mkdir_cmd = f"sudo mkdir -p {new_node_data['backrest']['repo1_path']}" - run_cmd( - cmd=mkdir_cmd, - node=new_node_data, - message=f"Creating backrest directory {new_node_data['backrest']['repo1_path']} on node '{new_node_data['name']}'", - verbose=verbose - ) - - # Ensure valid IP addresses. + if "public_ip" not in new_node_data and "private_ip" not in new_node_data: - util.exit_message("Both public_ip and private_ip are missing in target node data.") + util.exit_message( + "Both public_ip and private_ip are missing in target node data." + ) + if "public_ip" in source_node_data and "private_ip" in source_node_data: source_node_data["ip_address"] = source_node_data["public_ip"] else: - source_node_data["ip_address"] = source_node_data.get("public_ip", source_node_data.get("private_ip")) + source_node_data["ip_address"] = source_node_data.get( + "public_ip", source_node_data.get("private_ip") + ) + if "public_ip" in new_node_data and "private_ip" in new_node_data: new_node_data["ip_address"] = new_node_data["public_ip"] else: - new_node_data["ip_address"] = new_node_data.get("public_ip", new_node_data.get("private_ip")) - - # Process backrest configuration only if new_node_data contains a backrest block. - if "backrest" in new_node_data and new_node_data["backrest"]: - # Fetch backrest settings from source node. - backrest_settings = source_node_data.get("backrest", {}) - stanza = backrest_settings.get("stanza", f"pg{pg}") - repo1_retention_full = backrest_settings.get("repo1-retention-full", "7") - log_level_console = backrest_settings.get("log-level-console", "info") - repo1_cipher_type = backrest_settings.get("repo1-cipher-type", "aes-256-cbc") - - os_user = new_node_data["os_user"] - repo1_type = new_node_data.get("repo1_type", "posix") - port = source_node_data["port"] - pg1_path = f"{source_node_data['path']}/pgedge/data/pg{pg}" - - if not repo1_path: - cmd = f"{source_node_data['path']}/pgedge/pgedge install backrest" - message = f"Installing backrest" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - repo1_path_default = f"/var/lib/pgbackrest/{source_node_data['name']}" - repo1_path = backrest_settings.get("repo1_path", f"{repo1_path_default}") - args = ( - f"--repo1-path {repo1_path} --stanza {stanza} " - f"--pg1-path {pg1_path} --repo1-type {repo1_type} " - f"--log-level-console {log_level_console} --pg1-port {port} " - f"--db-socket-path /tmp --repo1-cipher-type {repo1_cipher_type}" - ) - cmd = f"{source_node_data['path']}/pgedge/pgedge backrest command stanza-create '{args}'" - message = f"Creating stanza {stanza}" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - cmd = (f"{source_node_data['path']}/pgedge/pgedge backrest set_postgresqlconf {stanza} " - f"{pg1_path} {repo1_path} {repo1_type}") - message = f"Modifying postgresql.conf file" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - cmd = f"{source_node_data['path']}/pgedge/pgedge backrest set_hbaconf" - message = f"Modifying pg_hba.conf file" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - sql_cmd = "select pg_reload_conf()" - cmd = f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" - message = f"Reload configuration pg_reload_conf()" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - args = args + f" --repo1-retention-full={repo1_retention_full} --type=full" - cmd = f"{source_node_data['path']}/pgedge/pgedge backrest command backup '{args}'" - message = f"Creating full backup" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) + new_node_data["ip_address"] = new_node_data.get( + "public_ip", new_node_data.get("private_ip") + ) + + # Fetch backrest settings from the source node directory + backrest_settings = source_node_data.get("backrest", {}) + # New check: if pgBackRest is not configured on the source node + if not backrest_settings: + # Step 1: Install pgBackRest on the source node + cmd_install_backrest = f"cd {source_node_data['path']}/pgedge && ./pgedge install backrest" + run_cmd(cmd_install_backrest, node=source_node_data, message="Installing pgBackRest", verbose=verbose) + + util.message("## Integrating pgBackRest into the cluster", "info") + util.message(f"### Configuring BackRest for node '{source_node_data['name']}'", "info") + # Create a unique stanza name using the cluster name and node name + stanza_source = f"{cluster_name}_stanza_{source_node_data['name']}" + + # Load additional BackRest settings with defaults. + repo1_retention_full = "7" + log_level_console = "info" + repo1_cipher_type = "aes-256-cbc" + repo1_type = "posix" # Could also be "s3" + # Get repo1_path from JSON if provided; otherwise, use default + json_repo1_path = backrest_settings.get("repo1_path") + if json_repo1_path: + repo1_path_source = json_repo1_path.rstrip('/') + if not repo1_path_source.endswith(source_node_data["name"]): + repo1_path_source = repo1_path_source + f"/{source_node_data['name']}" else: - cmd = (f"{source_node_data['path']}/pgedge/pgedge backrest set_postgresqlconf {stanza} " - f"{pg1_path} {repo1_path} {repo1_type}") - message = f"Modifying postgresql.conf file" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - cmd = f"{source_node_data['path']}/pgedge/pgedge backrest set_hbaconf" - message = f"Modifying pg_hba.conf file" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - sql_cmd = "select pg_reload_conf()" - cmd = f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" - message = f"Reload configuration pg_reload_conf()" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - repo1_type = new_node_data.get("repo1_type", "posix") - if repo1_type == "s3": - for env_var in [ - "PGBACKREST_REPO1_S3_KEY", - "PGBACKREST_REPO1_S3_BUCKET", - "PGBACKREST_REPO1_S3_KEY_SECRET", - "PGBACKREST_REPO1_CIPHER_PASS", - ]: - if env_var not in os.environ: - util.exit_message(f"Environment variable {env_var} not set.") - s3_export_cmds = [ - f"export {env_var}={os.environ[env_var]}" - for env_var in [ - "PGBACKREST_REPO1_S3_KEY", - "PGBACKREST_REPO1_S3_BUCKET", - "PGBACKREST_REPO1_S3_KEY_SECRET", - "PGBACKREST_REPO1_CIPHER_PASS", - ] - ] - run_cmd(" && ".join(s3_export_cmds), - source_node_data, - message="Setting S3 environment variables on source node", - verbose=verbose) - run_cmd(" && ".join(s3_export_cmds), - new_node_data, - message="Setting S3 environment variables on target node", - verbose=verbose) - # End of backrest configuration block + repo1_path_source = f"/var/lib/pgbackrest/{source_node_data['name']}" + + # Similarly, set restore_path to include node name + restore_path_source = "/var/lib/pgbackrest_restore" + if not restore_path_source.rstrip('/').endswith(source_node_data["name"]): + restore_path_source = restore_path_source.rstrip('/') + f"/{source_node_data['name']}" + + pg_version = db_settings["pg_version"] + pg1_path_source = f"{source_node_data['path']}/pgedge/data/pg{pg_version}" + port_source = source_node_data["port"] + + # Step 2: Configure postgresql.conf for BackRest (without --pg1-port) + cmd_set_postgresqlconf_source = ( + f"cd {source_node_data['path']}/pgedge && " + f"./pgedge backrest set_postgresqlconf " + f"--stanza {stanza_source} " + f"--pg1-path {pg1_path_source} " + f"--repo1-path {repo1_path_source} " + f"--repo1-type {repo1_type}" + ) + run_cmd(cmd_set_postgresqlconf_source, node=source_node_data, message="Modifying postgresql.conf for BackRest", verbose=verbose) + + # Step 3: Configure pg_hba.conf for BackRest (without --pg1-port) + cmd_set_hbaconf_source = f"cd {source_node_data['path']}/pgedge && ./pgedge backrest set_hbaconf" + run_cmd(cmd_set_hbaconf_source, node=source_node_data, message="Modifying pg_hba.conf for BackRest", verbose=verbose) + + # Step 4: Reload PostgreSQL configuration to apply changes + sql_reload_conf = "select pg_reload_conf()" + cmd_reload_conf_source = f"cd {source_node_data['path']}/pgedge && ./pgedge psql '{sql_reload_conf}' {db[0]['db_name']}" + run_cmd(cmd_reload_conf_source, node=source_node_data, message="Reloading PostgreSQL configuration", verbose=verbose) + + # Step 5: If using S3 as repository, export necessary environment variables + if repo1_type.lower() == "s3": + required_env_vars = [ + "PGBACKREST_REPO1_S3_KEY", + "PGBACKREST_REPO1_S3_BUCKET", + "PGBACKREST_REPO1_S3_KEY_SECRET", + "PGBACKREST_REPO1_CIPHER_PASS", + ] + missing_env_vars = [var for var in required_env_vars if var not in os.environ] + if missing_env_vars: + util.exit_message( + f"Environment variables {', '.join(missing_env_vars)} must be set for S3 BackRest configuration.", + 1, + ) + s3_exports = " && ".join([f"export {var}={os.environ[var]}" for var in required_env_vars]) + cmd_export_s3_source = f"cd {source_node_data['path']}/pgedge && {s3_exports}" + run_cmd(cmd_export_s3_source, node=source_node_data, message="Setting S3 environment variables for BackRest", verbose=verbose) + + # Step 6: Set all BackRest backup configuration values for the source node. + # (a) Set the backup stanza + cmd_set_backup_stanza_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP stanza {stanza_source}" + run_cmd(cmd_set_backup_stanza_source, node=source_node_data, message=f"Setting BACKUP stanza '{stanza_source}' on node '{source_node_data['name']}'", verbose=verbose) + # (b) Create restore directory and set restore_path for backups. + cmd_create_restore_dir_source = f"sudo mkdir -p {restore_path_source}" + run_cmd(cmd_create_restore_dir_source, node=source_node_data, message=f"Creating restore directory {restore_path_source}", verbose=verbose) + cmd_set_restore_path_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP restore_path {restore_path_source}" + run_cmd(cmd_set_restore_path_source, node=source_node_data, message=f"Setting BACKUP restore_path to {restore_path_source}", verbose=verbose) + # (c) Set BACKUP repo1-host-user to the OS user (default: postgres) + os_user_source = source_node_data.get("os_user", "postgres") + cmd_set_repo1_host_user_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP repo1-host-user {os_user_source}" + run_cmd(cmd_set_repo1_host_user_source, node=source_node_data, message=f"Setting BACKUP repo1-host-user to {os_user_source} on node '{source_node_data['name']}'", verbose=verbose) + # (d) Set BACKUP pg1-path to the PostgreSQL data directory + cmd_set_pg1_path_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP pg1-path {pg1_path_source}" + run_cmd(cmd_set_pg1_path_source, node=source_node_data, message=f"Setting BACKUP pg1-path to {pg1_path_source} on node '{source_node_data['name']}'", verbose=verbose) + # (e) Set BACKUP pg1-user to the OS user + cmd_set_pg1_user_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP pg1-user {os_user_source}" + run_cmd(cmd_set_pg1_user_source, node=source_node_data, message=f"Setting BACKUP pg1-user to {os_user_source} on node '{source_node_data['name']}'", verbose=verbose) + # (f) Set BACKUP pg1-port to the node's port value + cmd_set_pg1_port_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP pg1-port {port_source}" + run_cmd(cmd_set_pg1_port_source, node=source_node_data, message=f"Setting BACKUP pg1-port to {port_source} on node '{source_node_data['name']}'", verbose=verbose) + # (g) Create the BackRest stanza (this command uses --pg1-port because it connects to the DB) + cmd_create_stanza_source = ( + f"cd {source_node_data['path']}/pgedge && " + f"./pgedge backrest command stanza-create " + f"--stanza '{stanza_source}' " + f"--pg1-path '{pg1_path_source}' " + f"--repo1-cipher-type {repo1_cipher_type} " + f"--pg1-port {port_source} " + f"--repo1-path {repo1_path_source}" + ) + run_cmd(cmd_create_stanza_source, node=source_node_data, message=f"Creating BackRest stanza '{stanza_source}'", verbose=verbose) + # (h) Initiate a full backup using pgBackRest (again, passing the port) + backrest_backup_args_source = ( + f"--repo1-path {repo1_path_source} " + f"--stanza {stanza_source} " + f"--pg1-path {pg1_path_source} " + f"--repo1-type {repo1_type} " + f"--log-level-console {log_level_console} " + f"--pg1-port {port_source} " + f"--db-socket-path /tmp " + f"--repo1-cipher-type {repo1_cipher_type} " + f"--repo1-retention-full {repo1_retention_full} " + f"--type=full" + ) + cmd_create_backup_source = f"cd {source_node_data['path']}/pgedge && ./pgedge backrest command backup '{backrest_backup_args_source}'" + run_cmd(cmd_create_backup_source, node=source_node_data, message="Creating full BackRest backup", verbose=verbose) + # (i) (Optional) Reset BACKUP repo1-path if needed + cmd_set_repo1_path_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP repo1-path {repo1_path_source}" + run_cmd(cmd_set_repo1_path_source, node=source_node_data, message=f"Setting BACKUP repo1-path to {repo1_path_source} on node '{source_node_data['name']}'", verbose=verbose) + + # Update backrest_settings for further use downstream. + backrest_settings = { + "stanza": stanza_source, + "repo1_path": repo1_path_source, + "repo1-retention-full": repo1_retention_full, + "log-level-console": log_level_console, + "repo1-cipher-type": repo1_cipher_type, + "repo1_type": repo1_type + } + + # NEW: Override backrest settings with target JSON settings if present + if target_backrest_settings: + backrest_settings = target_backrest_settings + + # For subsequent steps we extract settings from backrest_settings. + stanza = backrest_settings.get("stanza", f"pg{pg}") + repo1_retention_full = backrest_settings.get("repo1-retention-full", "7") + log_level_console = backrest_settings.get("log-level-console", "info") + repo1_cipher_type = backrest_settings.get("repo1-cipher-type", "aes-256-cbc") + repo1_type = backrest_settings.get("repo1_type", "posix") rc = ssh_install_pgedge( cluster_name, @@ -1754,53 +1823,30 @@ def add_node( ) os_user = new_node_data["os_user"] - repo1_type = new_node_data.get("repo1_type", "posix") port = source_node_data["port"] pg1_path = f"{source_node_data['path']}/pgedge/data/pg{pg}" if not repo1_path: - cmd = f"{source_node_data['path']}/pgedge/pgedge install backrest" - message = f"Installing backrest" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) + # Do not install backrest on source node; simply fetch the repo1_path from source's settings. repo1_path_default = f"/var/lib/pgbackrest/{source_node_data['name']}" repo1_path = backrest_settings.get("repo1_path", f"{repo1_path_default}") - args = ( - f"--repo1-path {repo1_path} --stanza {stanza} " - f"--pg1-path {pg1_path} --repo1-type {repo1_type} " - f"--log-level-console {log_level_console} --pg1-port {port} " - f"--db-socket-path /tmp --repo1-cipher-type {repo1_cipher_type}" - ) - cmd = f"{source_node_data['path']}/pgedge/pgedge backrest command stanza-create '{args}'" - message = f"Creating stanza {stanza}" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - cmd = (f"{source_node_data['path']}/pgedge/pgedge backrest set_postgresqlconf {stanza} " - f"{pg1_path} {repo1_path} {repo1_type}") - message = f"Modifying postgresql.conf file" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - cmd = f"{source_node_data['path']}/pgedge/pgedge backrest set_hbaconf" - message = f"Modifying pg_hba.conf file" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - sql_cmd = "select pg_reload_conf()" - cmd = f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" - message = f"Reload configuration pg_reload_conf()" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) - args = args + f" --repo1-retention-full={repo1_retention_full} --type=full" - cmd = f"{source_node_data['path']}/pgedge/pgedge backrest command backup '{args}'" - message = f"Creating full backup" - run_cmd(cmd, source_node_data, message=message, verbose=verbose) else: - cmd = (f"{source_node_data['path']}/pgedge/pgedge backrest set_postgresqlconf {stanza} " - f"{pg1_path} {repo1_path} {repo1_type}") + cmd = ( + f"{source_node_data['path']}/pgedge/pgedge backrest set_postgresqlconf {stanza} " + f"{pg1_path} {repo1_path} {repo1_type}" + ) message = f"Modifying postgresql.conf file" run_cmd(cmd, source_node_data, message=message, verbose=verbose) + cmd = f"{source_node_data['path']}/pgedge/pgedge backrest set_hbaconf" message = f"Modifying pg_hba.conf file" run_cmd(cmd, source_node_data, message=message, verbose=verbose) + sql_cmd = "select pg_reload_conf()" cmd = f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" message = f"Reload configuration pg_reload_conf()" run_cmd(cmd, source_node_data, message=message, verbose=verbose) - repo1_type = new_node_data.get("repo1_type", "posix") + if repo1_type == "s3": for env_var in [ "PGBACKREST_REPO1_S3_KEY", @@ -1819,14 +1865,18 @@ def add_node( "PGBACKREST_REPO1_CIPHER_PASS", ] ] - run_cmd(" && ".join(s3_export_cmds), - source_node_data, - message="Setting S3 environment variables on source node", - verbose=verbose) - run_cmd(" && ".join(s3_export_cmds), - new_node_data, - message="Setting S3 environment variables on target node", - verbose=verbose) + run_cmd( + " && ".join(s3_export_cmds), + source_node_data, + message="Setting S3 environment variables on source node", + verbose=verbose, + ) + run_cmd( + " && ".join(s3_export_cmds), + new_node_data, + message="Setting S3 environment variables on target node", + verbose=verbose, + ) cmd = f"{new_node_data['path']}/pgedge/pgedge install backrest" message = f"Installing backrest" @@ -1840,35 +1890,50 @@ def add_node( args = f"--repo1-path {repo1_path} --repo1-cipher-type {repo1_cipher_type} " if backup_id: args += f"--set={backup_id} " - cmd = (f'{new_node_data["path"]}/pgedge/pgedge backrest command restore ' - f"--repo1-type={repo1_type} --stanza={stanza} " - f'--pg1-path={new_node_data["path"]}/pgedge/data/pg{pg} {args}') + + cmd = ( + f'{new_node_data["path"]}/pgedge/pgedge backrest command restore ' + f"--repo1-type={repo1_type} --stanza={stanza} " + f'--pg1-path={new_node_data["path"]}/pgedge/data/pg{pg} {args}' + ) message = f"Restoring backup" run_cmd(cmd, new_node_data, message=message, verbose=verbose) pgd = f'{new_node_data["path"]}/pgedge/data/pg{pg}' pgc = f"{pgd}/postgresql.conf" + cmd = f"echo \"ssl_cert_file='{pgd}/server.crt'\" >> {pgc}" message = f"Setting ssl_cert_file" run_cmd(cmd, new_node_data, message=message, verbose=verbose) + cmd = f"echo \"ssl_key_file='{pgd}/server.key'\" >> {pgc}" message = f"Setting ssl_key_file" run_cmd(cmd, new_node_data, message=message, verbose=verbose) + cmd = f"echo \"log_directory='{pgd}/log'\" >> {pgc}" message = f"Setting log_directory" run_cmd(cmd, new_node_data, message=message, verbose=verbose) - cmd = (f'echo "shared_preload_libraries = ' - f"'pg_stat_statements, snowflake, spock'\" >> {pgc}") + + cmd = ( + f'echo "shared_preload_libraries = ' + f"'pg_stat_statements, snowflake, spock'\" >> {pgc}" + ) message = f"Setting shared_preload_libraries" run_cmd(cmd, new_node_data, message=message, verbose=verbose) - cmd = (f'{new_node_data["path"]}/pgedge/pgedge backrest configure_replica {stanza} ' - f'{new_node_data["path"]}/pgedge/data/pg{pg} {source_node_data["ip_address"]} ' - f'{source_node_data["port"]} {source_node_data["os_user"]}') + + cmd = ( + f'{new_node_data["path"]}/pgedge/pgedge backrest configure_replica {stanza} ' + f'{new_node_data["path"]}/pgedge/data/pg{pg} {source_node_data["ip_address"]} ' + f'{source_node_data["port"]} {source_node_data["os_user"]}' + ) message = f"Configuring PITR on replica" run_cmd(cmd, new_node_data, message=message, verbose=verbose) + if script.strip() and os.path.isfile(script): util.echo_cmd(f"{script}") + terminate_cluster_transactions(nodes, db[0]["db_name"], f"pg{pg}", verbose) + spock = db_settings["spock_version"] v4 = True spock_maj = 4 @@ -1878,23 +1943,33 @@ def add_node( spock_min = ver[1] if spock_maj >= 4: v4 = True + set_cluster_readonly(nodes, True, db[0]["db_name"], f"pg{pg}", v4, verbose) manage_node(new_node_data, "start", f"pg{pg}", verbose) time.sleep(5) + check_cluster_lag(new_node_data, db[0]["db_name"], f"pg{pg}", verbose) + sql_cmd = "SELECT pg_promote()" cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" message = f"Promoting standby to primary" run_cmd(cmd, new_node_data, message=message, verbose=verbose) + for mdb in db: sql_cmd = "SELECT sub_name FROM spock.subscription" cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {mdb['db_name']}" message = "Fetch existing subscriptions" - result = run_cmd(cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True) - subscriptions = [re.sub(r"\x1b\[[0-9;]*m", "", line.strip()) - for line in result.stdout.splitlines()[2:] - if line.strip() and not line.strip().startswith("(")] + result = run_cmd( + cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True + ) + + subscriptions = [ + re.sub(r"\x1b\[[0-9;]*m", "", line.strip()) + for line in result.stdout.splitlines()[2:] + if line.strip() and not line.strip().startswith("(") + ] subscriptions = [sub for sub in subscriptions if sub] + if subscriptions: for sub_name in subscriptions: cmd = f"{new_node_data['path']}/pgedge/pgedge spock sub-drop {sub_name} {mdb['db_name']}" @@ -1902,15 +1977,22 @@ def add_node( run_cmd(cmd, node=new_node_data, message=message, verbose=verbose) else: print("No subscriptions to drop.") + sql_cmd = "SELECT node_name FROM spock.node" cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {mdb['db_name']}" message = "Check if there are nodes" - result = run_cmd(cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True) + result = run_cmd( + cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True + ) + print(f"\nRaw output:\n{result.stdout}") - nodes_list = [re.sub(r"\x1b\[[0-9;]*m", "", line.strip()) - for line in result.stdout.splitlines()[2:] - if line.strip() and not line.strip().startswith("(")] + nodes_list = [ + re.sub(r"\x1b\[[0-9;]*m", "", line.strip()) + for line in result.stdout.splitlines()[2:] + if line.strip() and not line.strip().startswith("(") + ] nodes_list = [node for node in nodes_list if node] + if nodes_list: for node_name in nodes_list: cmd = f"{new_node_data['path']}/pgedge/pgedge spock node-drop {node_name} {mdb['db_name']}" @@ -1918,76 +2000,166 @@ def add_node( run_cmd(cmd, node=new_node_data, message=message, verbose=verbose) else: print("No nodes to drop.") + create_node(new_node_data, mdb["db_name"], verbose) + if not v4: set_cluster_readonly(nodes, False, mdb["db_name"], f"pg{pg}", v4, verbose) + create_sub(nodes, new_node_data, mdb["db_name"], verbose) create_sub_new(nodes, new_node_data, mdb["db_name"], verbose) + nc = os.path.join(new_node_data['path'], "pgedge", "pgedge ") cmd = f'{nc} spock repset-add-table default "*" {mdb["db_name"]}' message = f"Adding all tables to repset" run_cmd(cmd, new_node_data, message=message, verbose=verbose) + cmd = f'{nc} spock repset-add-table default_insert_only "*" {mdb["db_name"]}' run_cmd(cmd, new_node_data, message=message, verbose=verbose) + if v4: set_cluster_readonly(nodes, False, db[0]["db_name"], f"pg{pg}", v4, verbose) - cmd = f'cd {new_node_data["path"]}/pgedge/; ./pgedge spock node-list {db[0]["db_name"]}' - message = f"Listing spock nodes" - result = run_cmd(cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True) - print(f"\n{result.stdout}") - sql_cmd = "select node_id,node_name from spock.node" - cmd = f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" - message = f"List nodes" - result = run_cmd(cmd, node=source_node_data, message=message, verbose=verbose, capture_output=True) - print(f"\n{result.stdout}") - for node in nodes: - sql_cmd = ("select sub_id,sub_name,sub_enabled,sub_slot_name," - "sub_replication_sets from spock.subscription") - cmd = f"{node['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" - message = f"List subscriptions" - result = run_cmd(cmd, node=node, message=message, verbose=verbose, capture_output=True) - print(f"\n{result.stdout}") - sql_cmd = ("select sub_id,sub_name,sub_enabled,sub_slot_name," - "sub_replication_sets from spock.subscription") - cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" - message = f"List subscriptions" - result = run_cmd(cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True) - print(f"\n{result.stdout}") - cluster_data["node_groups"].append(new_node_data) - cluster_data["update_date"] = datetime.datetime.now().astimezone().isoformat() - write_cluster_json(cluster_name, cluster_data) - # Combine target node backup configuration commands into one command with verbose disabled. - pgedge_dir = f"{new_node_data['path']}/pgedge" - combined_target_cmd = ( - f"sudo mkdir -p /var/lib/pgbackrest_restore/{new_node_data['name']} && " - f"cd {pgedge_dir} && ./pgedge set BACKUP restore_path /var/lib/pgbackrest_restore/{new_node_data['name']} && " - f"sudo mkdir -p /var/lib/pgbackrest/{new_node_data['name']} && " - f"cd {pgedge_dir} && ./pgedge set BACKUP repo1-path /var/lib/pgbackrest/{new_node_data['name']} && " - f"cd {pgedge_dir} && ./pgedge set BACKUP repo1-host-user {new_node_data.get('os_user', 'postgres')} && " - f"cd {pgedge_dir} && ./pgedge set BACKUP pg1-path {pgedge_dir}/data/pg{pg} && " - f"cd {pgedge_dir} && ./pgedge set BACKUP pg1-user {new_node_data.get('os_user', 'postgres')} && " - f"cd {pgedge_dir} && ./pgedge set BACKUP pg1-port {new_node_data.get('port', '6435')}" - ) - run_cmd( - cmd=combined_target_cmd, - node=new_node_data, - message="Setting all target node BACKUP configuration", - verbose=False - ) + # DEBUG: Reload the complete target JSON file and fetch repo1_path and stanza from its backrest settings. + try: + with open(target_node_file, "r") as f: + complete_target_json = json.load(f) + + # Since backrest settings are stored inside each node group, fetch from the first node group. + if "node_groups" in complete_target_json and complete_target_json["node_groups"]: + repo1_path_target_file = complete_target_json["node_groups"][0].get("backrest", {}).get("repo1_path") + target_stanza_target_file = complete_target_json["node_groups"][0].get("backrest", {}).get("stanza") + + pgedge_dir = f"{new_node_data['path']}/pgedge" + restore_path = backrest_settings.get("restore_path", f"/var/lib/pgbackrest_restore/{new_node_data['name']}") + target_repo1_host_user = backrest_settings.get("repo1_host_user", new_node_data.get("os_user", "postgres")) + target_pg1_path = backrest_settings.get("pg1_path", f"{pgedge_dir}/data/pg{pg}") + target_pg1_user = backrest_settings.get("pg1_user", new_node_data.get("os_user", "postgres")) + target_pg1_port = backrest_settings.get("pg1_port", new_node_data.get("port", "6435")) + + # --------------------------------------------------------------- + # If the repo1_path or stanza is missing, remove backrest instead + # --------------------------------------------------------------- + if not repo1_path_target_file or not target_stanza_target_file: + # No valid repo1_path or stanza -> remove backrest + repo1_path_target_file = None + target_stanza_target_file = None + + cmd_remove_backrest_target = ( + f"cd {pgedge_dir} && ./pgedge remove backrest" + ) + run_cmd( + cmd=cmd_remove_backrest_target, + node=new_node_data, + message="Removing backrest", + verbose=verbose + ) + else: + # Both repo1_path_target_file and target_stanza_target_file exist + target_repo1_path = repo1_path_target_file + target_stanza = target_stanza_target_file + + # Combined target node BACKUP configuration commands + combined_target_cmd = ( + f"sudo mkdir -p {restore_path} && " + f"sudo chown -R {target_repo1_host_user}:{target_repo1_host_user} {restore_path} && " + f"cd {pgedge_dir} && ./pgedge set BACKUP restore_path {restore_path} && " + f"sudo mkdir -p {target_repo1_path} && " + f"sudo chown -R {target_repo1_host_user}:{target_repo1_host_user} {target_repo1_path} && " + f"cd {pgedge_dir} && ./pgedge set BACKUP repo1-path {target_repo1_path} && " + f"cd {pgedge_dir} && ./pgedge set BACKUP repo1-host-user {target_repo1_host_user} && " + f"cd {pgedge_dir} && ./pgedge set BACKUP pg1-path {target_pg1_path} && " + f"cd {pgedge_dir} && ./pgedge set BACKUP pg1-user {target_pg1_user} && " + f"cd {pgedge_dir} && ./pgedge set BACKUP pg1-port {target_pg1_port} && " + f"cd {pgedge_dir} && ./pgedge set BACKUP stanza {target_stanza}" + ) + run_cmd( + cmd=combined_target_cmd, + node=new_node_data, + message="Setting all target node BACKUP configuration", + verbose=False + ) + + # Append the BACKUP settings to the target's PostgreSQL configuration + cmd_set_postgresqlconf_target = ( + f"cd {pgedge_dir} && ./pgedge backrest set_postgresqlconf " + f"{target_stanza} {target_pg1_path} {target_repo1_path} {repo1_type}" + ) + run_cmd( + cmd=cmd_set_postgresqlconf_target, + node=new_node_data, + message="Appending BACKUP settings to postgresql.conf for target node", + verbose=verbose + ) + + # Restart PostgreSQL to apply the new configuration + cmd_restart_postgres = f"cd {pgedge_dir} && ./pgedge restart" + run_cmd( + cmd=cmd_restart_postgres, + node=new_node_data, + message="Restarting PostgreSQL service", + verbose=verbose + ) + + # Now create the BackRest stanza on the target node + cmd_create_stanza_target = ( + f"cd {pgedge_dir} && " + f"./pgedge backrest command stanza-create " + f"--stanza '{target_stanza}' " + f"--pg1-path '{target_pg1_path}' " + f"--repo1-cipher-type {repo1_cipher_type} " + f"--pg1-port {target_pg1_port} " + f"--repo1-path {target_repo1_path}" + ) + run_cmd( + cmd=cmd_create_stanza_target, + node=new_node_data, + message=f"Creating BackRest stanza '{target_stanza}'", + verbose=verbose + ) + + # Create a full backup using pgBackRest + backrest_backup_args_target = ( + f"--repo1-path {target_repo1_path} " + f"--stanza {target_stanza} " + f"--pg1-path {target_pg1_path} " + f"--repo1-type {repo1_type} " + f"--log-level-console {log_level_console} " + f"--pg1-port {target_pg1_port} " + f"--db-socket-path /tmp " + f"--repo1-cipher-type {repo1_cipher_type} " + f"--repo1-retention-full {repo1_retention_full} " + f"--type=full" + ) + cmd_create_backup_target = ( + f"cd {pgedge_dir} && ./pgedge backrest command backup " + f"'{backrest_backup_args_target}'" + ) + run_cmd( + cmd=cmd_create_backup_target, + node=new_node_data, + message="Creating full BackRest backup", + verbose=verbose + ) + else: + # If no node_groups exist at all, remove backrest + repo1_path_target_file = None + target_stanza_target_file = None + + pgedge_dir = f"{new_node_data['path']}/pgedge" + cmd_remove_backrest_target = ( + f"cd {pgedge_dir} && ./pgedge remove backrest" + ) + run_cmd( + cmd=cmd_remove_backrest_target, + node=new_node_data, + message="Removing backrest", + verbose=verbose + ) + + except Exception as e: + print(f"Error fetching values from target JSON file: {e}") - # --- Begin Backup Section for Target Node: Ensure repo1_path is writable. - node_os_user = new_node_data.get("os_user", "postgres") - combined_chown_cmd = f"sudo chown -R {node_os_user}:{node_os_user} {repo1_path} && sudo chmod -R 775 {repo1_path}" - run_cmd( - cmd=combined_chown_cmd, - node=new_node_data, - message=f"Setting ownership and permissions on {repo1_path}", - verbose=False - ) - capture_backrest_config(cluster_name) -# Cleanup backrest settings if target JSON did not include them - cleanup_backrest_from_cluster(cluster_data, target_node_data) def capture_backrest_config(cluster_name, verbose=False): """ Capture and clean BackRest configuration for all nodes (and sub-nodes) in the cluster @@ -2078,6 +2250,8 @@ def capture_backrest_config(cluster_name, verbose=False): f"Node '{node['name']}' does not have BackRest enabled; skipping.", "info" ) + + def cleanup_backrest_from_cluster(cluster_json, target_json): """ Compare the node groups in the target JSON with the cluster JSON. From 3d410404e320321a5aac970099377458763400a3 Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Tue, 8 Apr 2025 19:45:52 +0500 Subject: [PATCH 07/19] backrest configuration setup modified on add-node for source node --- cli/scripts/cluster.py | 44 +++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index d960d939..ad42329b 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -1739,28 +1739,28 @@ def add_node( run_cmd(cmd_export_s3_source, node=source_node_data, message="Setting S3 environment variables for BackRest", verbose=verbose) # Step 6: Set all BackRest backup configuration values for the source node. - # (a) Set the backup stanza - cmd_set_backup_stanza_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP stanza {stanza_source}" - run_cmd(cmd_set_backup_stanza_source, node=source_node_data, message=f"Setting BACKUP stanza '{stanza_source}' on node '{source_node_data['name']}'", verbose=verbose) - # (b) Create restore directory and set restore_path for backups. - cmd_create_restore_dir_source = f"sudo mkdir -p {restore_path_source}" - run_cmd(cmd_create_restore_dir_source, node=source_node_data, message=f"Creating restore directory {restore_path_source}", verbose=verbose) - cmd_set_restore_path_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP restore_path {restore_path_source}" - run_cmd(cmd_set_restore_path_source, node=source_node_data, message=f"Setting BACKUP restore_path to {restore_path_source}", verbose=verbose) - # (c) Set BACKUP repo1-host-user to the OS user (default: postgres) - os_user_source = source_node_data.get("os_user", "postgres") - cmd_set_repo1_host_user_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP repo1-host-user {os_user_source}" - run_cmd(cmd_set_repo1_host_user_source, node=source_node_data, message=f"Setting BACKUP repo1-host-user to {os_user_source} on node '{source_node_data['name']}'", verbose=verbose) - # (d) Set BACKUP pg1-path to the PostgreSQL data directory - cmd_set_pg1_path_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP pg1-path {pg1_path_source}" - run_cmd(cmd_set_pg1_path_source, node=source_node_data, message=f"Setting BACKUP pg1-path to {pg1_path_source} on node '{source_node_data['name']}'", verbose=verbose) - # (e) Set BACKUP pg1-user to the OS user - cmd_set_pg1_user_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP pg1-user {os_user_source}" - run_cmd(cmd_set_pg1_user_source, node=source_node_data, message=f"Setting BACKUP pg1-user to {os_user_source} on node '{source_node_data['name']}'", verbose=verbose) - # (f) Set BACKUP pg1-port to the node's port value - cmd_set_pg1_port_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP pg1-port {port_source}" - run_cmd(cmd_set_pg1_port_source, node=source_node_data, message=f"Setting BACKUP pg1-port to {port_source} on node '{source_node_data['name']}'", verbose=verbose) - # (g) Create the BackRest stanza (this command uses --pg1-port because it connects to the DB) + # + # Build one compound shell command + compound_cmd = " && ".join([ + f"cd {source_node_data['path']}/pgedge", + f"./pgedge set BACKUP stanza {stanza_source}", + f"sudo mkdir -p {restore_path_source}", + f"./pgedge set BACKUP restore_path {restore_path_source}", + f"./pgedge set BACKUP repo1-host-user {source_node_data.get('os_user', 'postgres')}", + f"./pgedge set BACKUP pg1-path {pg1_path_source}", + f"./pgedge set BACKUP pg1-user {source_node_data.get('os_user', 'postgres')}", + f"./pgedge set BACKUP pg1-port {port_source}" + ]) + + # Execute once with verbose disabled + run_cmd( + compound_cmd, + node=source_node_data, + message=f"Configuring BACKUP settings on node '{source_node_data['name']}'", + verbose=False + ) + + # # (g) Create the BackRest stanza (this command uses --pg1-port because it connects to the DB) cmd_create_stanza_source = ( f"cd {source_node_data['path']}/pgedge && " f"./pgedge backrest command stanza-create " From fb44ac0009a091bd9aa20e61d2eee87033f4334a Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Tue, 8 Apr 2025 22:26:41 +0500 Subject: [PATCH 08/19] json validate add node function now validate backrest of target node --- cli/scripts/cluster.py | 85 ++++++++++++++++++++++++++++++++---------- 1 file changed, 65 insertions(+), 20 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index ad42329b..b39ca243 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -2274,9 +2274,29 @@ def cleanup_backrest_from_cluster(cluster_json, target_json): if "backrest" in node: del node["backrest"] def json_validate_add_node(data): - """Validate the structure of a node configuration JSON file.""" - required_keys = ["json_version", "node_groups"] - node_group_keys = [ + """ + Validate the structure of a node‑definition JSON file that will be fed to + the add‑node command. + + • The traditional checks (json_version, ssh, port, …) still apply. + • A node_group is not required to have a “backrest” block. + • If a “backrest” block is present, it must contain at least: + • stanza – unique stanza name + • repo1_path – absolute path to the repo directory + • repo1_type – 'posix' or 's3' + and the values must be non‑empty and valid. + """ + + # ---------- top‑level keys ------------------------------------------------ + required_top = {"json_version", "node_groups"} + if not required_top.issubset(data): + util.exit_message("Invalid add‑node JSON: missing json_version or node_groups.") + + if str(data.get("json_version")) != "1.0": + util.exit_message("Invalid or unsupported json_version (must be '1.0').") + + # ---------- per‑node_group validation ------------------------------------ + node_group_required = { "ssh", "name", "is_active", @@ -2284,31 +2304,56 @@ def json_validate_add_node(data): "private_ip", "port", "path", - ] - ssh_keys = ["os_user", "private_key"] - if "json_version" not in data or data["json_version"] == "1.0": - util.exit_message("Invalid or missing JSON version.") + } + ssh_required = {"os_user", "private_key"} - for key in required_keys: - if key not in data: - util.exit_message(f"Key '{key}' missing from JSON data.") + backrest_required = {"stanza", "repo1_path", "repo1_type"} + valid_repo1_types = {"posix", "s3"} for group in data["node_groups"]: - for node_group_key in node_group_keys: - if node_group_key not in group: - util.exit_message(f"Key '{node_group_key}' missing from node group.") + gname = group.get("name", "?") - ssh_info = group.get("ssh", {}) - for ssh_key in ssh_keys: - if ssh_key not in ssh_info: - util.exit_message(f"Key '{ssh_key}' missing from ssh configuration.") + # --- basic mandatory keys + missing_basic = node_group_required - set(group.keys()) + if missing_basic: + util.exit_message( + f"Node‑group '{gname}' missing keys: {', '.join(missing_basic)}" + ) - if "public_ip" not in group and "private_ip" not in group: + # --- ssh block + ssh_info = group["ssh"] + missing_ssh = ssh_required - set(ssh_info.keys()) + if missing_ssh: util.exit_message( - "Both 'public_ip' and 'private_ip' are missing from node group." + f"SSH block in node‑group '{gname}' missing: {', '.join(missing_ssh)}" ) - util.message(f"New node json file structure is valid.", "success") + # --- backrest (optional but validated if present) + if "backrest" in group and group["backrest"] is not None: + br = group["backrest"] + + # ensure required keys are present + missing_br = backrest_required - set(br.keys()) + if missing_br: + util.exit_message( + f"BackRest block in node‑group '{gname}' missing: {', '.join(missing_br)}" + ) + + # ensure values are non‑empty + for k in backrest_required: + if not str(br[k]).strip(): + util.exit_message( + f"BackRest key '{k}' in node‑group '{gname}' cannot be empty." + ) + + # verify repo1_type is valid + if br["repo1_type"] not in valid_repo1_types: + util.exit_message( + f"Invalid repo1_type '{br['repo1_type']}' in node‑group '{gname}'. " + f"Allowed: {', '.join(valid_repo1_types)}" + ) + + util.message("✔ add‑node JSON structure is valid.", "success") def remove_node(cluster_name, node_name): """Remove a node from the cluster configuration. From 89f9c9cf6b98770ad71f3dc6aeea9d875a61d3f8 Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Wed, 9 Apr 2025 00:04:28 +0500 Subject: [PATCH 09/19] source node cleanup --- cli/scripts/cluster.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index b39ca243..7c06bbb6 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -1565,8 +1565,29 @@ def init(cluster_name, install=True): # Configure etcd and Patroni etcd.configure_etcd(node, sub_nodes) ha_patroni.configure_patroni(node, sub_nodes, db[0], db_settings) - - +def check_source_backrest_config(source_node_data): + """ + Check the source node's JSON data for a BackRest configuration. + If a non‑empty 'backrest' block is found, display its configuration. + Otherwise, display a message that no BackRest configuration exists, + and remove any leftover BackRest configuration. + """ + if "backrest" in source_node_data and source_node_data["backrest"]: + util.message( + f"Source node '{source_node_data['name']}' already has BackRest configuration: {source_node_data['backrest']}", + "info" + ) + else: + util.message( + f"Source node '{source_node_data['name']}' does not have a BackRest configuration.", + "info" + ) + # Print the source node's path for debugging purposes + util.message(f"Source node path: {source_node_data['path']}", "info") + # Remove any leftover BackRest configuration + cmd = f"cd {source_node_data['path']}/pgedge && ./pgedge remove backrest" + run_cmd(cmd, node=source_node_data, message="Removing BackRest configuration from source node", verbose=True) + def add_node( cluster_name, source_node, @@ -2156,10 +2177,11 @@ def add_node( message="Removing backrest", verbose=verbose ) - + except Exception as e: print(f"Error fetching values from target JSON file: {e}") - + # NEW: Check and display BackRest configuration status in the source node + check_source_backrest_config(source_node_data) def capture_backrest_config(cluster_name, verbose=False): """ Capture and clean BackRest configuration for all nodes (and sub-nodes) in the cluster From 298e5fc5f0307d531665e445a49640575ba95ff0 Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Wed, 9 Apr 2025 02:33:14 +0500 Subject: [PATCH 10/19] target node append on cluster json file --- cli/scripts/cluster.py | 71 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 8 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index 7c06bbb6..7b5191c6 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -1578,13 +1578,7 @@ def check_source_backrest_config(source_node_data): "info" ) else: - util.message( - f"Source node '{source_node_data['name']}' does not have a BackRest configuration.", - "info" - ) - # Print the source node's path for debugging purposes - util.message(f"Source node path: {source_node_data['path']}", "info") - # Remove any leftover BackRest configuration + cmd = f"cd {source_node_data['path']}/pgedge && ./pgedge remove backrest" run_cmd(cmd, node=source_node_data, message="Removing BackRest configuration from source node", verbose=True) @@ -1665,7 +1659,9 @@ def add_node( "os_user": os_user, "ssh_key": ssh_key, } - + # If backrest settings are provided in the JSON, add them to new_node_data. + if backrest_info: + new_node_data["backrest"] = backrest_info if "public_ip" not in new_node_data and "private_ip" not in new_node_data: util.exit_message( "Both public_ip and private_ip are missing in target node data." @@ -2041,6 +2037,52 @@ def add_node( if v4: set_cluster_readonly(nodes, False, db[0]["db_name"], f"pg{pg}", v4, verbose) + cmd = f'cd {new_node_data["path"]}/pgedge/; ./pgedge spock node-list {db[0]["db_name"]}' + message = f"Listing spock nodes" + result = run_cmd( + cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True + ) + print(f"\n{result.stdout}") + + sql_cmd = "select node_id,node_name from spock.node" + cmd = ( + f"{source_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" + ) + message = f"List nodes" + result = run_cmd( + cmd, + node=source_node_data, + message=message, + verbose=verbose, + capture_output=True, + ) + print(f"\n{result.stdout}") + + for node in nodes: + sql_cmd = ( + "select sub_id,sub_name,sub_enabled,sub_slot_name," + "sub_replication_sets from spock.subscription" + ) + cmd = f"{node['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" + message = f"List subscriptions" + result = run_cmd( + cmd, node=node, message=message, verbose=verbose, capture_output=True + ) + print(f"\n{result.stdout}") + + sql_cmd = ( + "select sub_id,sub_name,sub_enabled,sub_slot_name," + "sub_replication_sets from spock.subscription" + ) + cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {db[0]['db_name']}" + message = f"List subscriptions" + result = run_cmd( + cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True + ) + print(f"\n{result.stdout}") + + + # DEBUG: Reload the complete target JSON file and fetch repo1_path and stanza from its backrest settings. try: with open(target_node_file, "r") as f: @@ -2181,7 +2223,20 @@ def add_node( except Exception as e: print(f"Error fetching values from target JSON file: {e}") # NEW: Check and display BackRest configuration status in the source node + # Remove unnecessary keys before appending new node to the cluster data + new_node_data.pop("ip_address", None) + new_node_data.pop("os_user", None) + new_node_data.pop("ssh_key", None) + + # Append new node data to the cluster JSON + node_group = target_node_data.get + cluster_data["node_groups"].append(new_node_data) + cluster_data["update_date"] = datetime.datetime.now().astimezone().isoformat() + + write_cluster_json(cluster_name, cluster_data) + check_source_backrest_config(source_node_data) + def capture_backrest_config(cluster_name, verbose=False): """ Capture and clean BackRest configuration for all nodes (and sub-nodes) in the cluster From 4eecdd4c683cf254e8a77afeefe4e59a7b95007f Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Wed, 9 Apr 2025 02:58:41 +0500 Subject: [PATCH 11/19] yaml file for backrest created for each node in add node --- cli/scripts/cluster.py | 28 ++++------------------------ n2.json | 0 2 files changed, 4 insertions(+), 24 deletions(-) create mode 100644 n2.json diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index 7b5191c6..a0d349e6 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -1806,7 +1806,7 @@ def add_node( # (i) (Optional) Reset BACKUP repo1-path if needed cmd_set_repo1_path_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP repo1-path {repo1_path_source}" run_cmd(cmd_set_repo1_path_source, node=source_node_data, message=f"Setting BACKUP repo1-path to {repo1_path_source} on node '{source_node_data['name']}'", verbose=verbose) - + # Update backrest_settings for further use downstream. backrest_settings = { "stanza": stanza_source, @@ -2234,7 +2234,7 @@ def add_node( cluster_data["update_date"] = datetime.datetime.now().astimezone().isoformat() write_cluster_json(cluster_name, cluster_data) - + capture_backrest_config(cluster_name, verbose=True) check_source_backrest_config(source_node_data) def capture_backrest_config(cluster_name, verbose=False): @@ -2315,7 +2315,7 @@ def capture_backrest_config(cluster_name, verbose=False): with open(file_path, "w") as yaml_file: yaml.dump(config_dict, yaml_file, default_flow_style=False) util.message( - f"Cleaned BackRest configuration for node '{node['name']}' written to {file_path}", + f" BackRest configuration for node '{node['name']}' written to {file_path}", "info" ) except Exception as e: @@ -2329,27 +2329,7 @@ def capture_backrest_config(cluster_name, verbose=False): ) -def cleanup_backrest_from_cluster(cluster_json, target_json): - """ - Compare the node groups in the target JSON with the cluster JSON. - For each node in cluster_json["node_groups"], if the corresponding node (by name) - is not present in target_json's node groups or does not contain a "backrest" key, - then remove the "backrest" key from that node in cluster_json. - - Args: - cluster_json (dict): The main cluster configuration. - target_json (dict): The target node configuration JSON. - """ - # Create a mapping of node names to their configuration from the target JSON. - target_nodes = {group.get("name"): group for group in target_json.get("node_groups", [])} - - for node in cluster_json.get("node_groups", []): - node_name = node.get("name") - target_group = target_nodes.get(node_name) - # If the target group does not exist or does not contain a backrest key, delete backrest in the main config. - if not target_group or "backrest" not in target_group: - if "backrest" in node: - del node["backrest"] + def json_validate_add_node(data): """ Validate the structure of a node‑definition JSON file that will be fed to diff --git a/n2.json b/n2.json new file mode 100644 index 00000000..e69de29b From f620d620e969a35d9a6401504a3c198f7368c7ae Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Wed, 9 Apr 2025 05:34:25 +0500 Subject: [PATCH 12/19] backrest config updated by using ./pgedge backrest update-config --- cli/scripts/cluster.py | 3 +- src/backrest/backrest.py | 97 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index a0d349e6..40a308e5 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -1548,7 +1548,7 @@ def init(cluster_name, install=True): # (f) Set BACKUP pg1-port to the node's port value cmd_set_pg1_port = f"cd {node['path']}/pgedge && ./pgedge set BACKUP repo1-path {repo1_path}" run_cmd(cmd_set_pg1_port, node=node, message=f"Setting BACKUP repo1-path to {repo1_path} on node '{node['name']}'", verbose=verbose) - + capture_backrest_config(cluster_name, verbose=True) # 7. If it's an HA cluster, handle Patroni/etcd, etc. if is_ha_cluster: pg_ver = db_settings["pg_version"] @@ -1565,6 +1565,7 @@ def init(cluster_name, install=True): # Configure etcd and Patroni etcd.configure_etcd(node, sub_nodes) ha_patroni.configure_patroni(node, sub_nodes, db[0], db_settings) + def check_source_backrest_config(source_node_data): """ Check the source node's JSON data for a BackRest configuration. diff --git a/src/backrest/backrest.py b/src/backrest/backrest.py index 4deef3f7..5840a1d2 100755 --- a/src/backrest/backrest.py +++ b/src/backrest/backrest.py @@ -7,6 +7,11 @@ import sys from datetime import datetime from tabulate import tabulate +import yaml +import glob +import io +import contextlib + def pgV(): """Return the first found PostgreSQL version (v14 thru v17).""" @@ -432,6 +437,97 @@ def run_external_command(command, **kwargs): except Exception as e: util.exit_message(f"Failed:{str(e)}") +def update_config(verbose=False): + """ + Update the BACKUP configuration using a node-specific YAML file. + + ./pgedge backrest update-config + In non-verbose mode (default), all intermediate output is suppressed. + With verbose mode enabled, detailed logs and error messages are displayed. + + Parameters: + verbose (bool): Set to True to display detailed logs (default is False). + """ + + + # Internal function that runs the update and prints messages if local_verbose is True. + def run_update_config(local_verbose): + overall_success = True + + script_dir = os.path.dirname(os.path.realpath(__file__)) + pgedge_dir = os.path.dirname(script_dir) + node_name = os.path.basename(os.path.dirname(pgedge_dir)) + + # Construct the path to the YAML configuration file. + yaml_file = os.path.join(pgedge_dir, "backrest", f"backrest_{node_name}.yaml") + if not os.path.isfile(yaml_file): + if local_verbose: + print(f"Error: YAML file '{yaml_file}' does not exist.") + return False + + # Read the YAML file. + try: + with open(yaml_file, 'r') as f: + yaml_content = f.read() + except Exception as e: + if local_verbose: + print(f"Error reading {yaml_file}: {e}") + return False + + # Parse the YAML content into a dictionary. + try: + config_data = yaml.safe_load(yaml_content) + if not isinstance(config_data, dict): + if local_verbose: + print("Error: YAML file must contain a key-value mapping.") + return False + except Exception as e: + if local_verbose: + print(f"Error parsing YAML file: {e}") + return False + + if local_verbose: + print("==============================================") + print(" Executing Backrest Configuration Update") + print("==============================================") + + # Execute the update command for each configuration key-value pair. + for key, value in config_data.items(): + command = ["./pgedge", "set", "BACKUP", str(key), str(value)] + if local_verbose: + print(f"\n-> Executing command: {' '.join(command)}") + try: + result = subprocess.run(command, cwd=pgedge_dir, capture_output=True, text=True) + if result.returncode != 0: + overall_success = False + if local_verbose: + print(f" Error for key '{key}': {result.stderr.strip()}") + else: + if local_verbose: + out = result.stdout.strip() + if out: + print(f" Output: {out}") + else: + print(" Command executed successfully.") + except Exception as e: + overall_success = False + if local_verbose: + print(f" Exception for key '{key}': {str(e)}") + return overall_success + + # In non-verbose mode, redirect all output to devnull. + if not verbose: + dummy = io.StringIO() + with contextlib.redirect_stdout(dummy): + success = run_update_config(False) + # Do not print any final message in non-verbose mode. + else: + # Verbose mode: show detailed logs. + success = run_update_config(True) + if success: + print("\n==============================================") + print(" Backrest configuration updated successfully.") + print("==============================================") if __name__ == "__main__": fire.Fire({ "backup": backup, @@ -445,4 +541,5 @@ def run_external_command(command, **kwargs): "set_hbaconf": modify_hba_conf, "set_postgresqlconf": modify_postgresql_conf, "command": run_external_command, + "update-config": update_config, }) From 0a7559510e6e78e396ed219fc806d893a4eeb11d Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Thu, 10 Apr 2025 02:26:40 +0500 Subject: [PATCH 13/19] if repo1_path is provided in flag and source node is already configured with backrest then it will throw error --- cli/scripts/cluster.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index 40a308e5..98ff2321 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -1606,10 +1606,7 @@ def add_node( backup_id (str): Backup ID. stanza (str): Stanza name. script (str): Bash script. - """ - if (repo1_path and not backup_id) or (backup_id and not repo1_path): - util.exit_message("Both repo1_path and backup_id must be supplied together.") - json_validate(cluster_name) + """ db, db_settings, nodes = load_json(cluster_name) cluster_data = get_cluster_json(cluster_name) @@ -1635,13 +1632,23 @@ def add_node( # NEW: Check if target JSON has backrest configuration target_backrest_settings = target_node_data.get("backrest", {}) - # Retrieve source node data source_node_data = next( - (node for node in nodes if node["name"] == source_node), None - ) + (node for node in nodes if node["name"] == source_node), None +) if source_node_data is None: - util.exit_message(f"Source node '{source_node}' not found in cluster data.") + util.exit_message(f"Source node '{source_node}' not found in cluster data.") + +# Extract backrest settings from source node (before using repo1_path flag) + backrest_settings = source_node_data.get("backrest", {}) + source_repo1_path = backrest_settings.get("repo1_path") + +# Check: if source node JSON already provides repo1_path and the flag is given then exit + if repo1_path and source_repo1_path: + util.exit_message( + "Error: The source node JSON already contains a repo1_path. " + "Do not provide the repo1_path flag when the source node has it configured." + ) for group in target_node_data.get("node_groups", []): ssh_info = group.get("ssh") From b4453d93fdce59c43ae1a34e482934efe3e5bf81 Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Thu, 10 Apr 2025 03:43:55 +0500 Subject: [PATCH 14/19] repo1_path flag is working --- cli/scripts/cluster.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index 98ff2321..adb2fc2f 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -1707,13 +1707,18 @@ def add_node( log_level_console = "info" repo1_cipher_type = "aes-256-cbc" repo1_type = "posix" # Could also be "s3" - # Get repo1_path from JSON if provided; otherwise, use default - json_repo1_path = backrest_settings.get("repo1_path") - if json_repo1_path: +# Determine the repository path for the source node. + if repo1_path: + # Use the provided flag value as-is (trimmed of any trailing slash). + repo1_path_source = repo1_path.rstrip('/') + else: + # No repo1_path flag provided: check the JSON configuration or default. + json_repo1_path = backrest_settings.get("repo1_path") + if json_repo1_path: repo1_path_source = json_repo1_path.rstrip('/') if not repo1_path_source.endswith(source_node_data["name"]): repo1_path_source = repo1_path_source + f"/{source_node_data['name']}" - else: + else: repo1_path_source = f"/var/lib/pgbackrest/{source_node_data['name']}" # Similarly, set restore_path to include node name From 466fb1f55037883f5d684f36a357bc1c41bd95d8 Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Mon, 14 Apr 2025 05:58:55 +0500 Subject: [PATCH 15/19] Revert "backrest config updated by using ./pgedge backrest update-config" This reverts commit 3c70598cf514daf2b6edde5b6e06ce57a16cd0fe. --- cli/scripts/cluster.py | 3 +- src/backrest/backrest.py | 97 ---------------------------------------- 2 files changed, 1 insertion(+), 99 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index adb2fc2f..9683718a 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -1548,7 +1548,7 @@ def init(cluster_name, install=True): # (f) Set BACKUP pg1-port to the node's port value cmd_set_pg1_port = f"cd {node['path']}/pgedge && ./pgedge set BACKUP repo1-path {repo1_path}" run_cmd(cmd_set_pg1_port, node=node, message=f"Setting BACKUP repo1-path to {repo1_path} on node '{node['name']}'", verbose=verbose) - capture_backrest_config(cluster_name, verbose=True) + # 7. If it's an HA cluster, handle Patroni/etcd, etc. if is_ha_cluster: pg_ver = db_settings["pg_version"] @@ -1565,7 +1565,6 @@ def init(cluster_name, install=True): # Configure etcd and Patroni etcd.configure_etcd(node, sub_nodes) ha_patroni.configure_patroni(node, sub_nodes, db[0], db_settings) - def check_source_backrest_config(source_node_data): """ Check the source node's JSON data for a BackRest configuration. diff --git a/src/backrest/backrest.py b/src/backrest/backrest.py index 5840a1d2..4deef3f7 100755 --- a/src/backrest/backrest.py +++ b/src/backrest/backrest.py @@ -7,11 +7,6 @@ import sys from datetime import datetime from tabulate import tabulate -import yaml -import glob -import io -import contextlib - def pgV(): """Return the first found PostgreSQL version (v14 thru v17).""" @@ -437,97 +432,6 @@ def run_external_command(command, **kwargs): except Exception as e: util.exit_message(f"Failed:{str(e)}") -def update_config(verbose=False): - """ - Update the BACKUP configuration using a node-specific YAML file. - - ./pgedge backrest update-config - In non-verbose mode (default), all intermediate output is suppressed. - With verbose mode enabled, detailed logs and error messages are displayed. - - Parameters: - verbose (bool): Set to True to display detailed logs (default is False). - """ - - - # Internal function that runs the update and prints messages if local_verbose is True. - def run_update_config(local_verbose): - overall_success = True - - script_dir = os.path.dirname(os.path.realpath(__file__)) - pgedge_dir = os.path.dirname(script_dir) - node_name = os.path.basename(os.path.dirname(pgedge_dir)) - - # Construct the path to the YAML configuration file. - yaml_file = os.path.join(pgedge_dir, "backrest", f"backrest_{node_name}.yaml") - if not os.path.isfile(yaml_file): - if local_verbose: - print(f"Error: YAML file '{yaml_file}' does not exist.") - return False - - # Read the YAML file. - try: - with open(yaml_file, 'r') as f: - yaml_content = f.read() - except Exception as e: - if local_verbose: - print(f"Error reading {yaml_file}: {e}") - return False - - # Parse the YAML content into a dictionary. - try: - config_data = yaml.safe_load(yaml_content) - if not isinstance(config_data, dict): - if local_verbose: - print("Error: YAML file must contain a key-value mapping.") - return False - except Exception as e: - if local_verbose: - print(f"Error parsing YAML file: {e}") - return False - - if local_verbose: - print("==============================================") - print(" Executing Backrest Configuration Update") - print("==============================================") - - # Execute the update command for each configuration key-value pair. - for key, value in config_data.items(): - command = ["./pgedge", "set", "BACKUP", str(key), str(value)] - if local_verbose: - print(f"\n-> Executing command: {' '.join(command)}") - try: - result = subprocess.run(command, cwd=pgedge_dir, capture_output=True, text=True) - if result.returncode != 0: - overall_success = False - if local_verbose: - print(f" Error for key '{key}': {result.stderr.strip()}") - else: - if local_verbose: - out = result.stdout.strip() - if out: - print(f" Output: {out}") - else: - print(" Command executed successfully.") - except Exception as e: - overall_success = False - if local_verbose: - print(f" Exception for key '{key}': {str(e)}") - return overall_success - - # In non-verbose mode, redirect all output to devnull. - if not verbose: - dummy = io.StringIO() - with contextlib.redirect_stdout(dummy): - success = run_update_config(False) - # Do not print any final message in non-verbose mode. - else: - # Verbose mode: show detailed logs. - success = run_update_config(True) - if success: - print("\n==============================================") - print(" Backrest configuration updated successfully.") - print("==============================================") if __name__ == "__main__": fire.Fire({ "backup": backup, @@ -541,5 +445,4 @@ def run_update_config(local_verbose): "set_hbaconf": modify_hba_conf, "set_postgresqlconf": modify_postgresql_conf, "command": run_external_command, - "update-config": update_config, }) From c890a0c5c61728c047717bd515fdefd3d6b63eb0 Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Mon, 14 Apr 2025 05:58:59 +0500 Subject: [PATCH 16/19] Revert "yaml file for backrest created for each node in add node" This reverts commit 279acfea05e3bd5ae9cb247b2bdcb492a036aaaf. --- cli/scripts/cluster.py | 28 ++++++++++++++++++++++++---- n2.json | 0 2 files changed, 24 insertions(+), 4 deletions(-) delete mode 100644 n2.json diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index 9683718a..83a49038 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -1818,7 +1818,7 @@ def add_node( # (i) (Optional) Reset BACKUP repo1-path if needed cmd_set_repo1_path_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP repo1-path {repo1_path_source}" run_cmd(cmd_set_repo1_path_source, node=source_node_data, message=f"Setting BACKUP repo1-path to {repo1_path_source} on node '{source_node_data['name']}'", verbose=verbose) - + # Update backrest_settings for further use downstream. backrest_settings = { "stanza": stanza_source, @@ -2246,7 +2246,7 @@ def add_node( cluster_data["update_date"] = datetime.datetime.now().astimezone().isoformat() write_cluster_json(cluster_name, cluster_data) - capture_backrest_config(cluster_name, verbose=True) + check_source_backrest_config(source_node_data) def capture_backrest_config(cluster_name, verbose=False): @@ -2327,7 +2327,7 @@ def capture_backrest_config(cluster_name, verbose=False): with open(file_path, "w") as yaml_file: yaml.dump(config_dict, yaml_file, default_flow_style=False) util.message( - f" BackRest configuration for node '{node['name']}' written to {file_path}", + f"Cleaned BackRest configuration for node '{node['name']}' written to {file_path}", "info" ) except Exception as e: @@ -2341,7 +2341,27 @@ def capture_backrest_config(cluster_name, verbose=False): ) - +def cleanup_backrest_from_cluster(cluster_json, target_json): + """ + Compare the node groups in the target JSON with the cluster JSON. + For each node in cluster_json["node_groups"], if the corresponding node (by name) + is not present in target_json's node groups or does not contain a "backrest" key, + then remove the "backrest" key from that node in cluster_json. + + Args: + cluster_json (dict): The main cluster configuration. + target_json (dict): The target node configuration JSON. + """ + # Create a mapping of node names to their configuration from the target JSON. + target_nodes = {group.get("name"): group for group in target_json.get("node_groups", [])} + + for node in cluster_json.get("node_groups", []): + node_name = node.get("name") + target_group = target_nodes.get(node_name) + # If the target group does not exist or does not contain a backrest key, delete backrest in the main config. + if not target_group or "backrest" not in target_group: + if "backrest" in node: + del node["backrest"] def json_validate_add_node(data): """ Validate the structure of a node‑definition JSON file that will be fed to diff --git a/n2.json b/n2.json deleted file mode 100644 index e69de29b..00000000 From 9bbf3ccc968943324ff3baab9aae9cf32834ac31 Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Tue, 15 Apr 2025 03:49:54 +0500 Subject: [PATCH 17/19] CleanUp Cleanup of code --- cli/scripts/cluster.py | 170 ++++++++--------------------------------- 1 file changed, 30 insertions(+), 140 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index 83a49038..d6dbfdc6 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -11,7 +11,7 @@ import re from tabulate import tabulate # type: ignore from ipaddress import ip_address -import yaml + try: import etcd import ha_patroni @@ -1528,7 +1528,7 @@ def init(cluster_name, install=True): f"--pg1-port {port} " f"--repo1-path {repo1_path}" ) - run_cmd(cmd_create_stanza, node=node, message=f"Creating BackRest stanza '{stanza}'", verbose=verbose) + run_cmd(cmd_create_stanza, node=node, message=f"Creating pgBackRest stanza '{stanza}'", verbose=verbose) # -- Step 8: Initiate a full backup using pgBackRest (again, passing the port) backrest_backup_args = ( @@ -1565,23 +1565,7 @@ def init(cluster_name, install=True): # Configure etcd and Patroni etcd.configure_etcd(node, sub_nodes) ha_patroni.configure_patroni(node, sub_nodes, db[0], db_settings) -def check_source_backrest_config(source_node_data): - """ - Check the source node's JSON data for a BackRest configuration. - If a non‑empty 'backrest' block is found, display its configuration. - Otherwise, display a message that no BackRest configuration exists, - and remove any leftover BackRest configuration. - """ - if "backrest" in source_node_data and source_node_data["backrest"]: - util.message( - f"Source node '{source_node_data['name']}' already has BackRest configuration: {source_node_data['backrest']}", - "info" - ) - else: - - cmd = f"cd {source_node_data['path']}/pgedge && ./pgedge remove backrest" - run_cmd(cmd, node=source_node_data, message="Removing BackRest configuration from source node", verbose=True) - + def add_node( cluster_name, source_node, @@ -1603,7 +1587,6 @@ def add_node( target_node (str): The new node. repo1_path (str): The repo1 path to use. backup_id (str): Backup ID. - stanza (str): Stanza name. script (str): Bash script. """ db, db_settings, nodes = load_json(cluster_name) @@ -1697,7 +1680,7 @@ def add_node( run_cmd(cmd_install_backrest, node=source_node_data, message="Installing pgBackRest", verbose=verbose) util.message("## Integrating pgBackRest into the cluster", "info") - util.message(f"### Configuring BackRest for node '{source_node_data['name']}'", "info") + util.message(f"### Configuring pgBackRest for node '{source_node_data['name']}'", "info") # Create a unique stanza name using the cluster name and node name stanza_source = f"{cluster_name}_stanza_{source_node_data['name']}" @@ -1729,7 +1712,7 @@ def add_node( pg1_path_source = f"{source_node_data['path']}/pgedge/data/pg{pg_version}" port_source = source_node_data["port"] - # Step 2: Configure postgresql.conf for BackRest (without --pg1-port) + # Step 2: Configure postgresql.conf for pgBackRest (without --pg1-port) cmd_set_postgresqlconf_source = ( f"cd {source_node_data['path']}/pgedge && " f"./pgedge backrest set_postgresqlconf " @@ -1738,11 +1721,11 @@ def add_node( f"--repo1-path {repo1_path_source} " f"--repo1-type {repo1_type}" ) - run_cmd(cmd_set_postgresqlconf_source, node=source_node_data, message="Modifying postgresql.conf for BackRest", verbose=verbose) + run_cmd(cmd_set_postgresqlconf_source, node=source_node_data, message="Modifying postgresql.conf for pgBackRest", verbose=verbose) - # Step 3: Configure pg_hba.conf for BackRest (without --pg1-port) + # Step 3: Configure pg_hba.conf for pgBackRest (without --pg1-port) cmd_set_hbaconf_source = f"cd {source_node_data['path']}/pgedge && ./pgedge backrest set_hbaconf" - run_cmd(cmd_set_hbaconf_source, node=source_node_data, message="Modifying pg_hba.conf for BackRest", verbose=verbose) + run_cmd(cmd_set_hbaconf_source, node=source_node_data, message="Modifying pg_hba.conf for pgBackRest", verbose=verbose) # Step 4: Reload PostgreSQL configuration to apply changes sql_reload_conf = "select pg_reload_conf()" @@ -1760,14 +1743,14 @@ def add_node( missing_env_vars = [var for var in required_env_vars if var not in os.environ] if missing_env_vars: util.exit_message( - f"Environment variables {', '.join(missing_env_vars)} must be set for S3 BackRest configuration.", + f"Environment variables {', '.join(missing_env_vars)} must be set for S3 pgBackRest configuration.", 1, ) s3_exports = " && ".join([f"export {var}={os.environ[var]}" for var in required_env_vars]) cmd_export_s3_source = f"cd {source_node_data['path']}/pgedge && {s3_exports}" - run_cmd(cmd_export_s3_source, node=source_node_data, message="Setting S3 environment variables for BackRest", verbose=verbose) + run_cmd(cmd_export_s3_source, node=source_node_data, message="Setting S3 environment variables for pgBackRest", verbose=verbose) - # Step 6: Set all BackRest backup configuration values for the source node. + # Step 6: Set all pgBackRest backup configuration values for the source node. # # Build one compound shell command compound_cmd = " && ".join([ @@ -1789,7 +1772,7 @@ def add_node( verbose=False ) - # # (g) Create the BackRest stanza (this command uses --pg1-port because it connects to the DB) + # Step 7:Create the pgBackRest stanza (this command uses --pg1-port because it connects to the DB) cmd_create_stanza_source = ( f"cd {source_node_data['path']}/pgedge && " f"./pgedge backrest command stanza-create " @@ -1799,8 +1782,8 @@ def add_node( f"--pg1-port {port_source} " f"--repo1-path {repo1_path_source}" ) - run_cmd(cmd_create_stanza_source, node=source_node_data, message=f"Creating BackRest stanza '{stanza_source}'", verbose=verbose) - # (h) Initiate a full backup using pgBackRest (again, passing the port) + run_cmd(cmd_create_stanza_source, node=source_node_data, message=f"Creating pgBackRest stanza '{stanza_source}'", verbose=verbose) + # Step 8: Initiate a full backup using pgBackRest (again, passing the port) backrest_backup_args_source = ( f"--repo1-path {repo1_path_source} " f"--stanza {stanza_source} " @@ -1814,12 +1797,12 @@ def add_node( f"--type=full" ) cmd_create_backup_source = f"cd {source_node_data['path']}/pgedge && ./pgedge backrest command backup '{backrest_backup_args_source}'" - run_cmd(cmd_create_backup_source, node=source_node_data, message="Creating full BackRest backup", verbose=verbose) + run_cmd(cmd_create_backup_source, node=source_node_data, message="Creating full pgBackRest backup", verbose=verbose) # (i) (Optional) Reset BACKUP repo1-path if needed cmd_set_repo1_path_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP repo1-path {repo1_path_source}" run_cmd(cmd_set_repo1_path_source, node=source_node_data, message=f"Setting BACKUP repo1-path to {repo1_path_source} on node '{source_node_data['name']}'", verbose=verbose) - # Update backrest_settings for further use downstream. + # Update pgbackrest_settings for further use downstream. backrest_settings = { "stanza": stanza_source, "repo1_path": repo1_path_source, @@ -1829,11 +1812,11 @@ def add_node( "repo1_type": repo1_type } - # NEW: Override backrest settings with target JSON settings if present + # NEW: Override pgbackrest settings with target JSON settings if present if target_backrest_settings: backrest_settings = target_backrest_settings - # For subsequent steps we extract settings from backrest_settings. + # For subsequent steps we extract settings from pgbackrest_settings. stanza = backrest_settings.get("stanza", f"pg{pg}") repo1_retention_full = backrest_settings.get("repo1-retention-full", "7") log_level_console = backrest_settings.get("log-level-console", "info") @@ -1856,7 +1839,7 @@ def add_node( pg1_path = f"{source_node_data['path']}/pgedge/data/pg{pg}" if not repo1_path: - # Do not install backrest on source node; simply fetch the repo1_path from source's settings. + # Do not install pgbackrest on source node; simply fetch the repo1_path from source's settings. repo1_path_default = f"/var/lib/pgbackrest/{source_node_data['name']}" repo1_path = backrest_settings.get("repo1_path", f"{repo1_path_default}") else: @@ -1908,7 +1891,7 @@ def add_node( ) cmd = f"{new_node_data['path']}/pgedge/pgedge install backrest" - message = f"Installing backrest" + message = f"Installing pgbackrest" run_cmd(cmd, new_node_data, message=message, verbose=verbose) manage_node(new_node_data, "stop", f"pg{pg}", verbose) @@ -2095,12 +2078,12 @@ def add_node( - # DEBUG: Reload the complete target JSON file and fetch repo1_path and stanza from its backrest settings. + # Reload the complete target JSON file and fetch repo1_path and stanza from its pgbackrest settings. try: with open(target_node_file, "r") as f: complete_target_json = json.load(f) - # Since backrest settings are stored inside each node group, fetch from the first node group. + # Since pgbackrest settings are stored inside each node group, fetch from the first node group. if "node_groups" in complete_target_json and complete_target_json["node_groups"]: repo1_path_target_file = complete_target_json["node_groups"][0].get("backrest", {}).get("repo1_path") target_stanza_target_file = complete_target_json["node_groups"][0].get("backrest", {}).get("stanza") @@ -2176,7 +2159,7 @@ def add_node( verbose=verbose ) - # Now create the BackRest stanza on the target node + # Now create the pgBackRest stanza on the target node cmd_create_stanza_target = ( f"cd {pgedge_dir} && " f"./pgedge backrest command stanza-create " @@ -2189,7 +2172,7 @@ def add_node( run_cmd( cmd=cmd_create_stanza_target, node=new_node_data, - message=f"Creating BackRest stanza '{target_stanza}'", + message=f"Creating pgBackRest stanza '{target_stanza}'", verbose=verbose ) @@ -2217,7 +2200,7 @@ def add_node( verbose=verbose ) else: - # If no node_groups exist at all, remove backrest + # If no node_groups exist at all, remove pgbackrest repo1_path_target_file = None target_stanza_target_file = None @@ -2234,7 +2217,7 @@ def add_node( except Exception as e: print(f"Error fetching values from target JSON file: {e}") - # NEW: Check and display BackRest configuration status in the source node + # NEW: Check and display pgBackRest configuration status in the source node # Remove unnecessary keys before appending new node to the cluster data new_node_data.pop("ip_address", None) new_node_data.pop("os_user", None) @@ -2248,99 +2231,7 @@ def add_node( write_cluster_json(cluster_name, cluster_data) check_source_backrest_config(source_node_data) - -def capture_backrest_config(cluster_name, verbose=False): - """ - Capture and clean BackRest configuration for all nodes (and sub-nodes) in the cluster - that have BackRest enabled. - - For each node with a non-empty 'backrest' configuration in the cluster JSON, - this function will: - 1. Change directory to the node's pgedge directory. - 2. Run "./pgedge backrest show-config" and capture the output. - 3. Clean the output by removing extraneous lines, ANSI escape codes, and literal "^[[0m" strings, - then parse it into a dictionary. - 4. Write the cleaned configuration as YAML to a file named - "backrest_{node_name}.yaml" in the node's "pgedge/backrest" directory. - """ - # Load the cluster configuration - db, db_settings, nodes = load_json(cluster_name) - - # Create a combined list of all nodes (including sub-nodes) - all_nodes = [] - for node in nodes: - all_nodes.append(node) - if "sub_nodes" in node and isinstance(node["sub_nodes"], list): - all_nodes.extend(node["sub_nodes"]) - - # Regex to remove ANSI escape sequences - ansi_escape = re.compile(r'\x1B[@-_][0-?]*[ -/]*[@-~]') - - for node in all_nodes: - if node.get("backrest"): - cmd = f"cd {node['path']}/pgedge && ./pgedge backrest show-config" - result = run_cmd( - cmd, - node, - message=f"Capturing BackRest configuration for node {node['name']}", - verbose=verbose, - capture_output=True - ) - # Debug: Check raw command output - util.message(f"[DEBUG] Raw output for node {node['name']}:\n{result.stdout}", "debug") - - # Remove ANSI escape sequences and literal "^[[0m" - output = ansi_escape.sub('', result.stdout) - output = output.replace("^[[0m", "") - - # Debug: Print cleaned output - util.message(f"[DEBUG] Cleaned output for node {node['name']}:\n{output}", "debug") - - # Parse the cleaned output into a dictionary - config_dict = {} - for line in output.splitlines(): - line = line.strip() - # Skip empty lines, header/footer lines, or lines without '=' - if not line or line.startswith('#') or ('=' not in line): - continue - parts = line.split('=', 1) - if len(parts) != 2: - continue - key = parts[0].strip() - value = parts[1].strip() - if value.isdigit(): - value = int(value) - config_dict[key] = value - - # Debug: Check the resulting dictionary - util.message(f"[DEBUG] Parsed config for node {node['name']}: {config_dict}", "debug") - - if not config_dict: - util.message(f"[WARNING] No configuration parsed for node {node['name']}.", "warning") - - # Ensure the "pgedge/backrest" directory exists on the node - backrest_dir = os.path.join(node["path"], "pgedge", "backrest") - os.makedirs(backrest_dir, exist_ok=True) - file_path = os.path.join(backrest_dir, f"backrest_{node['name']}.yaml") - - try: - with open(file_path, "w") as yaml_file: - yaml.dump(config_dict, yaml_file, default_flow_style=False) - util.message( - f"Cleaned BackRest configuration for node '{node['name']}' written to {file_path}", - "info" - ) - except Exception as e: - util.exit_message( - f"Failed to write cleaned BackRest configuration for node '{node['name']}': {e}" - ) - else: - util.message( - f"Node '{node['name']}' does not have BackRest enabled; skipping.", - "info" - ) - - + def cleanup_backrest_from_cluster(cluster_json, target_json): """ Compare the node groups in the target JSON with the cluster JSON. @@ -2362,6 +2253,7 @@ def cleanup_backrest_from_cluster(cluster_json, target_json): if not target_group or "backrest" not in target_group: if "backrest" in node: del node["backrest"] + def json_validate_add_node(data): """ Validate the structure of a node‑definition JSON file that will be fed to @@ -2376,7 +2268,6 @@ def json_validate_add_node(data): and the values must be non‑empty and valid. """ - # ---------- top‑level keys ------------------------------------------------ required_top = {"json_version", "node_groups"} if not required_top.issubset(data): util.exit_message("Invalid add‑node JSON: missing json_version or node_groups.") @@ -2384,7 +2275,6 @@ def json_validate_add_node(data): if str(data.get("json_version")) != "1.0": util.exit_message("Invalid or unsupported json_version (must be '1.0').") - # ---------- per‑node_group validation ------------------------------------ node_group_required = { "ssh", "name", @@ -2409,7 +2299,7 @@ def json_validate_add_node(data): f"Node‑group '{gname}' missing keys: {', '.join(missing_basic)}" ) - # --- ssh block + # ssh block ssh_info = group["ssh"] missing_ssh = ssh_required - set(ssh_info.keys()) if missing_ssh: @@ -2417,7 +2307,7 @@ def json_validate_add_node(data): f"SSH block in node‑group '{gname}' missing: {', '.join(missing_ssh)}" ) - # --- backrest (optional but validated if present) + # backrest (optional but validated if present) if "backrest" in group and group["backrest"] is not None: br = group["backrest"] From 8a0a8be2b60196411d9b278d5fe3695529163787 Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Tue, 15 Apr 2025 04:01:33 +0500 Subject: [PATCH 18/19] Small issues resolved after cleanup --- cli/scripts/cluster.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index d6dbfdc6..37114f95 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -1565,7 +1565,25 @@ def init(cluster_name, install=True): # Configure etcd and Patroni etcd.configure_etcd(node, sub_nodes) ha_patroni.configure_patroni(node, sub_nodes, db[0], db_settings) - + + +def check_source_backrest_config(source_node_data): + """ + Check the source node's JSON data for a BackRest configuration. + If a non‑empty 'backrest' block is found, display its configuration. + Otherwise, display a message that no BackRest configuration exists, + and remove any leftover BackRest configuration. + """ + if "backrest" in source_node_data and source_node_data["backrest"]: + util.message( + f"Source node '{source_node_data['name']}' already has BackRest configuration: {source_node_data['backrest']}", + "info" + ) + else: + + cmd = f"cd {source_node_data['path']}/pgedge && ./pgedge remove backrest" + run_cmd(cmd, node=source_node_data, message="Removing BackRest configuration from source node", verbose=True) + def add_node( cluster_name, source_node, From 409f5e6166b7f6c4281ba4c4f1961d2ef537acaa Mon Sep 17 00:00:00 2001 From: Matthew Mols Date: Tue, 15 Apr 2025 14:38:12 -0500 Subject: [PATCH 19/19] formatting --- cli/scripts/cluster.py | 304 ++++++++++++++++++++++++++--------------- 1 file changed, 192 insertions(+), 112 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index 37114f95..aee0ece0 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -633,8 +633,6 @@ def ssh_cross_wire_pgedge( print(result.stdout) - - def remove(cluster_name, force=False): """Remove a cluster. @@ -894,7 +892,7 @@ def get_cluster_info(cluster_name): # Store 'is_ha_cluster' in the cluster JSON cluster_json["is_ha_cluster"] = is_ha_cluster - # Ask if BackRest should be enabled + # Ask if pgBackRest should be enabled if force: backrest_enabled = False else: @@ -917,7 +915,7 @@ def get_cluster_info(cluster_name): ) if backrest_archive_mode not in ["on", "off"]: util.exit_message( - "Invalid BackRest archive mode. Allowed values are 'on' or 'off'." + "Invalid pgBackRest archive mode. Allowed values are 'on' or 'off'." ) # Optionally, ask for repo1_type or default to posix repo1_type = ( @@ -930,7 +928,7 @@ def get_cluster_info(cluster_name): util.exit_message( "Invalid pgBackRest repository type. Allowed values are 'posix' or 's3'." ) - # Create base BackRest configuration + # Create base pgBackRest configuration backrest_json = { "stanza": f"{cluster_name}_stanza_", # base stanza; node name will be appended later "repo1_path": backrest_storage_path, @@ -1327,7 +1325,6 @@ def update_json(cluster_name, db_json): util.exit_message("Unable to update JSON file", 1) - def init(cluster_name, install=True): """ Initialize a cluster via cluster configuration JSON file. @@ -1340,7 +1337,7 @@ def init(cluster_name, install=True): 5. Integrates pgBackRest on nodes where it is enabled. - Creates unique stanza names: {cluster_name}_stanza_{node_name} - Removes --pg1-port from set_postgresqlconf / set_hbaconf to avoid errors. - - Ensures that every BackRest configuration value is set before creating the stanza. + - Ensures that every pgBackRest configuration value is set before creating the stanza. - Appends the node name to both the repo1-path and restore_path. 6. Creates an initial full backup using pgBackRest. 7. Performs HA-specific configurations if enabled. @@ -1418,12 +1415,12 @@ def init(cluster_name, install=True): backrest = node.get("backrest") if backrest: util.message("## Integrating pgBackRest into the cluster", "info") - util.message(f"### Configuring BackRest for node '{node['name']}'", "info") + util.message(f"### Configuring pgBackRest for node '{node['name']}'", "info") # Create a unique stanza name: {cluster_name}_stanza_{node_name} stanza = f"{cluster_name_from_json}_stanza_{node['name']}" - # Load additional BackRest settings from JSON with defaults. + # Load additional pgBackRest settings from JSON with defaults. repo1_retention_full = backrest.get("repo1_retention_full", "7") log_level_console = backrest.get("log_level_console", "info") repo1_cipher_type = backrest.get("repo1_cipher-type", "aes-256-cbc") @@ -1451,7 +1448,7 @@ def init(cluster_name, install=True): cmd_install_backrest = f"cd {node['path']}/pgedge && ./pgedge install backrest" run_cmd(cmd_install_backrest, node=node, message="Installing pgBackRest", verbose=verbose) - # -- Step 2: Configure postgresql.conf for BackRest (without --pg1-port) + # -- Step 2: Configure postgresql.conf for pgBackRest (without --pg1-port) cmd_set_postgresqlconf = ( f"cd {node['path']}/pgedge && " f"./pgedge backrest set_postgresqlconf " @@ -1460,11 +1457,11 @@ def init(cluster_name, install=True): f"--repo1-path {repo1_path} " f"--repo1-type {repo1_type}" ) - run_cmd(cmd_set_postgresqlconf, node=node, message="Modifying postgresql.conf for BackRest", verbose=verbose) + run_cmd(cmd_set_postgresqlconf, node=node, message="Modifying postgresql.conf for pgBackRest", verbose=verbose) - # -- Step 3: Configure pg_hba.conf for BackRest (without --pg1-port) + # -- Step 3: Configure pg_hba.conf for pgBackRest (without --pg1-port) cmd_set_hbaconf = f"cd {node['path']}/pgedge && ./pgedge backrest set_hbaconf" - run_cmd(cmd_set_hbaconf, node=node, message="Modifying pg_hba.conf for BackRest", verbose=verbose) + run_cmd(cmd_set_hbaconf, node=node, message="Modifying pg_hba.conf for pgBackRest", verbose=verbose) # -- Step 4: Reload PostgreSQL configuration to apply changes sql_reload_conf = "select pg_reload_conf()" @@ -1482,14 +1479,14 @@ def init(cluster_name, install=True): missing_env_vars = [var for var in required_env_vars if var not in os.environ] if missing_env_vars: util.exit_message( - f"Environment variables {', '.join(missing_env_vars)} must be set for S3 BackRest configuration.", + f"Environment variables {', '.join(missing_env_vars)} must be set for S3 pgBackRest configuration.", 1, ) s3_exports = " && ".join([f"export {var}={os.environ[var]}" for var in required_env_vars]) cmd_export_s3 = f"cd {node['path']}/pgedge && {s3_exports}" - run_cmd(cmd_export_s3, node=node, message="Setting S3 environment variables for BackRest", verbose=verbose) + run_cmd(cmd_export_s3, node=node, message="Setting S3 environment variables for pgBackRest", verbose=verbose) - # -- Step 6: Set all BackRest backup configuration values + # -- Step 6: Set all pgBackRest backup configuration values # (a) Set the backup stanza cmd_set_backup_stanza = f"cd {node['path']}/pgedge && ./pgedge set BACKUP stanza {stanza}" @@ -1518,7 +1515,7 @@ def init(cluster_name, install=True): cmd_set_pg1_port = f"cd {node['path']}/pgedge && ./pgedge set BACKUP pg1-port {port}" run_cmd(cmd_set_pg1_port, node=node, message=f"Setting BACKUP pg1-port to {port} on node '{node['name']}'", verbose=verbose) - # -- Step 7: Create the BackRest stanza (this command uses --pg1-port because it connects to the DB) + # -- Step 7: Create the pgBackRest stanza (this command uses --pg1-port because it connects to the DB) cmd_create_stanza = ( f"cd {node['path']}/pgedge && " f"./pgedge backrest command stanza-create " @@ -1544,7 +1541,7 @@ def init(cluster_name, install=True): f"--type=full" ) cmd_create_backup = f"cd {node['path']}/pgedge && ./pgedge backrest command backup '{backrest_backup_args}'" - run_cmd(cmd_create_backup, node=node, message="Creating full BackRest backup", verbose=verbose) + run_cmd(cmd_create_backup, node=node, message="Creating full pgBackRest backup", verbose=verbose) # (f) Set BACKUP pg1-port to the node's port value cmd_set_pg1_port = f"cd {node['path']}/pgedge && ./pgedge set BACKUP repo1-path {repo1_path}" run_cmd(cmd_set_pg1_port, node=node, message=f"Setting BACKUP repo1-path to {repo1_path} on node '{node['name']}'", verbose=verbose) @@ -1569,21 +1566,22 @@ def init(cluster_name, install=True): def check_source_backrest_config(source_node_data): """ - Check the source node's JSON data for a BackRest configuration. + Check the source node's JSON data for a pgBackRest configuration. If a non‑empty 'backrest' block is found, display its configuration. - Otherwise, display a message that no BackRest configuration exists, - and remove any leftover BackRest configuration. + Otherwise, display a message that no pgBackRest configuration exists, + and remove any leftover pgBackRest configuration. """ if "backrest" in source_node_data and source_node_data["backrest"]: util.message( - f"Source node '{source_node_data['name']}' already has BackRest configuration: {source_node_data['backrest']}", + f"Source node '{source_node_data['name']}' already has pgBackRest configuration: {source_node_data['backrest']}", "info" ) else: - + cmd = f"cd {source_node_data['path']}/pgedge && ./pgedge remove backrest" - run_cmd(cmd, node=source_node_data, message="Removing BackRest configuration from source node", verbose=True) - + run_cmd(cmd, node=source_node_data, message="Removing pgBackRest configuration from source node", verbose=True) + + def add_node( cluster_name, source_node, @@ -1591,7 +1589,6 @@ def add_node( repo1_path=None, backup_id=None, script=" ", - stanza=" ", install=True, ): """ @@ -1606,7 +1603,7 @@ def add_node( repo1_path (str): The repo1 path to use. backup_id (str): Backup ID. script (str): Bash script. - """ + """ db, db_settings, nodes = load_json(cluster_name) cluster_data = get_cluster_json(cluster_name) @@ -1634,21 +1631,21 @@ def add_node( target_backrest_settings = target_node_data.get("backrest", {}) # Retrieve source node data source_node_data = next( - (node for node in nodes if node["name"] == source_node), None -) + (node for node in nodes if node["name"] == source_node), None + ) if source_node_data is None: - util.exit_message(f"Source node '{source_node}' not found in cluster data.") + util.exit_message(f"Source node '{source_node}' not found in cluster data.") -# Extract backrest settings from source node (before using repo1_path flag) + # Extract backrest settings from source node (before using repo1_path flag) backrest_settings = source_node_data.get("backrest", {}) source_repo1_path = backrest_settings.get("repo1_path") -# Check: if source node JSON already provides repo1_path and the flag is given then exit + # Check: if source node JSON already provides repo1_path and the flag is given then exit if repo1_path and source_repo1_path: - util.exit_message( - "Error: The source node JSON already contains a repo1_path. " - "Do not provide the repo1_path flag when the source node has it configured." - ) + util.exit_message( + "Error: The source node JSON already contains a repo1_path. " + "Do not provide the repo1_path flag when the source node has it configured." + ) for group in target_node_data.get("node_groups", []): ssh_info = group.get("ssh") @@ -1667,7 +1664,8 @@ def add_node( "os_user": os_user, "ssh_key": ssh_key, } - # If backrest settings are provided in the JSON, add them to new_node_data. + + # If backrest settings are provided in the JSON, add them to new_node_data. if backrest_info: new_node_data["backrest"] = backrest_info if "public_ip" not in new_node_data and "private_ip" not in new_node_data: @@ -1694,37 +1692,50 @@ def add_node( # New check: if pgBackRest is not configured on the source node if not backrest_settings: # Step 1: Install pgBackRest on the source node - cmd_install_backrest = f"cd {source_node_data['path']}/pgedge && ./pgedge install backrest" - run_cmd(cmd_install_backrest, node=source_node_data, message="Installing pgBackRest", verbose=verbose) + cmd_install_backrest = ( + f"cd {source_node_data['path']}/pgedge && ./pgedge install backrest" + ) + run_cmd( + cmd_install_backrest, + node=source_node_data, + message="Installing pgBackRest", + verbose=verbose, + ) util.message("## Integrating pgBackRest into the cluster", "info") - util.message(f"### Configuring pgBackRest for node '{source_node_data['name']}'", "info") + util.message( + f"### Configuring pgBackRest for node '{source_node_data['name']}'", "info" + ) # Create a unique stanza name using the cluster name and node name stanza_source = f"{cluster_name}_stanza_{source_node_data['name']}" - # Load additional BackRest settings with defaults. + # Load additional pgBackRest settings with defaults. repo1_retention_full = "7" log_level_console = "info" repo1_cipher_type = "aes-256-cbc" repo1_type = "posix" # Could also be "s3" -# Determine the repository path for the source node. + # Determine the repository path for the source node. if repo1_path: - # Use the provided flag value as-is (trimmed of any trailing slash). - repo1_path_source = repo1_path.rstrip('/') + # Use the provided flag value as-is (trimmed of any trailing slash). + repo1_path_source = repo1_path.rstrip("/") else: - # No repo1_path flag provided: check the JSON configuration or default. - json_repo1_path = backrest_settings.get("repo1_path") - if json_repo1_path: - repo1_path_source = json_repo1_path.rstrip('/') - if not repo1_path_source.endswith(source_node_data["name"]): - repo1_path_source = repo1_path_source + f"/{source_node_data['name']}" - else: - repo1_path_source = f"/var/lib/pgbackrest/{source_node_data['name']}" + # No repo1_path flag provided: check the JSON configuration or default. + json_repo1_path = backrest_settings.get("repo1_path") + if json_repo1_path: + repo1_path_source = json_repo1_path.rstrip("/") + if not repo1_path_source.endswith(source_node_data["name"]): + repo1_path_source = ( + repo1_path_source + f"/{source_node_data['name']}" + ) + else: + repo1_path_source = f"/var/lib/pgbackrest/{source_node_data['name']}" # Similarly, set restore_path to include node name restore_path_source = "/var/lib/pgbackrest_restore" - if not restore_path_source.rstrip('/').endswith(source_node_data["name"]): - restore_path_source = restore_path_source.rstrip('/') + f"/{source_node_data['name']}" + if not restore_path_source.rstrip("/").endswith(source_node_data["name"]): + restore_path_source = ( + restore_path_source.rstrip("/") + f"/{source_node_data['name']}" + ) pg_version = db_settings["pg_version"] pg1_path_source = f"{source_node_data['path']}/pgedge/data/pg{pg_version}" @@ -1739,16 +1750,33 @@ def add_node( f"--repo1-path {repo1_path_source} " f"--repo1-type {repo1_type}" ) - run_cmd(cmd_set_postgresqlconf_source, node=source_node_data, message="Modifying postgresql.conf for pgBackRest", verbose=verbose) + run_cmd( + cmd_set_postgresqlconf_source, + node=source_node_data, + message="Modifying postgresql.conf for pgBackRest", + verbose=verbose, + ) # Step 3: Configure pg_hba.conf for pgBackRest (without --pg1-port) - cmd_set_hbaconf_source = f"cd {source_node_data['path']}/pgedge && ./pgedge backrest set_hbaconf" - run_cmd(cmd_set_hbaconf_source, node=source_node_data, message="Modifying pg_hba.conf for pgBackRest", verbose=verbose) + cmd_set_hbaconf_source = ( + f"cd {source_node_data['path']}/pgedge && ./pgedge backrest set_hbaconf" + ) + run_cmd( + cmd_set_hbaconf_source, + node=source_node_data, + message="Modifying pg_hba.conf for pgBackRest", + verbose=verbose, + ) # Step 4: Reload PostgreSQL configuration to apply changes sql_reload_conf = "select pg_reload_conf()" cmd_reload_conf_source = f"cd {source_node_data['path']}/pgedge && ./pgedge psql '{sql_reload_conf}' {db[0]['db_name']}" - run_cmd(cmd_reload_conf_source, node=source_node_data, message="Reloading PostgreSQL configuration", verbose=verbose) + run_cmd( + cmd_reload_conf_source, + node=source_node_data, + message="Reloading PostgreSQL configuration", + verbose=verbose, + ) # Step 5: If using S3 as repository, export necessary environment variables if repo1_type.lower() == "s3": @@ -1758,37 +1786,50 @@ def add_node( "PGBACKREST_REPO1_S3_KEY_SECRET", "PGBACKREST_REPO1_CIPHER_PASS", ] - missing_env_vars = [var for var in required_env_vars if var not in os.environ] + missing_env_vars = [ + var for var in required_env_vars if var not in os.environ + ] if missing_env_vars: util.exit_message( f"Environment variables {', '.join(missing_env_vars)} must be set for S3 pgBackRest configuration.", 1, ) - s3_exports = " && ".join([f"export {var}={os.environ[var]}" for var in required_env_vars]) - cmd_export_s3_source = f"cd {source_node_data['path']}/pgedge && {s3_exports}" - run_cmd(cmd_export_s3_source, node=source_node_data, message="Setting S3 environment variables for pgBackRest", verbose=verbose) + s3_exports = " && ".join( + [f"export {var}={os.environ[var]}" for var in required_env_vars] + ) + cmd_export_s3_source = ( + f"cd {source_node_data['path']}/pgedge && {s3_exports}" + ) + run_cmd( + cmd_export_s3_source, + node=source_node_data, + message="Setting S3 environment variables for pgBackRest", + verbose=verbose, + ) # Step 6: Set all pgBackRest backup configuration values for the source node. # # Build one compound shell command - compound_cmd = " && ".join([ - f"cd {source_node_data['path']}/pgedge", - f"./pgedge set BACKUP stanza {stanza_source}", - f"sudo mkdir -p {restore_path_source}", - f"./pgedge set BACKUP restore_path {restore_path_source}", - f"./pgedge set BACKUP repo1-host-user {source_node_data.get('os_user', 'postgres')}", - f"./pgedge set BACKUP pg1-path {pg1_path_source}", - f"./pgedge set BACKUP pg1-user {source_node_data.get('os_user', 'postgres')}", - f"./pgedge set BACKUP pg1-port {port_source}" - ]) + compound_cmd = " && ".join( + [ + f"cd {source_node_data['path']}/pgedge", + f"./pgedge set BACKUP stanza {stanza_source}", + f"sudo mkdir -p {restore_path_source}", + f"./pgedge set BACKUP restore_path {restore_path_source}", + f"./pgedge set BACKUP repo1-host-user {source_node_data.get('os_user', 'postgres')}", + f"./pgedge set BACKUP pg1-path {pg1_path_source}", + f"./pgedge set BACKUP pg1-user {source_node_data.get('os_user', 'postgres')}", + f"./pgedge set BACKUP pg1-port {port_source}", + ] + ) # Execute once with verbose disabled run_cmd( compound_cmd, node=source_node_data, - message=f"Configuring BACKUP settings on node '{source_node_data['name']}'", - verbose=False - ) + message=f"Configuring BACKUP settings on node '{source_node_data['name']}'", + verbose=False, + ) # Step 7:Create the pgBackRest stanza (this command uses --pg1-port because it connects to the DB) cmd_create_stanza_source = ( @@ -1800,7 +1841,12 @@ def add_node( f"--pg1-port {port_source} " f"--repo1-path {repo1_path_source}" ) - run_cmd(cmd_create_stanza_source, node=source_node_data, message=f"Creating pgBackRest stanza '{stanza_source}'", verbose=verbose) + run_cmd( + cmd_create_stanza_source, + node=source_node_data, + message=f"Creating pgBackRest stanza '{stanza_source}'", + verbose=verbose, + ) # Step 8: Initiate a full backup using pgBackRest (again, passing the port) backrest_backup_args_source = ( f"--repo1-path {repo1_path_source} " @@ -1815,10 +1861,20 @@ def add_node( f"--type=full" ) cmd_create_backup_source = f"cd {source_node_data['path']}/pgedge && ./pgedge backrest command backup '{backrest_backup_args_source}'" - run_cmd(cmd_create_backup_source, node=source_node_data, message="Creating full pgBackRest backup", verbose=verbose) + run_cmd( + cmd_create_backup_source, + node=source_node_data, + message="Creating full pgBackRest backup", + verbose=verbose, + ) # (i) (Optional) Reset BACKUP repo1-path if needed cmd_set_repo1_path_source = f"cd {source_node_data['path']}/pgedge && ./pgedge set BACKUP repo1-path {repo1_path_source}" - run_cmd(cmd_set_repo1_path_source, node=source_node_data, message=f"Setting BACKUP repo1-path to {repo1_path_source} on node '{source_node_data['name']}'", verbose=verbose) + run_cmd( + cmd_set_repo1_path_source, + node=source_node_data, + message=f"Setting BACKUP repo1-path to {repo1_path_source} on node '{source_node_data['name']}'", + verbose=verbose, + ) # Update pgbackrest_settings for further use downstream. backrest_settings = { @@ -1827,7 +1883,7 @@ def add_node( "repo1-retention-full": repo1_retention_full, "log-level-console": log_level_console, "repo1-cipher-type": repo1_cipher_type, - "repo1_type": repo1_type + "repo1_type": repo1_type, } # NEW: Override pgbackrest settings with target JSON settings if present @@ -1990,7 +2046,11 @@ def add_node( cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {mdb['db_name']}" message = "Fetch existing subscriptions" result = run_cmd( - cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True + cmd, + node=new_node_data, + message=message, + verbose=verbose, + capture_output=True, ) subscriptions = [ @@ -2012,7 +2072,11 @@ def add_node( cmd = f"{new_node_data['path']}/pgedge/pgedge psql '{sql_cmd}' {mdb['db_name']}" message = "Check if there are nodes" result = run_cmd( - cmd, node=new_node_data, message=message, verbose=verbose, capture_output=True + cmd, + node=new_node_data, + message=message, + verbose=verbose, + capture_output=True, ) print(f"\nRaw output:\n{result.stdout}") @@ -2039,7 +2103,7 @@ def add_node( create_sub(nodes, new_node_data, mdb["db_name"], verbose) create_sub_new(nodes, new_node_data, mdb["db_name"], verbose) - nc = os.path.join(new_node_data['path'], "pgedge", "pgedge ") + nc = os.path.join(new_node_data["path"], "pgedge", "pgedge ") cmd = f'{nc} spock repset-add-table default "*" {mdb["db_name"]}' message = f"Adding all tables to repset" run_cmd(cmd, new_node_data, message=message, verbose=verbose) @@ -2094,24 +2158,41 @@ def add_node( ) print(f"\n{result.stdout}") - - # Reload the complete target JSON file and fetch repo1_path and stanza from its pgbackrest settings. try: with open(target_node_file, "r") as f: complete_target_json = json.load(f) # Since pgbackrest settings are stored inside each node group, fetch from the first node group. - if "node_groups" in complete_target_json and complete_target_json["node_groups"]: - repo1_path_target_file = complete_target_json["node_groups"][0].get("backrest", {}).get("repo1_path") - target_stanza_target_file = complete_target_json["node_groups"][0].get("backrest", {}).get("stanza") + if ( + "node_groups" in complete_target_json + and complete_target_json["node_groups"] + ): + repo1_path_target_file = ( + complete_target_json["node_groups"][0] + .get("backrest", {}) + .get("repo1_path") + ) + target_stanza_target_file = ( + complete_target_json["node_groups"][0].get("backrest", {}).get("stanza") + ) pgedge_dir = f"{new_node_data['path']}/pgedge" - restore_path = backrest_settings.get("restore_path", f"/var/lib/pgbackrest_restore/{new_node_data['name']}") - target_repo1_host_user = backrest_settings.get("repo1_host_user", new_node_data.get("os_user", "postgres")) - target_pg1_path = backrest_settings.get("pg1_path", f"{pgedge_dir}/data/pg{pg}") - target_pg1_user = backrest_settings.get("pg1_user", new_node_data.get("os_user", "postgres")) - target_pg1_port = backrest_settings.get("pg1_port", new_node_data.get("port", "6435")) + restore_path = backrest_settings.get( + "restore_path", f"/var/lib/pgbackrest_restore/{new_node_data['name']}" + ) + target_repo1_host_user = backrest_settings.get( + "repo1_host_user", new_node_data.get("os_user", "postgres") + ) + target_pg1_path = backrest_settings.get( + "pg1_path", f"{pgedge_dir}/data/pg{pg}" + ) + target_pg1_user = backrest_settings.get( + "pg1_user", new_node_data.get("os_user", "postgres") + ) + target_pg1_port = backrest_settings.get( + "pg1_port", new_node_data.get("port", "6435") + ) # --------------------------------------------------------------- # If the repo1_path or stanza is missing, remove backrest instead @@ -2128,7 +2209,7 @@ def add_node( cmd=cmd_remove_backrest_target, node=new_node_data, message="Removing backrest", - verbose=verbose + verbose=verbose, ) else: # Both repo1_path_target_file and target_stanza_target_file exist @@ -2153,7 +2234,7 @@ def add_node( cmd=combined_target_cmd, node=new_node_data, message="Setting all target node BACKUP configuration", - verbose=False + verbose=False, ) # Append the BACKUP settings to the target's PostgreSQL configuration @@ -2165,7 +2246,7 @@ def add_node( cmd=cmd_set_postgresqlconf_target, node=new_node_data, message="Appending BACKUP settings to postgresql.conf for target node", - verbose=verbose + verbose=verbose, ) # Restart PostgreSQL to apply the new configuration @@ -2174,7 +2255,7 @@ def add_node( cmd=cmd_restart_postgres, node=new_node_data, message="Restarting PostgreSQL service", - verbose=verbose + verbose=verbose, ) # Now create the pgBackRest stanza on the target node @@ -2191,7 +2272,7 @@ def add_node( cmd=cmd_create_stanza_target, node=new_node_data, message=f"Creating pgBackRest stanza '{target_stanza}'", - verbose=verbose + verbose=verbose, ) # Create a full backup using pgBackRest @@ -2214,8 +2295,8 @@ def add_node( run_cmd( cmd=cmd_create_backup_target, node=new_node_data, - message="Creating full BackRest backup", - verbose=verbose + message="Creating full pgBackRest backup", + verbose=verbose, ) else: # If no node_groups exist at all, remove pgbackrest @@ -2223,33 +2304,32 @@ def add_node( target_stanza_target_file = None pgedge_dir = f"{new_node_data['path']}/pgedge" - cmd_remove_backrest_target = ( - f"cd {pgedge_dir} && ./pgedge remove backrest" - ) + cmd_remove_backrest_target = f"cd {pgedge_dir} && ./pgedge remove backrest" run_cmd( cmd=cmd_remove_backrest_target, node=new_node_data, message="Removing backrest", - verbose=verbose + verbose=verbose, ) - + except Exception as e: print(f"Error fetching values from target JSON file: {e}") - # NEW: Check and display pgBackRest configuration status in the source node - # Remove unnecessary keys before appending new node to the cluster data + + # NEW: Check and display pgBackRest configuration status in the source node + # Remove unnecessary keys before appending new node to the cluster data new_node_data.pop("ip_address", None) new_node_data.pop("os_user", None) new_node_data.pop("ssh_key", None) # Append new node data to the cluster JSON - node_group = target_node_data.get cluster_data["node_groups"].append(new_node_data) cluster_data["update_date"] = datetime.datetime.now().astimezone().isoformat() write_cluster_json(cluster_name, cluster_data) - + check_source_backrest_config(source_node_data) - + + def cleanup_backrest_from_cluster(cluster_json, target_json): """ Compare the node groups in the target JSON with the cluster JSON. @@ -2271,7 +2351,7 @@ def cleanup_backrest_from_cluster(cluster_json, target_json): if not target_group or "backrest" not in target_group: if "backrest" in node: del node["backrest"] - + def json_validate_add_node(data): """ Validate the structure of a node‑definition JSON file that will be fed to @@ -2333,14 +2413,14 @@ def json_validate_add_node(data): missing_br = backrest_required - set(br.keys()) if missing_br: util.exit_message( - f"BackRest block in node‑group '{gname}' missing: {', '.join(missing_br)}" + f"pgBackRest block in node‑group '{gname}' missing: {', '.join(missing_br)}" ) # ensure values are non‑empty for k in backrest_required: if not str(br[k]).strip(): util.exit_message( - f"BackRest key '{k}' in node‑group '{gname}' cannot be empty." + f"pgBackRest key '{k}' in node‑group '{gname}' cannot be empty." ) # verify repo1_type is valid