From 4beebac3b960e7b902019f9ca354b40fd357de57 Mon Sep 17 00:00:00 2001 From: Yun Zheng Hu Date: Sun, 15 Oct 2023 17:00:12 +0000 Subject: [PATCH 1/5] T2405: add Git support to commit-archive --- debian/control | 3 + .../system-config-mgmt.xml.in | 2 +- python/vyos/config_mgmt.py | 16 +- python/vyos/remote.py | 142 +++++++++++++++++- python/vyos/utils/process.py | 2 + 5 files changed, 154 insertions(+), 11 deletions(-) diff --git a/debian/control b/debian/control index 98c064417f..cfd2186c39 100644 --- a/debian/control +++ b/debian/control @@ -230,6 +230,9 @@ Depends: lcdproc, lcdproc-extra-drivers, # End "system lcd" +# For "system config-management commit-archive" + git +# End "system config-management commit-archive" # For firewall libndp-tools, libnetfilter-conntrack3, diff --git a/interface-definitions/system-config-mgmt.xml.in b/interface-definitions/system-config-mgmt.xml.in index de5a8cc16f..f40a0e4c3f 100644 --- a/interface-definitions/system-config-mgmt.xml.in +++ b/interface-definitions/system-config-mgmt.xml.in @@ -21,7 +21,7 @@ Uniform Resource Identifier - + (http|https|ftp|ftps|sftp|ssh|scp|tftp|git|git\+(\w+)):\/\/.* diff --git a/python/vyos/config_mgmt.py b/python/vyos/config_mgmt.py index 654a8d698d..5d4ee86ab9 100644 --- a/python/vyos/config_mgmt.py +++ b/python/vyos/config_mgmt.py @@ -22,7 +22,7 @@ from typing import Optional, Tuple, Union from filecmp import cmp from datetime import datetime -from textwrap import dedent +from textwrap import dedent, indent from pathlib import Path from tabulate import tabulate from shutil import copy, chown @@ -377,9 +377,19 @@ def commit_archive(self): remote_file = f'config.boot-{hostname}.{timestamp}' source_address = self.source_address + if self.effective_locations: + print("Archiving config...") for location in self.effective_locations: - upload(archive_config_file, f'{location}/{remote_file}', - source_host=source_address) + print(f" {location}", end=" ", flush=True) + try: + upload(archive_config_file, f'{location}/{remote_file}', + source_host=source_address, raise_error=True) + print("OK") + except Exception as e: + print("FAILED!") + print() + print(indent(str(e), " > ")) + print() # op-mode functions # diff --git a/python/vyos/remote.py b/python/vyos/remote.py index 4be477d242..2fcba74d0d 100644 --- a/python/vyos/remote.py +++ b/python/vyos/remote.py @@ -14,6 +14,7 @@ # License along with this library. If not, see . import os +import pwd import shutil import socket import ssl @@ -22,6 +23,9 @@ import tempfile import urllib.parse +from contextlib import contextmanager +from pathlib import Path + from ftplib import FTP from ftplib import FTP_TLS @@ -37,11 +41,22 @@ from vyos.utils.io import is_interactive from vyos.utils.io import print_error from vyos.utils.misc import begin -from vyos.utils.process import cmd +from vyos.utils.process import cmd, rc_cmd from vyos.version import get_version CHUNK_SIZE = 8192 +@contextmanager +def umask(mask: int): + """ + Context manager that temporarily sets the process umask. + """ + oldmask = os.umask(mask) + try: + yield + finally: + os.umask(oldmask) + class InteractivePolicy(MissingHostKeyPolicy): """ Paramiko policy for interactively querying the user on whether to proceed @@ -310,35 +325,148 @@ def upload(self, location: str): with open(location, 'rb') as f: cmd(f'{self.command} -T - "{self.urlstring}"', input=f.read()) +class GitC: + def __init__(self, + url, + progressbar=False, + check_space=False, + source_host=None, + source_port=0, + timeout=10, + ): + self.command = 'git' + self.url = url + self.urlstring = urllib.parse.urlunsplit(url) + if self.urlstring.startswith("git+"): + self.urlstring = self.urlstring.replace("git+", "", 1) + + def download(self, location: str): + raise NotImplementedError("not supported") + + @umask(0o077) + def upload(self, location: str): + scheme = self.url.scheme + _, _, scheme = scheme.partition("+") + netloc = self.url.netloc + url = Path(self.url.path).parent + with tempfile.TemporaryDirectory(prefix="git-commit-archive-") as directory: + # Determine username, fullname, email for Git commit + pwd_entry = pwd.getpwuid(os.getuid()) + user = pwd_entry.pw_name + name = pwd_entry.pw_gecos.split(",")[0] or user + fqdn = socket.getfqdn() + email = f"{user}@{fqdn}" + + # environment vars for our git commands + env = { + "GIT_TERMINAL_PROMPT": "0", + "GIT_AUTHOR_NAME": name, + "GIT_AUTHOR_EMAIL": email, + "GIT_COMMITTER_NAME": name, + "GIT_COMMITTER_EMAIL": email, + } + + # build ssh command for git + ssh_command = ["ssh"] + + # Try to use /config/auth/commit-archive.key as SSH identity + # We copy over the key so we can control the permissions + try: + path_privatekey = Path(directory) / "private.key" + path_privatekey.write_bytes( + Path("/config/auth/commit-archive.key").read_bytes() + ) + ssh_command += ["-i", str(path_privatekey)] + except Exception: + pass + + # if we are not interactive, we use StrictHostKeyChecking=yes to avoid any prompts + if not sys.stdout.isatty(): + ssh_command += ["-o", "StrictHostKeyChecking=yes"] + + env["GIT_SSH_COMMAND"] = " ".join(ssh_command) + + # git clone + path_repository = Path(directory) / "repository" + scheme = f"{scheme}://" if scheme else "" + rc, out = rc_cmd( + [self.command, "clone", f"{scheme}{netloc}{url}", str(path_repository), "--depth=1"], + env=env, + shell=False, + ) + if rc: + raise Exception(out) + + # git add + filename = Path(Path(self.url.path).name).stem + dst = path_repository / filename + shutil.copy2(location, dst) + rc, out = rc_cmd( + [self.command, "-C", str(path_repository), "add", filename], + env=env, + shell=False, + ) + + # git commit -m + commit_message = os.environ.get("COMMIT_COMMENT", "commit") + rc, out = rc_cmd( + [self.command, "-C", str(path_repository), "commit", "-m", commit_message], + env=env, + shell=False, + ) + + # git push + rc, out = rc_cmd( + [self.command, "-C", str(path_repository), "push"], + env=env, + shell=False, + ) + if rc: + raise Exception(out) + def urlc(urlstring, *args, **kwargs): """ Dynamically dispatch the appropriate protocol class. """ - url_classes = {'http': HttpC, 'https': HttpC, 'ftp': FtpC, 'ftps': FtpC, \ - 'sftp': SshC, 'ssh': SshC, 'scp': SshC, 'tftp': TftpC} + url_classes = { + "http": HttpC, + "https": HttpC, + "ftp": FtpC, + "ftps": FtpC, + "sftp": SshC, + "ssh": SshC, + "scp": SshC, + "tftp": TftpC, + "git": GitC, + } url = urllib.parse.urlsplit(urlstring) + scheme, _, _ = url.scheme.partition("+") try: - return url_classes[url.scheme](url, *args, **kwargs) + return url_classes[scheme](url, *args, **kwargs) except KeyError: - raise ValueError(f'Unsupported URL scheme: "{url.scheme}"') + raise ValueError(f'Unsupported URL scheme: "{scheme}"') -def download(local_path, urlstring, progressbar=False, check_space=False, +def download(local_path, urlstring, progressbar=False, raise_error=False, check_space=False, source_host='', source_port=0, timeout=10.0): try: progressbar = progressbar and is_interactive() urlc(urlstring, progressbar, check_space, source_host, source_port, timeout).download(local_path) except Exception as err: + if raise_error: + raise print_error(f'Unable to download "{urlstring}": {err}') except KeyboardInterrupt: print_error('\nDownload aborted by user.') -def upload(local_path, urlstring, progressbar=False, +def upload(local_path, urlstring, progressbar=False, raise_error=False, source_host='', source_port=0, timeout=10.0): try: progressbar = progressbar and is_interactive() urlc(urlstring, progressbar, source_host, source_port, timeout).upload(local_path) except Exception as err: + if raise_error: + raise print_error(f'Unable to upload "{urlstring}": {err}') except KeyboardInterrupt: print_error('\nUpload aborted by user.') diff --git a/python/vyos/utils/process.py b/python/vyos/utils/process.py index e09c7d86d7..150d0eca74 100644 --- a/python/vyos/utils/process.py +++ b/python/vyos/utils/process.py @@ -138,6 +138,7 @@ def cmd(command, flag='', shell=None, input=None, timeout=None, env=None, (default is OSError) with the error code expect: a list of error codes to consider as normal """ + command = command.lstrip() if isinstance(command, str) else command decoded, code = popen( command, flag, stdout=stdout, stderr=stderr, @@ -169,6 +170,7 @@ def rc_cmd(command, flag='', shell=None, input=None, timeout=None, env=None, % rc_cmd('ip link show dev eth99') (1, 'Device "eth99" does not exist.') """ + command = command.lstrip() if isinstance(command, str) else command out, code = popen( command, flag, stdout=stdout, stderr=stderr, From 620b5da9d84675c5a6260f14c364f17c3d8e3fd3 Mon Sep 17 00:00:00 2001 From: Yun Zheng Hu Date: Tue, 12 Sep 2023 17:01:25 +0200 Subject: [PATCH 2/5] T2405: revert command.lstrip() --- python/vyos/utils/process.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/vyos/utils/process.py b/python/vyos/utils/process.py index 150d0eca74..e09c7d86d7 100644 --- a/python/vyos/utils/process.py +++ b/python/vyos/utils/process.py @@ -138,7 +138,6 @@ def cmd(command, flag='', shell=None, input=None, timeout=None, env=None, (default is OSError) with the error code expect: a list of error codes to consider as normal """ - command = command.lstrip() if isinstance(command, str) else command decoded, code = popen( command, flag, stdout=stdout, stderr=stderr, @@ -170,7 +169,6 @@ def rc_cmd(command, flag='', shell=None, input=None, timeout=None, env=None, % rc_cmd('ip link show dev eth99') (1, 'Device "eth99" does not exist.') """ - command = command.lstrip() if isinstance(command, str) else command out, code = popen( command, flag, stdout=stdout, stderr=stderr, From 754390f4d42fa5cbbccba0e91a4b56628b52ccaa Mon Sep 17 00:00:00 2001 From: Yun Zheng Hu Date: Thu, 14 Sep 2023 21:54:06 +0200 Subject: [PATCH 3/5] T2405: Add regex for git and ssh protocol --- interface-definitions/system-config-mgmt.xml.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/interface-definitions/system-config-mgmt.xml.in b/interface-definitions/system-config-mgmt.xml.in index f40a0e4c3f..794f9f1a0c 100644 --- a/interface-definitions/system-config-mgmt.xml.in +++ b/interface-definitions/system-config-mgmt.xml.in @@ -21,7 +21,8 @@ Uniform Resource Identifier - (http|https|ftp|ftps|sftp|ssh|scp|tftp|git|git\+(\w+)):\/\/.* + + (ssh|git|git\+(\w+)):\/\/.* From d09ed94b32aa0b4f20d530d534fc8481a26b8d91 Mon Sep 17 00:00:00 2001 From: Yun Zheng Hu Date: Fri, 10 Nov 2023 21:41:58 +0000 Subject: [PATCH 4/5] T2405: Redact username and password --- python/vyos/config_mgmt.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/vyos/config_mgmt.py b/python/vyos/config_mgmt.py index 5d4ee86ab9..df7240c88d 100644 --- a/python/vyos/config_mgmt.py +++ b/python/vyos/config_mgmt.py @@ -26,6 +26,7 @@ from pathlib import Path from tabulate import tabulate from shutil import copy, chown +from urllib.parse import urlsplit, urlunsplit from vyos.config import Config from vyos.configtree import ConfigTree, ConfigTreeError, show_diff @@ -380,7 +381,10 @@ def commit_archive(self): if self.effective_locations: print("Archiving config...") for location in self.effective_locations: - print(f" {location}", end=" ", flush=True) + url = urlsplit(location) + _, _, netloc = url.netloc.rpartition("@") + redacted_location = urlunsplit(url._replace(netloc=netloc)) + print(f" {redacted_location}", end=" ", flush=True) try: upload(archive_config_file, f'{location}/{remote_file}', source_host=source_address, raise_error=True) From 529216ace868711d65cec251175b5e322f3ba52c Mon Sep 17 00:00:00 2001 From: Yun Zheng Hu Date: Fri, 10 Nov 2023 21:44:51 +0000 Subject: [PATCH 5/5] T2405: Remove commit-archive.key support --- python/vyos/remote.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/python/vyos/remote.py b/python/vyos/remote.py index 2fcba74d0d..8b90e45307 100644 --- a/python/vyos/remote.py +++ b/python/vyos/remote.py @@ -369,17 +369,6 @@ def upload(self, location: str): # build ssh command for git ssh_command = ["ssh"] - # Try to use /config/auth/commit-archive.key as SSH identity - # We copy over the key so we can control the permissions - try: - path_privatekey = Path(directory) / "private.key" - path_privatekey.write_bytes( - Path("/config/auth/commit-archive.key").read_bytes() - ) - ssh_command += ["-i", str(path_privatekey)] - except Exception: - pass - # if we are not interactive, we use StrictHostKeyChecking=yes to avoid any prompts if not sys.stdout.isatty(): ssh_command += ["-o", "StrictHostKeyChecking=yes"]