Merge pull request #7116 from freedomofpress/tails6-ftw

Add support for Tails 6

Makefile

@@ -32,7 +32,6 @@ update-python3-requirements:  ## Update Python 3 requirements with pip-compile.
 	@SLIM_BUILD=1 $(DEVSHELL) pip-compile --generate-hashes \
 		--allow-unsafe \
 		--output-file requirements/python3/develop-requirements.txt \
-		../admin/requirements-ansible.in \
 		../admin/requirements.in \
 		requirements/python3/translation-requirements.in \
 		requirements/python3/develop-requirements.in

admin/Dockerfile

@@ -1,5 +1,5 @@
-# debian:bullseye 2022-10-04
-FROM debian@sha256:9b0e3056b8cd8630271825665a0613cc27829d6a24906dc0122b3b4834312f7d
+# debian:bookworm 2024-01-12
+FROM debian@sha256:b16cef8cbcb20935c0f052e37fc3d38dc92bfec0bcfb894c328547f81e932d67
 ARG USER_NAME
 ENV USER_NAME ${USER_NAME:-root}
 ARG USER_ID

admin/bootstrap.py

@@ -66,7 +66,7 @@ def run_command(command: List[str]) -> Iterator[bytes]:
 
 def is_tails() -> bool:
     with open("/etc/os-release") as f:
-        return "TAILS_PRODUCT_NAME" in f.read()
+        return 'NAME="Tails"' in f.read()
 
 
 def clean_up_old_tails_venv(virtualenv_dir: str = VENV_DIR) -> None:
@@ -80,15 +80,15 @@ def clean_up_old_tails_venv(virtualenv_dir: str = VENV_DIR) -> None:
         with open("/etc/os-release") as f:
             os_release = f.readlines()
             for line in os_release:
-                if line.startswith("TAILS_VERSION_ID="):
+                if line.startswith("VERSION="):
                     version = line.split("=")[1].strip().strip('"')
-                    if version.startswith("5."):
-                        # Tails 5 is based on Python 3.9
-                        python_lib_path = os.path.join(virtualenv_dir, "lib/python3.7")
+                    if version.startswith("6."):
+                        # Tails 6 is based on Python 3.11
+                        python_lib_path = os.path.join(virtualenv_dir, "lib/python3.9")
                         if os.path.exists(python_lib_path):
-                            sdlog.info("Tails 4 virtualenv detected. Removing it.")
+                            sdlog.info("Tails 5 virtualenv detected. Removing it.")
                             shutil.rmtree(virtualenv_dir)
-                            sdlog.info("Tails 4 virtualenv deleted.")
+                            sdlog.info("Tails 5 virtualenv deleted.")
                     break
 
 
@@ -146,13 +146,7 @@ def install_apt_dependencies(args: argparse.Namespace) -> None:
         " which was set on Tails login screen"
     )
 
-    apt_command = [
-        "sudo",
-        "su",
-        "-c",
-        f"apt-get update && \
-                   apt-get -q -o=Dpkg::Use-Pty=0 install -y {APT_DEPENDENCIES_STR}",
-    ]
+    apt_command = f"sudo apt-get -q -o=Dpkg::Use-Pty=0 install -y {APT_DEPENDENCIES_STR}".split(" ")
 
     try:
         # Print command results in real-time, to keep Admin apprised

admin/requirements-ansible.in

@@ -1,4 +1,4 @@
-ansible==6.7.0
+ansible==8.7.0
 cryptography>=41.0.5  # >= OpenSSL 3.1.4 for CVE-2023-5363
 jinja2>=3.1.3
 netaddr

admin/requirements-dev.in

@@ -1,15 +1,16 @@
-coverage>=5.0  # #6091
+coverage>=7.0  # #6091
 d2to1
 flaky
 mock
+packaging==21.3
 pbr
 pip>=21.1
 pip-tools>=6.1.0
-py>=1.10.0
-pylint>=2.15.04
-pytest==3.2.0
+py>=1.11.0
+pylint>=3.0.0
+pytest==7.2.0
 requests>=2.26.0
-tox
+tox==3.28.0
 pexpect
 urllib3>=1.26.5
 pytest-catchlog

admin/requirements-testinfra.in

@@ -1,4 +1,4 @@
-pytest==6.1.1
+pytest==7.2.0
 testinfra==5.3.1
 py>=1.10.0
 pytest-xdist==2.1.0

admin/requirements.in

@@ -1,5 +1,6 @@
+cffi>=1.16.0
 markupsafe>=1.1
 prompt_toolkit==2.0.9
-pyyaml>=5.4.1
-setuptools>=56.0.0
+pyyaml>=6.0.1
+setuptools==56.0.0
 six==1.15.0

admin/securedrop_admin/__init__.py

@@ -63,6 +63,29 @@
 I18N_DEFAULT_LOCALES = {"en_US"}
 
 
+# Check OpenSSH version - ansible requires an extra argument for scp on OpenSSH 9
+def openssh_version() -> int:
+    try:
+        result = subprocess.run(["ssh", "-V"], capture_output=True, text=True)
+        if result.stderr.startswith("OpenSSH_9"):
+            return 9
+        elif result.stderr.startswith("OpenSSH_8"):
+            return 8
+        else:
+            return 0
+    except subprocess.CalledProcessError:
+        return 0
+        pass
+    return 0
+
+
+def ansible_command() -> List[str]:
+    cmd = ["ansible-playbook"]
+    if openssh_version() == 9:
+        cmd = ["ansible-playbook", "--scp-extra-args='-O'"]
+    return cmd
+
+
 class FingerprintException(Exception):
     pass
 
@@ -845,7 +868,8 @@ def install_securedrop(args: argparse.Namespace) -> int:
     sdlog.info("You will be prompted for the sudo password on the " "servers.")
     sdlog.info("The sudo password is only necessary during initial " "installation.")
     return subprocess.check_call(
-        [os.path.join(args.ansible_path, "securedrop-prod.yml"), "--ask-become-pass"],
+        ansible_command()
+        + [os.path.join(args.ansible_path, "securedrop-prod.yml"), "--ask-become-pass"],
         cwd=args.ansible_path,
     )
 
@@ -864,11 +888,8 @@ def backup_securedrop(args: argparse.Namespace) -> int:
     Creates a tarball of submissions and server config, and fetches
     back to the Admin Workstation. Future `restore` actions can be performed
     with the backup tarball."""
-    sdlog.info("Backing up the SecureDrop Application Server")
-    ansible_cmd = [
-        "ansible-playbook",
-        os.path.join(args.ansible_path, "securedrop-backup.yml"),
-    ]
+    sdlog.info("Backing up the Sec Application Server")
+    ansible_cmd = ansible_command() + [os.path.join(args.ansible_path, "securedrop-backup.yml")]
     return subprocess.check_call(ansible_cmd, cwd=args.ansible_path)
 
 
@@ -887,8 +908,7 @@ def restore_securedrop(args: argparse.Namespace) -> int:
     # Would like readable output if there's a problem
     os.environ["ANSIBLE_STDOUT_CALLBACK"] = "debug"
 
-    ansible_cmd = [
-        "ansible-playbook",
+    ansible_cmd = ansible_command() + [
         os.path.join(args.ansible_path, "securedrop-restore.yml"),
         "-e",
     ]
@@ -915,7 +935,7 @@ def run_tails_config(args: argparse.Namespace) -> int:
         "You'll be prompted for the temporary Tails admin password,"
         " which was set on Tails login screen"
     )
-    ansible_cmd = [
+    ansible_cmd = ansible_command() + [
         os.path.join(args.ansible_path, "securedrop-tails.yml"),
         "--ask-become-pass",
         # Passing an empty inventory file to override the automatic dynamic
@@ -1077,10 +1097,10 @@ def update(args: argparse.Namespace) -> int:
 def get_logs(args: argparse.Namespace) -> int:
     """Get logs for forensics and debugging purposes"""
     sdlog.info("Gathering logs for forensics and debugging")
-    ansible_cmd = [
-        "ansible-playbook",
+    ansible_cmd = ansible_command() + [
         os.path.join(args.ansible_path, "securedrop-logs.yml"),
     ]
+
     subprocess.check_call(ansible_cmd, cwd=args.ansible_path)
     sdlog.info(
         "Please send the encrypted logs to securedrop@freedom.press or "
@@ -1107,8 +1127,7 @@ def reset_admin_access(args: argparse.Namespace) -> int:
     """Resets SSH access to the SecureDrop servers, locking it to
     this Admin Workstation."""
     sdlog.info("Resetting SSH access to the SecureDrop servers")
-    ansible_cmd = [
-        "ansible-playbook",
+    ansible_cmd = ansible_command() + [
         os.path.join(args.ansible_path, "securedrop-reset-ssh-key.yml"),
     ]
     return subprocess.check_call(ansible_cmd, cwd=args.ansible_path)

admin/setup.cfg

@@ -7,11 +7,11 @@
 name = securedrop-admin
 version = 0.1.0
 summary = SecureDrop Admin Toolkit
-description-file =
+description_file =
     README.rst
 author = Loic Dachary
-author-email = loic@dachary.org
-home-page = https://securedrop.org
+author_email = loic@dachary.org
+home_page = https://securedrop.org
 classifier =
     Environment :: Console
     Intended Audience :: Information Technology

admin/tests/test_securedrop-admin-setup.py

@@ -75,39 +75,39 @@ def test_install_pip_dependencies_fail(self, caplog):
             bootstrap.install_pip_dependencies(args)
         assert "Failed to install" in caplog.text
 
-    def test_python3_buster_venv_deleted_in_bullseye(self, tmpdir, caplog):
+    def test_python3_bullseye_venv_deleted_in_bookworm(self, tmpdir, caplog):
         venv_path = str(tmpdir)
-        python_lib_path = os.path.join(str(tmpdir), "lib/python3.7")
+        python_lib_path = os.path.join(str(tmpdir), "lib/python3.9")
         os.makedirs(python_lib_path)
         with mock.patch("bootstrap.is_tails", return_value=True):
-            with mock.patch("builtins.open", mock.mock_open(read_data='TAILS_VERSION_ID="5.0"')):
+            with mock.patch("builtins.open", mock.mock_open(read_data='VERSION="6.0"')):
                 bootstrap.clean_up_old_tails_venv(venv_path)
-                assert "Tails 4 virtualenv detected." in caplog.text
-                assert "Tails 4 virtualenv deleted." in caplog.text
+                assert "Tails 5 virtualenv detected." in caplog.text
+                assert "Tails 5 virtualenv deleted." in caplog.text
                 assert not os.path.exists(venv_path)
 
-    def test_python3_bullseye_venv_not_deleted_in_bullseye(self, tmpdir, caplog):
+    def test_python3_bookworm_venv_not_deleted_in_bookworm(self, tmpdir, caplog):
         venv_path = str(tmpdir)
-        python_lib_path = os.path.join(venv_path, "lib/python3.9")
+        python_lib_path = os.path.join(venv_path, "lib/python3.11")
         os.makedirs(python_lib_path)
         with mock.patch("bootstrap.is_tails", return_value=True):
-            with mock.patch("subprocess.check_output", return_value="bullseye"):
+            with mock.patch("subprocess.check_output", return_value="bookworm"):
                 bootstrap.clean_up_old_tails_venv(venv_path)
-                assert "Tails 4 virtualenv detected" not in caplog.text
+                assert "Tails 5 virtualenv detected" not in caplog.text
                 assert os.path.exists(venv_path)
 
     def test_python3_buster_venv_not_deleted_in_buster(self, tmpdir, caplog):
         venv_path = str(tmpdir)
-        python_lib_path = os.path.join(venv_path, "lib/python3.7")
+        python_lib_path = os.path.join(venv_path, "lib/python3.9")
         os.makedirs(python_lib_path)
         with mock.patch("bootstrap.is_tails", return_value=True):
-            with mock.patch("subprocess.check_output", return_value="buster"):
+            with mock.patch("subprocess.check_output", return_value="bullseye"):
                 bootstrap.clean_up_old_tails_venv(venv_path)
                 assert os.path.exists(venv_path)
 
     def test_venv_cleanup_subprocess_exception(self, tmpdir, caplog):
         venv_path = str(tmpdir)
-        python_lib_path = os.path.join(venv_path, "lib/python3.7")
+        python_lib_path = os.path.join(venv_path, "lib/python3.9")
         os.makedirs(python_lib_path)
         with mock.patch("bootstrap.is_tails", return_value=True), mock.patch(
             "subprocess.check_output", side_effect=subprocess.CalledProcessError(1, ":o")

admin/tests/test_securedrop-admin.py

@@ -53,6 +53,15 @@ def test_not_verbose(self, capsys):
         assert "HIDDEN" not in out
         assert "VISIBLE" in out
 
+    def test_openssh_detection(self):
+        with mock.patch("securedrop_admin.openssh_version", side_effect=[9]):
+            assert securedrop_admin.ansible_command() == [
+                "ansible-playbook",
+                "--scp-extra-args='-O'",
+            ]
+        with mock.patch("securedrop_admin.openssh_version", side_effect=[8]):
+            assert securedrop_admin.ansible_command() == ["ansible-playbook"]
+
     def test_update_check_decorator_when_no_update_needed(self, caplog):
         """
         When a function decorated with `@update_check_required` is run

admin/tox.ini

@@ -1,6 +1,12 @@
 [tox]
 envlist = pylint,py3
 
+[pytest]
+minversion = 7.2.0
+
+[flaky]
+minversion = 3.6.0
+
 [testenv]
 usedevelop = true
 deps =

install_files/ansible-base/ansible.cfg

@@ -13,5 +13,6 @@ agnostic_become_prompt=False
 
 [ssh_connection]
 scp_if_ssh=True
+ssh_transfer_method=scp
 ssh_args = -o ControlMaster=auto -o ControlPersist=1200 -o ServerAliveInterval=10 -o ServerAliveCountMax=3
 pipelining=True

install_files/ansible-base/callback_plugins/ansible_version_check.py

@@ -17,8 +17,8 @@ class CallbackModule(CallbackBase):
     def __init__(self):
         # The acceptable version range needs to be synchronized with
         # requirements files.
-        viable_start = [2, 11, 0]
-        viable_end = [2, 15, 0]
+        viable_start = [2, 13, 0]
+        viable_end = [2, 15, 10]
         ansible_version = [int(v) for v in ansible.__version__.split(".")]
         if not (viable_start <= ansible_version < viable_end):
             print_red_bold(

install_files/ansible-base/inventory-dynamic

@@ -131,7 +131,7 @@ def host_is_tails():
     within Tails. We don't want to add SSH extra args if True.
     """
     with open("/etc/os-release") as f:
-        return "TAILS_PRODUCT_NAME" in f.read()
+        return "NAME=\"Tails\"" in f.read()
 
 
 def build_inventory():

install_files/ansible-base/roles/tails-config/files/securedrop_init.py

@@ -120,9 +120,9 @@
     )
     subprocess.call(["gio", "set", path_desktop + shortcut, "metadata::trusted", "true"], env=env)
 
-# in Tails 4, reload gnome-shell desktop icons extension to update with changes above
+# in Tails 4 or greater, reload gnome-shell desktop icons extension to update with changes above
 with open("/etc/os-release") as f:
-    is_tails = "TAILS_PRODUCT_NAME" in f.read()
+    is_tails = 'NAME="Tails"' in f.read()
 if is_tails:
     subprocess.call(["gnome-shell-extension-tool", "-r", "desktop-icons@csoriano"], env=env)
 
@@ -165,7 +165,7 @@
     for line in file:
         try:
             k, v = line.strip().split("=")
-            if k == "TAILS_VERSION_ID":
+            if k == "VERSION":
                 tails_current_version = v.strip('"').split(".")
         except ValueError:
             continue