Add integ tests for GDB and a flag for the sandbox tool

tests/framework/microvm.py

@@ -287,6 +287,8 @@ def __init__(
 
         self.help = MicrovmHelpers(self)
 
+        self.gdb_socket = None
+
     def __repr__(self):
         return f"<Microvm id={self.id}>"
 
@@ -1198,6 +1200,11 @@ def wait_for_ssh_up(self):
         # run commands. The actual connection retry loop happens in SSHConnection._init_connection
         _ = self.ssh_iface(0)
 
+    def enable_gdb(self):
+        """Enables GDB debugging"""
+        self.gdb_socket = "gdb.socket"
+        self.api.machine_config.patch(gdb_socket_path=self.gdb_socket)
+
 
 class MicroVMFactory:
     """MicroVM factory"""

tests/framework/microvm_helpers.py

@@ -7,6 +7,7 @@
 import os
 import platform
 import subprocess
+import tempfile
 from pathlib import Path
 
 
@@ -245,3 +246,30 @@ def trace_cmd_guest(self, fns, cmd, port=4321):
             f"trace-cmd record -N {host_ip}:{port} -p function {' '.join(fns)} {cmd}"
         )
         return list(Path(".").glob("trace.*.dat"))
+
+    def tmux_gdb(self):
+        """Run GDB on a new tmux window"""
+        chroot_gdb_socket = Path(self.vm.jailer.chroot_path(), self.vm.gdb_socket)
+
+        with tempfile.NamedTemporaryFile(
+            mode="w", suffix=".gdb", delete=False, prefix="fc_gdb_"
+        ) as f:
+            f.write(
+                f"""
+                target remote {chroot_gdb_socket}
+                directory resources/linux
+                hbreak start_kernel
+                continue
+            """
+            )
+            gdb_script = f.name
+
+        self.tmux_neww(
+            f"""
+            until [ -S {chroot_gdb_socket} ]; do
+                echo 'waiting for {chroot_gdb_socket}';
+                sleep 1;
+            done;
+            gdb {self.vm.kernel_file} -x {gdb_script}
+            """
+        )

tests/integration_tests/build/test_gdb.py

@@ -2,15 +2,91 @@
 # SPDX-License-Identifier: Apache-2.0
 """A test that ensures that firecracker builds with GDB feature enabled at integration time."""
 
+import os
 import platform
+import signal
+import subprocess
+import tempfile
+from pathlib import Path
 
 import host_tools.cargo_build as host
+import pytest
+from framework.defs import LOCAL_BUILD_PATH
+from framework.microvm import MicroVMFactory
 
 MACHINE = platform.machine()
 TARGET = "{}-unknown-linux-musl".format(MACHINE)
+BUILD_PATH = LOCAL_BUILD_PATH / "gdb"
+
+
+def build_gdb():
+    """Builds Firecracker with GDB feature enabled"""
+
+    host.cargo(
+        "build",
+        f"--features gdb --target {TARGET} --all",
+        env={"CARGO_TARGET_DIR": BUILD_PATH},
+    )
 
 
 def test_gdb_compiles():
     """Checks that Firecracker compiles with GDB enabled"""
 
-    host.cargo("build", f"--features gdb --target {TARGET}")
+    build_gdb()
+
+
+@pytest.mark.skipif(
+    platform.machine() != "x86_64",
+    reason="GDB requires a vmlinux but we ship a uImage for ARM in our CI",
+)
+def test_gdb_connects(guest_kernel_linux_6_1, rootfs):
+    """Checks that GDB works in a FC VM"""
+
+    build_gdb()
+
+    vmfcty = MicroVMFactory(BUILD_PATH / TARGET / "debug")
+    kernel_dbg = guest_kernel_linux_6_1.parent / "debug" / guest_kernel_linux_6_1.name
+    uvm = vmfcty.build(kernel_dbg, rootfs)
+    uvm.spawn(validate_api=False)
+    uvm.add_net_iface()
+    uvm.basic_config()
+    uvm.enable_gdb()
+
+    chroot_gdb_socket = Path(uvm.jailer.chroot_path(), uvm.gdb_socket)
+
+    gdb_commands = f"""
+    target remote {chroot_gdb_socket}
+    hbreak start_kernel
+    # continue to start_kernel
+    continue
+    # continue boot until interrupted
+    continue
+    """
+
+    with tempfile.NamedTemporaryFile(
+        mode="w", suffix=".gdb", delete=False, prefix="fc_gdb_"
+    ) as f:
+        f.write(gdb_commands)
+        gdb_script = f.name
+
+    gdb_proc = subprocess.Popen(
+        f"""
+        until [ -S {chroot_gdb_socket} ]; do
+            echo 'waiting for {chroot_gdb_socket}';
+            sleep 1;
+        done;
+        gdb {kernel_dbg} -batch -x {gdb_script}
+        """,
+        shell=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    uvm.start()
+    os.kill(uvm.firecracker_pid, signal.SIGKILL)
+    gdb_proc.terminate()
+    uvm.mark_killed()
+    stdout, stderr = gdb_proc.communicate(timeout=10)
+    assert (
+        "hit Breakpoint 1, start_kernel" in stdout
+    ), f"Breakpoint wasn't hit:\nstdout:\n{stdout}\n\nstderr:\n{stderr}"

tools/devtool

@@ -944,7 +944,7 @@ cmd_test_debug() {
 cmd_fmt() {
     cmd_sh "cargo fmt --all -- --config $(tr '\n' ',' <tests/fmt.toml)"
     cmd_sh "cargo sort"
-    cmd_sh "cd tests; black . ../tools ../.buildkite"
+    cmd_sh "cd tests; black --config pyproject.toml . ../tools ../.buildkite"
     cmd_sh "cd tests; isort . ../tools ../.buildkite"
     cmd_sh "mdformat $(git ls-files '*.md' | tr '\n' ' ')"
 }

tools/sandbox.py

@@ -10,11 +10,13 @@
 
 import argparse
 import json
+import platform
 import re
 from pathlib import Path
 
+import host_tools.cargo_build as build_tools
 from framework.artifacts import disks, kernels
-from framework.defs import DEFAULT_BINARY_DIR
+from framework.defs import DEFAULT_BINARY_DIR, LOCAL_BUILD_PATH
 from framework.microvm import MicroVMFactory
 
 kernels = list(kernels("vmlinux-*"))
@@ -58,12 +60,31 @@ def parse_byte_size(param):
 parser.add_argument("--rootfs-size", type=parse_byte_size, default=1 * 2**30)  # 1GB
 parser.add_argument("--binary-dir", help="Path to the firecracker binaries")
 parser.add_argument("--cpu-template-path", help="CPU template to use", type=Path)
+parser.add_argument(
+    "--debug", action="store_true", default=False, help="Use debug kernel"
+)
+parser.add_argument(
+    "--gdb", action="store_true", default=False, help="Connect to Firecracker guest GDB"
+)
 args = parser.parse_args()
 print(args)
 
 binary_dir = None
 if args.binary_dir:
     binary_dir = Path(args.binary_dir).resolve()
+elif args.gdb:
+    # Build Firecracker with GDB feature if needed
+    print("Building Firecracker with GDB feature...")
+    machine = platform.machine()
+    target = f"{machine}-unknown-linux-musl"
+    build_dir = LOCAL_BUILD_PATH / "gdb"
+    build_tools.cargo(
+        "build",
+        f"--features gdb --target {target} --all",
+        env={"CARGO_TARGET_DIR": build_dir},
+    )
+    print("Build complete!")
+    binary_dir = build_dir / target / "debug"
 else:
     binary_dir = DEFAULT_BINARY_DIR
 
@@ -72,28 +93,26 @@ def parse_byte_size(param):
     cpu_template = json.loads(args.cpu_template_path.read_text())
 vmfcty = MicroVMFactory(binary_dir)
 
-print(f"uvm with kernel {args.kernel} ...")
-uvm = vmfcty.build(args.kernel, args.rootfs)
+if args.debug or args.gdb:
+    kernel = args.kernel.parent / "debug" / args.kernel.name
+else:
+    kernel = args.kernel
+
+print(f"uvm with kernel {kernel} ...")
+uvm = vmfcty.build(kernel, args.rootfs)
 uvm.help.enable_console()
 uvm.help.resize_disk(uvm.rootfs_file, args.rootfs_size)
-uvm.spawn(log_show_level=True)
+uvm.spawn(log_show_level=True, validate_api=False)
 uvm.help.print_log()
 uvm.add_net_iface()
 uvm.basic_config(vcpu_count=args.vcpus, mem_size_mib=args.guest_mem_size // 2**20)
 if cpu_template is not None:
     uvm.api.cpu_config.put(**cpu_template)
     print(cpu_template)
+
+if args.gdb:
+    uvm.enable_gdb()
+    uvm.help.tmux_gdb()
+
 uvm.start()
 uvm.get_all_metrics()
-
-kernel_dbg_dir = args.kernel.parent / "debug"
-kernel_dbg = kernel_dbg_dir / args.kernel.name
-print(f"uvm2 with kernel {kernel_dbg} ...")
-uvm2 = vmfcty.build(kernel_dbg, args.rootfs)
-uvm2.spawn()
-uvm2.add_net_iface()
-uvm2.basic_config(vcpu_count=args.vcpus, mem_size_mib=args.guest_mem_size // 2**20)
-uvm2.start()
-# trace-cmd needs this (DNS resolution?)
-uvm2.help.enable_ip_forwarding()
-files = uvm2.help.trace_cmd_guest(["-l", "read_msr"], cmd="sleep 5")