Remote journald services for systemd log collection.

Signed-off-by: Manuel Bluhm <manuel@ssrc.tii.ae>
tiiuae · May 10, 2024 · 1c7f37a · 1c7f37a
1 parent 388ebb9
commit 1c7f37a
Show file tree

Hide file tree

Showing 7 changed files with 229 additions and 0 deletions.
diff --git a/modules/common/systemd/base.nix b/modules/common/systemd/base.nix
@@ -37,6 +37,7 @@
         withOomd = true;
         withPam = true;
         inherit (cfg) withPolkit;
+        withRemote = cfg.withRemoteJournal.enable || cfg.withRemoteJournalServer.enable;
         inherit (cfg) withResolved;
         inherit (cfg) withRepart;
         withShellCompletions = cfg.withDebug;
@@ -171,6 +172,12 @@
     ]);
 in
   with lib; {
+
+    imports = [
+      ./debug/journal-upload.nix
+      ./debug/journal-remote.nix
+    ];
+
     options.ghaf.systemd = {
       enable = mkEnableOption "Enable minimal systemd configuration.";
 

diff --git a/modules/common/systemd/debug/journal-remote.nix b/modules/common/systemd/debug/journal-remote.nix
@@ -0,0 +1,103 @@
+# Copyright 2022-2024 TII (SSRC) and the Ghaf contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Original source: https://gitlab.com/distrosync/nixos/-/blob/master/modules/journal/journal-remote.nix
+# Run this on your development machine, or a virtual machine to collect logs.
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}: let
+  # Ghaf configuration flag
+  cfg = config.ghaf.systemd.withRemoteJournalServer;
+in
+ with lib; {
+  options.ghaf.systemd.withRemoteJournalServer = {
+    enable = mkOption {
+      description = ''
+        Enable remote journaling server for systemd debugging. Note that this option uses
+        insecure http and is only intended for local debugging purposes.
+      '';
+      type = types.bool;
+      default = false;
+    };
+  };
+
+  config = mkIf cfg.enable {
+
+    assertions = [
+      {
+        assertion = config.ghaf.systemd.withJournal;
+        message = ''
+          The systemd journal must be enabled when enabling systemd-journal-upload.
+          Hint: Set `ghaf.systemd.withJournal` to true.
+        '';
+      }
+      {
+        assertion = !config.ghaf.profiles.release.enable;
+        message = ''
+          This module should never by used in release.
+        '';
+      }
+    ];
+
+    # This configuration is adapted from this service file example:
+    # /run/current-system/systemd/example/systemd/system/systemd-journal-remote.service
+
+    # To allow journal-upload clients to access the journal-remote listener
+    networking.firewall.allowedTCPPorts = [ 19532 ];
+    # Create a new user for journal-remote, and use existing systemd-journal group
+    users.users.systemd-journal-remote = {
+      isSystemUser = true;
+      group = "systemd-journal";
+    };
+    # Create a directory for the remote logs, so that they inherit the ACLs of the parent /var/log/journal directory.
+    # This is probably not necessary, but it is part of trying to debug journald configs not applying to remote journal files.
+    systemd.tmpfiles.rules = [ "d /var/log/journal/remote 755 systemd-journal-remote systemd-journal" ];
+
+    systemd.services.systemd-journal-remote = {
+      enable = true;
+      description = "Journal Remote Sink Service";
+      documentation = [ "man:systemd-journal-remote(8)" "man:journal-remote.conf(5)" ];
+      requires = [ "systemd-journal-remote.socket" ];
+
+      serviceConfig = {
+        ExecStart = "/run/current-system/systemd/lib/systemd/systemd-journal-remote --listen-http=-3 --output=/var/log/journal/remote/";
+        LockPersonality = "yes";
+        LogsDirectory = "journal/remote";
+        MemoryDenyWriteExecute = "yes";
+        NoNewPrivileges = "yes";
+        PrivateDevices = "yes";
+        PrivateNetwork = "yes";
+        PrivateTmp = "yes";
+        ProtectControlGroups = "yes";
+        ProtectHome = "yes";
+        ProtectHostname = "yes";
+        ProtectKernelModules = "yes";
+        ProtectKernelTunables = "yes";
+        ProtectSystem = "strict";
+        RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6";
+        RestrictNamespaces = "yes";
+        RestrictRealtime = "yes";
+        RestrictSUIDSGID = "yes";
+        SystemCallArchitectures = "native";
+        User = "systemd-journal-remote";
+        Group = "systemd-journal";
+        WatchdogSec = "10";
+        # If there are many split up journal files we need a lot of fds to access them all in parallel.
+        LimitNOFILE = "524288";
+      };
+      # Added so that the service will start automatically.
+      # Possibly also add a "Restart" to the serviceConfig if it doesn't recover from failures.
+      wantedBy = [ "multi-user.target" ];
+    };
+
+    systemd.sockets.systemd-journal-remote = {
+      enable = true;
+      description = "Journal Remote Sink Socket";
+      listenStreams = [ "19532" ];
+      wantedBy = [ "sockets.target" ];
+    };
+  };
+}
diff --git a/modules/common/systemd/debug/journal-upload.nix b/modules/common/systemd/debug/journal-upload.nix
@@ -0,0 +1,112 @@
+# Copyright 2022-2024 TII (SSRC) and the Ghaf contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Original configuration source: https://gitlab.com/distrosync/nixos/-/blob/master/modules/journal/journal-upload.nix
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}: let
+  # Ghaf configuration flag
+  cfg = config.ghaf.systemd.withRemoteJournal;
+in
+ with lib; {
+  options.ghaf.systemd.withRemoteJournal = {
+    enable = mkOption {
+      description = ''
+        Enable remote journaling for systemd debugging. Note that this option uses
+        insecure http and is only intended for local debugging purposes.
+      '';
+      type = types.bool;
+      default = false;
+    };
+    debugServerIpv4 = mkOption {
+      description = "The IPv4 address of the debug server to which the journal should be uploaded.";
+      type = types.str;
+      default = "192.168.101.1";
+      example = "192.168.101.1";
+    };
+  };
+
+  config = mkIf cfg.enable {
+
+    assertions = [
+      {
+        assertion = config.ghaf.systemd.withJournal;
+        message = ''
+          The systemd journal must be enabled when enabling systemd-journal-upload.
+          Hint: Set `ghaf.systemd.withJournal` to true.
+        '';
+      }
+      {
+        assertion = cfg.debugServerIpv4 != "";
+        message = ''
+          The debug server IP address must be set when enabling systemd-journal-upload.
+          Hint: Set `ghaf.systemd.withRemoteJournal.debugServerIpv4` to the IP address of the debug server.
+        '';
+      }
+      {
+        assertion = !config.ghaf.profiles.release.enable;
+        message = ''
+          This module should never by used in release.
+        '';
+      }
+    ];
+
+    # Systemd >= 255 / unstable implementation
+    # services.journald.upload = {
+    #   enable = true;
+    #   settings = {
+    #     Upload.URL = "http://${cfg.debugServerIpv4}:19532";
+    #   };
+    # };
+
+    users = {
+      users.systemd-journal-upload = {
+        isSystemUser = true;
+        group = "systemd-journal-upload";
+      };
+      groups.systemd-journal-upload = {};
+    };
+
+    systemd.services.systemd-journal-upload = {
+      enable = true;
+      description = "Journal Remote Upload Service";
+      documentation = [ "man:systemd-journal-upload(8)" ];
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        ExecStart = "/run/current-system/systemd/lib/systemd/systemd-journal-upload --save-state -u http://${cfg.debugServerIpv4}:19532";
+        DynamicUser = "yes";
+        LockPersonality = "yes";
+        MemoryDenyWriteExecute = "yes";
+        PrivateDevices = "yes";
+        ProtectControlGroups = "yes";
+        ProtectHome = "yes";
+        ProtectHostname = "yes";
+        ProtectKernelModules = "yes";
+        ProtectKernelTunables = "yes";
+        Restart = "always";
+        RestartSec = "10";
+        RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6";
+        RestrictNamespaces = "yes";
+        RestrictRealtime = "yes";
+        StateDirectory = "systemd/journal-upload";
+        SupplementaryGroups = "systemd-journal";
+        SystemCallArchitectures = "native";
+        User = "systemd-journal-upload";
+        WatchdogSec = "10";
+        # If there are many split up journal files we need a lot of fds to access them all in parallel.
+        LimitNOFILE = "524288";
+      };
+    };
+
+    # Add route to enable remote logging (for debugging only)
+    systemd.network.networks."10-virbr0".routes =
+      if (config.system.name == "ghaf-host") then
+        [{ routeConfig.Gateway = "192.168.101.1"; }]
+      else [];
+  };
+}
diff --git a/modules/microvm/virtualization/microvm/appvm.nix b/modules/microvm/virtualization/microvm/appvm.nix
@@ -60,6 +60,10 @@
               withResolved = true;
               withPolkit = true;
               withDebug = configHost.ghaf.profiles.debug.enable;
+              withRemoteJournal = {
+                enable = true;
+                debugServerIpv4 = "192.168.100.1";
+              };
             };
           };
 

diff --git a/modules/microvm/virtualization/microvm/guivm.nix b/modules/microvm/virtualization/microvm/guivm.nix
@@ -37,6 +37,7 @@
             withResolved = true;
             withTimesyncd = true;
             withDebug = configHost.ghaf.profiles.debug.enable;
+            withRemoteJournal.enable = true;
           };
         };
 

diff --git a/modules/microvm/virtualization/microvm/microvm-host.nix b/modules/microvm/virtualization/microvm/microvm-host.nix
@@ -30,6 +30,7 @@ in
         withResolved = cfg.networkSupport;
         withSerial = config.ghaf.profiles.debug.enable;
         withDebug = config.ghaf.profiles.debug.enable;
+        withRemoteJournal.enable = true;
       };
     };
   }
diff --git a/modules/microvm/virtualization/microvm/netvm.nix b/modules/microvm/virtualization/microvm/netvm.nix
@@ -35,6 +35,7 @@
             withName = "netvm-systemd";
             withPolkit = true;
             withDebug = configHost.ghaf.profiles.debug.enable;
+            withRemoteJournalServer.enable = true;
           };
         };