feat: present kernel log as talosctl logs kernel

Extracted from #12115

The idea is that kernel log can be delivered/persisted along with any
other service logs.

Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>
(cherry picked from commit e715f38)

Author: smira

hack/release.toml

@@ -203,6 +203,12 @@ The Kubernetes API server in Talos has been updated to use a more secure set of
 This is in line with a set of best practices documented in CIS 1.12 benchmark.
 
 You can still expand the list of supported cipher suites via the `cluster.apiServer.extraArgs."tls-cipher-suites"` machine configuration field if needed.
+"""
+
+    [notes.kernel-log]
+        title = "Kernel Log"
+        description = """\
+The kernel log (dmesg) is now also available as the service log named `kernel` (`talosctl logs kernel`).
 """
 
 [make_deps]

internal/app/machined/pkg/controllers/runtime/kmsg_log_storage.go

@@ -0,0 +1,92 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package runtime
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/cosi-project/runtime/pkg/controller"
+	"github.com/siderolabs/go-kmsg"
+	"go.uber.org/zap"
+
+	"github.com/siderolabs/talos/internal/app/machined/pkg/runtime"
+)
+
+// KmsgLogStorageController presents kernel message log as a 'kernel' log.
+type KmsgLogStorageController struct {
+	V1Alpha1Logging runtime.LoggingManager
+	V1Alpha1Mode    runtime.Mode
+}
+
+// Name implements controller.Controller interface.
+func (ctrl *KmsgLogStorageController) Name() string {
+	return "runtime.KmsgLogStorageController"
+}
+
+// Inputs implements controller.Controller interface.
+func (ctrl *KmsgLogStorageController) Inputs() []controller.Input {
+	return nil
+}
+
+// Outputs implements controller.Controller interface.
+func (ctrl *KmsgLogStorageController) Outputs() []controller.Output {
+	return nil
+}
+
+// Run implements controller.Controller interface.
+//
+//nolint:gocyclo
+func (ctrl *KmsgLogStorageController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error {
+	if ctrl.V1Alpha1Mode.InContainer() {
+		return nil
+	}
+
+	var err error
+
+	logWriter, err := ctrl.V1Alpha1Logging.ServiceLog("kernel").Writer()
+	if err != nil {
+		return fmt.Errorf("error opening logger: %w", err)
+	}
+	defer logWriter.Close() //nolint:errcheck
+
+	// initilalize kmsg reader early, so that we don't lose position on config changes
+	reader, err := kmsg.NewReader(kmsg.Follow())
+	if err != nil {
+		return fmt.Errorf("error reading kernel messages: %w", err)
+	}
+
+	defer reader.Close() //nolint:errcheck
+
+	kmsgCh := reader.Scan(ctx)
+
+	// wait for the initial event to start processing messages
+	select {
+	case <-ctx.Done():
+		return nil
+	case <-r.EventCh():
+	}
+
+	for {
+		var msg kmsg.Packet
+
+		select {
+		case <-ctx.Done():
+			return nil
+		case msg = <-kmsgCh:
+		}
+
+		if msg.Err != nil {
+			return fmt.Errorf("error receiving kernel logs: %w", msg.Err)
+		}
+
+		if _, err = logWriter.Write(
+			fmt.Appendf(nil, "%s: %7s: [%s]: %s", msg.Message.Facility, msg.Message.Priority, msg.Message.Timestamp.Format(time.RFC3339Nano), msg.Message.Message),
+		); err != nil {
+			return err
+		}
+	}
+}

internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go

@@ -423,6 +423,10 @@ func (ctrl *Controller) Run(ctx context.Context, drainer *runtime.Drainer) error
 		&runtimecontrollers.KmsgLogDeliveryController{
 			Drainer: drainer,
 		},
+		&runtimecontrollers.KmsgLogStorageController{
+			V1Alpha1Logging: ctrl.v1alpha1Runtime.Logging(),
+			V1Alpha1Mode:    ctrl.v1alpha1Runtime.State().Platform().Mode(),
+		},
 		&runtimecontrollers.LoadedKernelModuleController{
 			V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(),
 		},

internal/integration/api/logs.go

@@ -10,9 +10,12 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"fmt"
 	"io"
+	"strings"
 	"time"
 
+	"github.com/siderolabs/talos/internal/app/machined/pkg/runtime/logging"
 	"github.com/siderolabs/talos/internal/integration/base"
 	"github.com/siderolabs/talos/pkg/machinery/api/common"
 	"github.com/siderolabs/talos/pkg/machinery/client"
@@ -108,6 +111,46 @@ func (suite *LogsSuite) TestAuditdLogs() {
 	suite.Require().Greater(n, int64(1024))
 }
 
+// TestKernelLogs verifies that kernel logs are present and valid.
+func (suite *LogsSuite) TestKernelLogs() {
+	if suite.Cluster == nil || suite.Cluster.Provisioner() != base.ProvisionerQEMU {
+		suite.T().Skip("skip kernel logs test for non-QEMU clusters")
+	}
+
+	logsStream, err := suite.Client.Logs(
+		suite.nodeCtx,
+		constants.SystemContainerdNamespace,
+		common.ContainerDriver_CONTAINERD,
+		"kernel",
+		false,
+		-1,
+	)
+	suite.Require().NoError(err)
+
+	logReader, err := client.ReadStream(logsStream)
+	suite.Require().NoError(err)
+
+	var buf bytes.Buffer
+
+	n, err := io.Copy(&buf, logReader)
+	suite.Require().NoError(err)
+
+	// kernel logs shouldn't be empty
+	suite.Require().Greater(n, int64(1024))
+
+	version := strings.Contains(buf.String(), fmt.Sprintf("Linux version %s", constants.DefaultKernelVersion))
+	overflow := n >= logging.DesiredCapacity/2
+
+	if !version && overflow {
+		suite.T().Skip("skipping the test as kernel log buffer might have overflowed")
+	}
+
+	suite.Require().Truef(
+		version,
+		"did not find kernel version (%s) in the log buffer", constants.DefaultKernelVersion,
+	)
+}
+
 // TestTail verifies that log tail might be requested.
 func (suite *LogsSuite) TestTail() {
 	// invoke machined enough times to generate