feat: implement logs persistence

Implement a log persistence controller, rotate logs and bufferize writes.

Fixes #11461

Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>
Co-authored-by: Dmitrii Sharshakov <dmitry.sharshakov@siderolabs.com>
Signed-off-by: Dmitrii Sharshakov <dmitry.sharshakov@siderolabs.com>
(cherry picked from commit a0cfc35)

Author: 2 people

hack/release.toml

@@ -209,6 +209,13 @@ You can still expand the list of supported cipher suites via the `cluster.apiSer
         title = "Kernel Log"
         description = """\
 The kernel log (dmesg) is now also available as the service log named `kernel` (`talosctl logs kernel`).
+"""
+
+    [notes.persistent-logs]
+        title = "Persistent logs"
+        description = """\
+Talos now stores system component logs in /var/log, featuring automatic log rotation and keeping two most
+recent log files. This change allows collecting logs from Talos like on any other Linux system.
 """
 
 [make_deps]

internal/app/machined/pkg/controllers/block/volume_config_test.go

@@ -162,7 +162,7 @@ func (suite *VolumeConfigSuite) TestReconcileDefaults() {
 	})
 
 	ctest.AssertResources(suite, []resource.ID{
-		"/var/log",
+		constants.LogMountPoint,
 		"/var/log/audit",
 		"/var/log/containers",
 		"/var/log/pods",

internal/app/machined/pkg/controllers/runtime/internal/logfile/logfile.go

@@ -0,0 +1,120 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// Package logfile implements a buffered, rotating log file.
+package logfile
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"sync"
+)
+
+// LogFile is an implementation of a buffered and rotated log file.
+type LogFile struct {
+	mu   sync.Mutex
+	file *os.File
+	buf  bufio.Writer
+
+	path              string
+	size              int64
+	rotationThreshold int64
+}
+
+// NewLogFile creates a LogFile.
+func NewLogFile(path string, rotationThreshold int64) *LogFile {
+	return &LogFile{
+		path:              path,
+		rotationThreshold: rotationThreshold,
+	}
+}
+
+// Write appends a line to the end of file, handling file creation and rotation.
+func (lf *LogFile) Write(line []byte) error {
+	var err error
+
+	lf.mu.Lock()
+	defer lf.mu.Unlock()
+
+	if lf.file == nil {
+		lf.file, err = os.OpenFile(lf.path, os.O_CREATE|os.O_WRONLY, 0o640)
+		if err != nil {
+			return fmt.Errorf("error opening log file %q: %w", lf.path, err)
+		}
+
+		lf.size, err = lf.file.Seek(0, io.SeekEnd)
+		if err != nil {
+			return fmt.Errorf("error determining log file %q length: %w", lf.path, err)
+		}
+
+		lf.buf.Reset(lf.file)
+	}
+
+	var n int
+	if n, err = lf.buf.Write(append(line, '\n')); err != nil {
+		return fmt.Errorf("error writing log line to file %q: %w", lf.path, err)
+	}
+
+	lf.size += int64(n)
+	if lf.size < lf.rotationThreshold {
+		return nil
+	}
+
+	if err = lf.close(); err != nil {
+		return err
+	}
+
+	if err = os.Rename(lf.path, lf.path+".1"); err != nil {
+		return fmt.Errorf("error renaming log file %q: %w", lf.path, err)
+	}
+
+	return nil
+}
+
+func (lf *LogFile) flush() error {
+	if err := lf.buf.Flush(); err != nil {
+		return fmt.Errorf("failed to flush log file %s buffer: %w", lf.path, err)
+	}
+
+	return nil
+}
+
+// Flush flushes the internal buffer to persist data to the filesystem.
+func (lf *LogFile) Flush() error {
+	lf.mu.Lock()
+	defer lf.mu.Unlock()
+
+	return lf.flush()
+}
+
+func (lf *LogFile) close() error {
+	if err := lf.flush(); err != nil {
+		return err
+	}
+
+	lf.buf.Reset(nil)
+
+	if lf.file == nil {
+		return nil
+	}
+
+	err := lf.file.Close()
+	lf.file = nil
+
+	if err != nil {
+		return fmt.Errorf("failed to close log file %s: %w", lf.path, err)
+	}
+
+	return nil
+}
+
+// Close flushes and closes the underlying file.
+func (lf *LogFile) Close() error {
+	lf.mu.Lock()
+	defer lf.mu.Unlock()
+
+	return lf.close()
+}

internal/app/machined/pkg/controllers/runtime/internal/logfile/logfile_test.go

@@ -0,0 +1,238 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+package logfile_test
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/siderolabs/talos/internal/app/machined/pkg/controllers/runtime/internal/logfile"
+)
+
+func TestWrite(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.log")
+
+	lf := logfile.NewLogFile(path, 1024)
+	defer require.NoError(t, lf.Close())
+
+	err := lf.Write([]byte("hello world"))
+	require.NoError(t, err)
+
+	// Expect write to retain data in the buffer
+	st, err := os.Stat(path)
+	require.NoError(t, err)
+	require.Equal(t, int64(0), st.Size(), "file should be empty before flush")
+	require.NoError(t, lf.Flush())
+
+	// After flush, check the data got written to the file
+	content, err := os.ReadFile(path)
+	require.NoError(t, err)
+	require.Equal(t, "hello world\n", string(content))
+}
+
+func TestWriteMultipleLines(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.log")
+
+	lf := logfile.NewLogFile(path, 1024)
+	defer require.NoError(t, lf.Close())
+
+	lines := []string{"line1", "line2", "line3"}
+	for _, line := range lines {
+		require.NoError(t, lf.Write([]byte(line)))
+	}
+
+	require.NoError(t, lf.Flush())
+
+	content, err := os.ReadFile(path)
+	require.NoError(t, err)
+	require.Equal(t, "line1\nline2\nline3\n", string(content))
+}
+
+func TestLogRotation(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.log")
+	expectedRotatedPath := path + ".1"
+
+	lf := logfile.NewLogFile(path, 50)
+	defer require.NoError(t, lf.Close())
+
+	// We write 4 lines (indices 0-3)
+	// expecting 0-2 to be written before rotation and 3 after rotation
+	for i := range 4 {
+		line := []byte("_20_character_line_" + strconv.Itoa(i))
+		require.NoError(t, lf.Write(line))
+	}
+
+	_, err := os.Stat(expectedRotatedPath)
+	require.NoError(t, err)
+
+	// Verify the rotated file contains the written data
+	rotatedContent, err := os.ReadFile(expectedRotatedPath)
+	require.NoError(t, err)
+	require.Len(t, rotatedContent, 63)
+	require.Contains(t, string(rotatedContent), "_20_character_line_2")
+	require.NoError(t, lf.Flush())
+
+	currentContent, err := os.ReadFile(path)
+	require.NoError(t, err)
+	require.Len(t, currentContent, 21)
+	require.Contains(t, string(currentContent), "_20_character_line_3")
+}
+
+func TestLogRotationMultipleTimes(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.log")
+	rotatedPath := path + ".1"
+
+	lf := logfile.NewLogFile(path, 40)
+	defer require.NoError(t, lf.Close())
+
+	for i := range 10 {
+		line := []byte("_20_character_line_" + strconv.Itoa(i))
+		require.NoError(t, lf.Write(line))
+	}
+
+	// Rotated file should exist and contain most recent events before the current
+	rotatedContent, err := os.ReadFile(rotatedPath)
+	require.NoError(t, err)
+	require.Len(t, rotatedContent, 42)
+	require.Contains(t, string(rotatedContent), "_20_character_line_8")
+	require.Contains(t, string(rotatedContent), "_20_character_line_9")
+}
+
+func TestFlushWithoutFile(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.log")
+
+	lf := logfile.NewLogFile(path, 1024)
+	defer require.NoError(t, lf.Close())
+
+	require.NoError(t, lf.Flush())
+}
+
+func TestClose(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.log")
+
+	lf := logfile.NewLogFile(path, 1024)
+
+	require.NoError(t, lf.Write([]byte("data")))
+
+	err := lf.Close()
+	require.NoError(t, err)
+
+	// Expect Close to have flushed the buffer
+	content, err := os.ReadFile(path)
+	require.NoError(t, err)
+	require.Contains(t, string(content), "data")
+}
+
+func TestCloseWithoutWrite(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.log")
+
+	lf := logfile.NewLogFile(path, 1024)
+	require.NoError(t, lf.Close())
+}
+
+func TestConcurrentWrites(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.log")
+
+	// Do not rotate while the test runs
+	lf := logfile.NewLogFile(path, 100000)
+	defer require.NoError(t, lf.Close())
+
+	var wg sync.WaitGroup
+
+	numGoroutines := 10
+	writesPerGoroutine := 100
+
+	for range numGoroutines {
+		wg.Go(func() {
+			for range writesPerGoroutine {
+				require.NoError(t, lf.Write([]byte("goroutine write")))
+			}
+		})
+	}
+
+	wg.Wait()
+
+	require.NoError(t, lf.Flush())
+
+	content, err := os.ReadFile(path)
+	require.NoError(t, err)
+
+	// Count lines to verify all writes succeeded
+	lineCount := bytes.Count(content, []byte("\n"))
+	expectedLines := numGoroutines * writesPerGoroutine
+	require.Equal(t, expectedLines, lineCount)
+}
+
+func TestConcurrentWriteAndFlush(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.log")
+
+	lf := logfile.NewLogFile(path, 10000)
+	defer require.NoError(t, lf.Close())
+
+	var wg sync.WaitGroup
+
+	// Writer goroutines
+	for range 5 {
+		wg.Go(func() {
+			for range 50 {
+				require.NoError(t, lf.Write([]byte("concurrent data")))
+			}
+		})
+	}
+
+	// Flusher goroutines
+	for range 3 {
+		wg.Go(func() {
+			for range 10 {
+				require.NoError(t, lf.Flush())
+			}
+		})
+	}
+
+	wg.Wait()
+
+	require.NoError(t, lf.Flush())
+}
+
+func TestConcurrentWritesWithRotation(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.log")
+
+	// Small threshold to trigger rotation during concurrent writes
+	lf := logfile.NewLogFile(path, 100)
+	defer require.NoError(t, lf.Close())
+
+	var wg sync.WaitGroup
+
+	numGoroutines := 5
+	writesPerGoroutine := 50
+
+	for range numGoroutines {
+		wg.Go(func() {
+			for range writesPerGoroutine {
+				require.NoError(t, lf.Write([]byte("rotation test line")))
+			}
+		})
+	}
+
+	wg.Wait()
+
+	_, err := os.Stat(path + ".1")
+	require.NoError(t, err)
+}