diff --git a/go.mod b/go.mod index 81e2b0b8f9..80cc14fa62 100644 --- a/go.mod +++ b/go.mod @@ -78,6 +78,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/hetznercloud/hcloud-go/v2 v2.6.0 github.com/insomniacslk/dhcp v0.0.0-20240204152450-ca2dc33955c1 + github.com/jeromer/syslogparser v1.1.0 github.com/jsimonetti/rtnetlink v1.4.1 github.com/jxskiss/base62 v1.1.0 github.com/martinlindhe/base36 v1.1.1 diff --git a/go.sum b/go.sum index 5127eca486..a1b231e6ad 100644 --- a/go.sum +++ b/go.sum @@ -439,6 +439,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/insomniacslk/dhcp v0.0.0-20240204152450-ca2dc33955c1 h1:L3pm9Kf2G6gJVYawz2SrI5QnV1wzHYbqmKnSHHXJAb8= github.com/insomniacslk/dhcp v0.0.0-20240204152450-ca2dc33955c1/go.mod h1:izxuNQZeFrbx2nK2fAyN5iNUB34Fe9j0nK4PwLzAkKw= +github.com/jeromer/syslogparser v1.1.0 h1:HES0EviO9iPvCu56LjVFVhbM3o0BckDlIbQfkkaRJAw= +github.com/jeromer/syslogparser v1.1.0/go.mod h1:zfowyus/j2SEgW31bIntTvEBE2zCSndtFsCC6NcW4S4= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= diff --git a/hack/release.toml b/hack/release.toml index 5712801bf4..7bbbcfb4b7 100644 --- a/hack/release.toml +++ b/hack/release.toml @@ -122,6 +122,16 @@ machine: kubespan: harvestExtraEndpoints: true ``` +""" + + [notes.syslog] + title = "Syslog" + description = """\ +Talos Linux now starts a basic syslog receiver listening on `/dev/log`. +The receiver can mostly parse both RFC3164 and RFC5424 messages and writes them as JSON formatted message. +The logs can be viewed via `talosctl logs syslogd`. + +This is mostly implemented for extension services that log to syslog. """ [make_deps] diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go index 712f18a9e1..a5085f4ee0 100644 --- a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go @@ -106,6 +106,7 @@ func (*Sequencer) Initialize(r runtime.Runtime) []runtime.Phase { "earlyServices", StartUdevd, StartMachined, + StartSyslogd, StartContainerd, ).Append( "usb", diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go index cc7b37034d..6796510e9f 100644 --- a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go @@ -620,6 +620,15 @@ func StartMachined(_ runtime.Sequence, _ any) (runtime.TaskExecutionFunc, string }, "startMachined" } +// StartSyslogd represents the task to start syslogd. +func StartSyslogd(r runtime.Sequence, _ any) (runtime.TaskExecutionFunc, string) { + return func(_ context.Context, _ *log.Logger, r runtime.Runtime) error { + system.Services(r).LoadAndStart(&services.Syslogd{}) + + return nil + }, "startSyslogd" +} + // StartDashboard represents the task to start dashboard. func StartDashboard(_ runtime.Sequence, _ interface{}) (runtime.TaskExecutionFunc, string) { return func(_ context.Context, _ *log.Logger, r runtime.Runtime) error { diff --git a/internal/app/machined/pkg/system/services/syslogd.go b/internal/app/machined/pkg/system/services/syslogd.go new file mode 100644 index 0000000000..700a2cb67e --- /dev/null +++ b/internal/app/machined/pkg/system/services/syslogd.go @@ -0,0 +1,68 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package services + +import ( + "context" + + "github.com/siderolabs/talos/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/internal/app/machined/pkg/system" + "github.com/siderolabs/talos/internal/app/machined/pkg/system/events" + "github.com/siderolabs/talos/internal/app/machined/pkg/system/health" + "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner" + "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner/goroutine" + "github.com/siderolabs/talos/internal/app/syslogd" + "github.com/siderolabs/talos/pkg/conditions" +) + +const syslogServiceID = "syslogd" + +var _ system.HealthcheckedService = (*Syslogd)(nil) + +// Syslogd implements the Service interface. It serves as the concrete type with +// the required methods. +type Syslogd struct{} + +// ID implements the Service interface. +func (s *Syslogd) ID(r runtime.Runtime) string { + return syslogServiceID +} + +// PreFunc implements the Service interface. +func (s *Syslogd) PreFunc(ctx context.Context, r runtime.Runtime) error { + return nil +} + +// PostFunc implements the Service interface. +func (s *Syslogd) PostFunc(r runtime.Runtime, state events.ServiceState) (err error) { + return nil +} + +// Condition implements the Service interface. +func (s *Syslogd) Condition(r runtime.Runtime) conditions.Condition { + return nil +} + +// DependsOn implements the Service interface. +func (s *Syslogd) DependsOn(r runtime.Runtime) []string { + return []string{machinedServiceID} +} + +// Runner implements the Service interface. +func (s *Syslogd) Runner(r runtime.Runtime) (runner.Runner, error) { + return goroutine.NewRunner(r, syslogServiceID, syslogd.Main, runner.WithLoggingManager(r.Logging())), nil +} + +// HealthFunc implements the HealthcheckedService interface. +func (s *Syslogd) HealthFunc(runtime.Runtime) health.Check { + return func(ctx context.Context) error { + return nil + } +} + +// HealthSettings implements the HealthcheckedService interface. +func (s *Syslogd) HealthSettings(runtime.Runtime) *health.Settings { + return &health.DefaultSettings +} diff --git a/internal/app/syslogd/internal/parser/parse.go b/internal/app/syslogd/internal/parser/parse.go new file mode 100644 index 0000000000..1c39ddcc61 --- /dev/null +++ b/internal/app/syslogd/internal/parser/parse.go @@ -0,0 +1,62 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package parser provides a syslog parser that can parse both RFC3164 and RFC5424 with best effort. +package parser + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/jeromer/syslogparser" + "github.com/jeromer/syslogparser/rfc3164" + "github.com/jeromer/syslogparser/rfc5424" +) + +// Parse parses a syslog message and returns a json encoded representation of the message. +// If an RFC3164 message is detected and there is no hostname field in the message, +// the hostname field is set to "localhost". +func Parse(b []byte) (string, error) { + // Detect the RFC version + rfc, err := syslogparser.DetectRFC(b) + if err != nil { + return "", err + } + + var parser syslogparser.LogParser + + switch rfc { + case syslogparser.RFC_3164: + parser = rfc3164.NewParser(b) + + if rfc3164ContainsHostname(b) { + parser.WithHostname("localhost") + } + case syslogparser.RFC_5424: + parser = rfc5424.NewParser(b) + default: + return "", fmt.Errorf("unsupported RFC version: %v", rfc) + } + + if err = parser.Parse(); err != nil { + return "", err + } + + msg, err := json.Marshal(parser.Dump()) + if err != nil { + return "", err + } + + return string(msg), nil +} + +func rfc3164ContainsHostname(buf []byte) bool { + indx := bytes.Index(buf, []byte(`]:`)) + if indx == -1 { + return false + } + + return bytes.Count(buf[:indx], []byte(` `)) == 3 +} diff --git a/internal/app/syslogd/internal/parser/parse_test.go b/internal/app/syslogd/internal/parser/parse_test.go new file mode 100644 index 0000000000..9ce40bbf61 --- /dev/null +++ b/internal/app/syslogd/internal/parser/parse_test.go @@ -0,0 +1,50 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// package parser provides a syslog parser that can parse both RFC3164 and RFC5424 with best effort. +package parser_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/siderolabs/talos/internal/app/syslogd/internal/parser" +) + +func TestParser(t *testing.T) { + for _, tc := range []struct { + name string + input []byte + expected string + }{ + { + name: "RFC3164 without tag and hostname", + input: []byte(`<4>Feb 16 17:54:19 time="2024-02-16T17:54:19.857755073Z" level=warning msg="Could not add /dev/mshv to the devices cgroup`), + expected: `{"content":"msg=\"Could not add /dev/mshv to the devices cgroup","facility":0,"hostname":"time=\"2024-02-16T17:54:19.857755073Z\"","priority":4,"severity":4,"tag":"level=warning","timestamp":"2024-02-16T17:54:19Z"}`, //nolint:lll + }, + { + name: "RFC3164 without hostname", + input: []byte(`<4>Feb 16 17:54:19 kata[2569]: time="2024-02-16T17:54:19.857755073Z" level=warning msg="Could not add /dev/mshv to the devices cgroup`), + expected: `{"content":"time=\"2024-02-16T17:54:19.857755073Z\" level=warning msg=\"Could not add /dev/mshv to the devices cgroup","facility":0,"hostname":"localhost","priority":4,"severity":4,"tag":"kata","timestamp":"2024-02-16T17:54:19Z"}`, //nolint:lll + }, + { + name: "RFC3164 with hostname", + input: []byte(`<4>Feb 16 17:54:19 hostname kata[2569]: time="2024-02-16T17:54:19.857755073Z" level=warning msg="Could not add /dev/mshv to the devices cgroup`), + expected: `{"content":"time=\"2024-02-16T17:54:19.857755073Z\" level=warning msg=\"Could not add /dev/mshv to the devices cgroup","facility":0,"hostname":"hostname","priority":4,"severity":4,"tag":"kata","timestamp":"2024-02-16T17:54:19Z"}`, //nolint:lll + }, + { + name: "RFC5424", + input: []byte(`<165>1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"] An application event log entry...`), + expected: `{"app_name":"evntslog","facility":20,"hostname":"mymachine.example.com","message":"An application event log entry...","msg_id":"ID47","priority":165,"proc_id":"-","severity":5,"structured_data":"[exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"]","timestamp":"2003-10-11T22:14:15.003Z","version":1}`, //nolint:lll + }, + } { + t.Run(tc.name, func(t *testing.T) { + parsedJSON, err := parser.Parse(tc.input) + require.NoError(t, err) + + require.Equal(t, tc.expected, parsedJSON) + }) + } +} diff --git a/internal/app/syslogd/syslogd.go b/internal/app/syslogd/syslogd.go new file mode 100644 index 0000000000..d1f64591ef --- /dev/null +++ b/internal/app/syslogd/syslogd.go @@ -0,0 +1,61 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package syslogd provides a syslogd service that listens on a unix socket +package syslogd + +import ( + "context" + "fmt" + "io" + "net" + + "github.com/siderolabs/talos/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/internal/app/syslogd/internal/parser" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// Main is an entrypoint to the API service. +func Main(ctx context.Context, _ runtime.Runtime, logWriter io.Writer) error { + return Run(ctx, logWriter, constants.SyslogListenSocketPath) +} + +// Run starts the syslogd service. +func Run(ctx context.Context, logWriter io.Writer, listenSocketPath string) error { + unixAddr, err := net.ResolveUnixAddr("unixgram", listenSocketPath) + if err != nil { + return err + } + + connection, err := net.ListenUnixgram("unixgram", unixAddr) + if err != nil { + return err + } + + if err = connection.SetReadBuffer(65536); err != nil { + return fmt.Errorf("failed to set read buffer: %w", err) + } + + buf := make([]byte, 1024) + + go func(con *net.UnixConn) { + for { + n, err := con.Read(buf) + if err != nil { + continue + } + + syslogJSON, err := parser.Parse(buf[:n]) + if err != nil { // if the message is not a valid syslog message, skip it + continue + } + + fmt.Fprintln(logWriter, syslogJSON) + } + }(connection) + + <-ctx.Done() + + return connection.Close() +} diff --git a/internal/app/syslogd/syslogd_test.go b/internal/app/syslogd/syslogd_test.go new file mode 100644 index 0000000000..5f21913c96 --- /dev/null +++ b/internal/app/syslogd/syslogd_test.go @@ -0,0 +1,86 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package syslogd_test + +import ( + "context" + "encoding/json" + "log/syslog" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/siderolabs/talos/internal/app/syslogd" +) + +type chanWriter struct { + ch chan []byte +} + +func (w *chanWriter) Write(p []byte) (n int, err error) { + w.ch <- p + + return len(p), nil +} + +func TestParsing(t *testing.T) { + ch := chanWriter{ + ch: make(chan []byte), + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + + defer cancel() + + socketPath := t.TempDir() + "/syslogd.sock" + + errChan := make(chan error) + + go func() { + errChan <- syslogd.Run(ctx, &ch, socketPath) + }() + + assert.Eventually(t, func() bool { + if _, err := os.Stat(socketPath); err == nil { + return true + } + + return false + }, 1*time.Second, 100*time.Millisecond) + + // Send a message to the syslogd service + syslog, err := syslog.Dial("unixgram", socketPath, syslog.LOG_INFO, "syslogd_test") + require.NoError(t, err) + + _, err = syslog.Write([]byte("Hello, syslogd!")) + require.NoError(t, err) + + defer syslog.Close() //nolint:errcheck + + select { + case msg := <-ch.ch: + var parsed map[string]interface{} + + require.NoError(t, json.Unmarshal(msg, &parsed)) + + // {"content":"Hello, syslogd!\n","facility":0,"hostname":"localhost","priority":6,"severity":6,"tag":"syslogd_test","timestamp":"2024-02-20T00:20:55Z" + assert.Equal(t, "syslogd_test", parsed["tag"]) + assert.Equal(t, "localhost", parsed["hostname"]) + assert.Equal(t, float64(6), parsed["priority"]) + assert.Equal(t, float64(6), parsed["severity"]) + assert.Equal(t, float64(0), parsed["facility"]) + assert.Equal(t, "Hello, syslogd!\n", parsed["content"]) + assert.NotEmpty(t, parsed["timestamp"]) + case <-ctx.Done(): + require.Fail(t, "timed out waiting for message") + } + + cancel() + + require.NoError(t, <-errChan) +} diff --git a/pkg/machinery/constants/constants.go b/pkg/machinery/constants/constants.go index 03a1800507..8175e06b67 100644 --- a/pkg/machinery/constants/constants.go +++ b/pkg/machinery/constants/constants.go @@ -974,6 +974,9 @@ const ( // PodResolvConfPath is the path to the pod resolv.conf file. PodResolvConfPath = "/system/resolved/resolv.conf" + + // SyslogListenSocketPath is the path to the syslog socket. + SyslogListenSocketPath = "/dev/log" ) // See https://linux.die.net/man/3/klogctl