-
Notifications
You must be signed in to change notification settings - Fork 64
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: implement Talos kernel log receiver
Fixes #527 Talos logs (see siderolabs/talos#4600) are delivered to Sidero over the SideroLink tunnel. Logs can be seen with: ``` $ kubectl logs -n sidero-system deployment/sidero-controller-manager -c serverlogs -f {"clock":67194673,"cluster":"management-cluster","facility":"user","machine":"default/management-cluster-cp-4j8f4","metal_machine":"default/management-cluster-cp-hbq57","msg":"[talos] phase bootloader (19/19): done, 176.795226ms\n","priority":"warning","seq":768,"server_uuid":"5b72932a-c482-4aa5-b00e-4b8773d3ac48","talos-level":"warn","talos-time":"2021-11-26T19:34:42.444342392Z"} ``` Logs are annotated on the fly with the information about `Server`, `MetalMachine`, `Machine` and `Cluster`. Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>
- Loading branch information
Showing
16 changed files
with
449 additions
and
166 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
138 changes: 138 additions & 0 deletions
138
app/sidero-controller-manager/cmd/log-receiver/handler.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,138 @@ | ||
| // This Source Code Form is subject to the terms of the Mozilla Public | ||
| // License, v. 2.0. If a copy of the MPL was not distributed with this | ||
| // file, You can obtain one at http://mozilla.org/MPL/2.0/. | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "fmt" | ||
| "os" | ||
| "strings" | ||
| "sync" | ||
| "time" | ||
|
|
||
| "go.uber.org/zap" | ||
| "inet.af/netaddr" | ||
| "k8s.io/apimachinery/pkg/runtime/schema" | ||
| "k8s.io/apimachinery/pkg/types" | ||
| clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" | ||
| runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" | ||
|
|
||
| "github.com/talos-systems/siderolink/pkg/logreceiver" | ||
|
|
||
| sidero "github.com/talos-systems/sidero/app/caps-controller-manager/api/v1alpha3" | ||
| ) | ||
|
|
||
| type sourceAnnotation struct { | ||
| ServerUUID string | ||
| MetalMachineName string | ||
| MachineName string | ||
| ClusterName string | ||
| } | ||
|
|
||
| var sourceMap sync.Map | ||
|
|
||
| func fetchSourceAnnotation(ctx context.Context, metalClient runtimeclient.Client, srcAddr netaddr.IP) (sourceAnnotation, error) { | ||
| var ( | ||
| annotation sourceAnnotation | ||
| serverbindings sidero.ServerBindingList | ||
| ) | ||
|
|
||
| if err := metalClient.List(ctx, &serverbindings); err != nil { | ||
| return annotation, fmt.Errorf("error getting server bindings: %w", err) | ||
| } | ||
|
|
||
| srcAddress := srcAddr.String() | ||
|
|
||
| var serverBinding *sidero.ServerBinding | ||
|
|
||
| for _, item := range serverbindings.Items { | ||
| item := item | ||
|
|
||
| if strings.HasPrefix(item.Spec.SideroLink.NodeAddress, srcAddress) { | ||
| serverBinding = &item | ||
|
|
||
| break | ||
| } | ||
| } | ||
|
|
||
| if serverBinding == nil { | ||
| // no matching server binding, leave things as is | ||
| return annotation, nil | ||
| } | ||
|
|
||
| annotation.ServerUUID = serverBinding.Name | ||
| annotation.MetalMachineName = fmt.Sprintf("%s/%s", serverBinding.Spec.MetalMachineRef.Namespace, serverBinding.Spec.MetalMachineRef.Name) | ||
| annotation.ClusterName = serverBinding.Labels[clusterv1.ClusterLabelName] | ||
|
|
||
| var metalMachine sidero.MetalMachine | ||
|
|
||
| if err := metalClient.Get(ctx, | ||
| types.NamespacedName{ | ||
| Namespace: serverBinding.Spec.MetalMachineRef.Namespace, | ||
| Name: serverBinding.Spec.MetalMachineRef.Name, | ||
| }, | ||
| &metalMachine); err != nil { | ||
| return annotation, fmt.Errorf("error getting metal machine: %w", err) | ||
| } | ||
|
|
||
| for _, ref := range metalMachine.OwnerReferences { | ||
| gv, err := schema.ParseGroupVersion(ref.APIVersion) | ||
| if err != nil { | ||
| continue | ||
| } | ||
|
|
||
| if ref.Kind == "Machine" && gv.Group == clusterv1.GroupVersion.Group { | ||
| annotation.MachineName = fmt.Sprintf("%s/%s", metalMachine.Namespace, ref.Name) | ||
|
|
||
| break | ||
| } | ||
| } | ||
|
|
||
| return annotation, nil | ||
| } | ||
|
|
||
| func logHandler(metalClient runtimeclient.Client, logger *zap.Logger) logreceiver.Handler { | ||
| return func(srcAddr netaddr.IP, msg map[string]interface{}) { | ||
| var annotation sourceAnnotation | ||
|
|
||
| v, ok := sourceMap.Load(srcAddr) | ||
| if !ok { | ||
| ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) | ||
| defer cancel() | ||
|
|
||
| var err error | ||
|
|
||
| annotation, err = fetchSourceAnnotation(ctx, metalClient, srcAddr) | ||
| if err != nil { | ||
| logger.Error("error fetching server information", zap.Error(err), zap.Stringer("source_addr", srcAddr)) | ||
| } | ||
|
|
||
| sourceMap.Store(srcAddr, annotation) | ||
| } else { | ||
| annotation = v.(sourceAnnotation) | ||
| } | ||
|
|
||
| if annotation.ServerUUID != "" { | ||
| msg["server_uuid"] = annotation.ServerUUID | ||
| } | ||
|
|
||
| if annotation.ClusterName != "" { | ||
| msg["cluster"] = annotation.ClusterName | ||
| } | ||
|
|
||
| if annotation.MetalMachineName != "" { | ||
| msg["metal_machine"] = annotation.MetalMachineName | ||
| } | ||
|
|
||
| if annotation.MachineName != "" { | ||
| msg["machine"] = annotation.MachineName | ||
| } | ||
|
|
||
| if err := json.NewEncoder(os.Stdout).Encode(msg); err != nil { | ||
| logger.Error("error printing log message", zap.Error(err)) | ||
| } | ||
| } | ||
| } |
33 changes: 33 additions & 0 deletions
33
app/sidero-controller-manager/cmd/log-receiver/kubernetes.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| // This Source Code Form is subject to the terms of the Mozilla Public | ||
| // License, v. 2.0. If a copy of the MPL was not distributed with this | ||
| // file, You can obtain one at http://mozilla.org/MPL/2.0/. | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "k8s.io/apimachinery/pkg/runtime" | ||
| clientgoscheme "k8s.io/client-go/kubernetes/scheme" | ||
| "k8s.io/client-go/rest" | ||
| ctrl "sigs.k8s.io/controller-runtime" | ||
| runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" | ||
|
|
||
| sidero "github.com/talos-systems/sidero/app/caps-controller-manager/api/v1alpha3" | ||
| ) | ||
|
|
||
| func getMetalClient() (runtimeclient.Client, *rest.Config, error) { | ||
| kubeconfig := ctrl.GetConfigOrDie() | ||
|
|
||
| scheme := runtime.NewScheme() | ||
|
|
||
| if err := clientgoscheme.AddToScheme(scheme); err != nil { | ||
| return nil, nil, err | ||
| } | ||
|
|
||
| if err := sidero.AddToScheme(scheme); err != nil { | ||
| return nil, nil, err | ||
| } | ||
|
|
||
| client, err := runtimeclient.New(kubeconfig, runtimeclient.Options{Scheme: scheme}) | ||
|
|
||
| return client, kubeconfig, err | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| // This Source Code Form is subject to the terms of the Mozilla Public | ||
| // License, v. 2.0. If a copy of the MPL was not distributed with this | ||
| // file, You can obtain one at http://mozilla.org/MPL/2.0/. | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "fmt" | ||
| "net" | ||
| "os" | ||
| "os/signal" | ||
| "syscall" | ||
|
|
||
| "go.uber.org/zap" | ||
| "golang.org/x/sync/errgroup" | ||
|
|
||
| "github.com/talos-systems/sidero/app/sidero-controller-manager/internal/siderolink" | ||
| "github.com/talos-systems/siderolink/pkg/logreceiver" | ||
| ) | ||
|
|
||
| func main() { | ||
| if err := run(); err != nil { | ||
| fmt.Fprintf(os.Stderr, "error: %s", err) | ||
| os.Exit(1) | ||
| } | ||
| } | ||
|
|
||
| func run() error { | ||
| logger, err := zap.NewProduction() | ||
| if err != nil { | ||
| return fmt.Errorf("failed to initialize logger: %w", err) | ||
| } | ||
|
|
||
| zap.ReplaceGlobals(logger) | ||
| zap.RedirectStdLog(logger) | ||
|
|
||
| metalclient, _, err := getMetalClient() | ||
| if err != nil { | ||
| return fmt.Errorf("error building runtime client: %w", err) | ||
| } | ||
|
|
||
| ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) | ||
| defer cancel() | ||
|
|
||
| eg, ctx := errgroup.WithContext(ctx) | ||
|
|
||
| listener, err := net.Listen("tcp", fmt.Sprintf(":%d", siderolink.LogReceiverPort)) | ||
| if err != nil { | ||
| return fmt.Errorf("error listening for log endpoint: %w", err) | ||
| } | ||
|
|
||
| srv, err := logreceiver.NewServer(logger, listener, logHandler(metalclient, logger)) | ||
| if err != nil { | ||
| return fmt.Errorf("error initializing log receiver: %w", err) | ||
| } | ||
|
|
||
| eg.Go(func() error { | ||
| return srv.Serve() | ||
| }) | ||
|
|
||
| eg.Go(func() error { | ||
| <-ctx.Done() | ||
|
|
||
| srv.Stop() | ||
|
|
||
| return nil | ||
| }) | ||
|
|
||
| if err := eg.Wait(); err != nil && !errors.Is(err, context.Canceled) { | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.