Skip to content

Commit

Permalink
chore: import siderolink as siderolink-launch subcommand
Browse files Browse the repository at this point in the history
This PR ensures that we can test our siderolink communication using embedded siderolink-agent.
If `--with-siderolink` provided during `talos cluster create` talosctl will embed proper kernel string and setup `siderolink-agent` as a separate process. It should be used with combination of `--skip-injecting-config` and `--with-apply-config` (the latter will use newly generated IPv6 siderolink addresses which talosctl passes to the agent as a "pre-bind").

Fixes siderolabs#8392

Signed-off-by: Dmitriy Matrenichev <dmitry.matrenichev@siderolabs.com>
  • Loading branch information
DmitriyMV committed Mar 21, 2024
1 parent 84ec8c1 commit c638d32
Show file tree
Hide file tree
Showing 19 changed files with 450 additions and 25 deletions.
1 change: 1 addition & 0 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ linters-settings:
- gopkg.in/yaml.v3
- github.com/coredns/coredns
- github.com/mdlayher/kobject
- github.com/siderolabs/siderolink
retract-allow-no-explanation: false
exclude-forbidden: true

Expand Down
59 changes: 58 additions & 1 deletion cmd/talosctl/cmd/mgmt/cluster/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (
"errors"
"fmt"
"math/big"
"math/rand/v2"
"net"
"net/netip"
"net/url"
"os"
Expand Down Expand Up @@ -172,6 +174,7 @@ var (
diskEncryptionKeyTypes []string
withFirewall string
withUUIDHostnames bool
withSiderolinkAgent bool
)

// createCmd represents the cluster up command.
Expand Down Expand Up @@ -422,6 +425,7 @@ func create(ctx context.Context, flags *pflag.FlagSet) error {
provision.WithTPM2(tpm2Enabled),
provision.WithExtraUEFISearchPaths(extraUEFISearchPaths),
provision.WithTargetArch(targetArch),
provision.WithSideorlinkAgent(withSiderolinkAgent),
}

var configBundleOpts []bundle.Option
Expand Down Expand Up @@ -746,9 +750,38 @@ func create(ctx context.Context, flags *pflag.FlagSet) error {
extraKernelArgs = procfs.NewCmdline(extraBootKernelArgs)
}

if withSiderolinkAgent {
if extraKernelArgs == nil {
extraKernelArgs = procfs.NewCmdline("")
}

if extraKernelArgs.Get("siderolink.api") != nil || extraKernelArgs.Get("talos.events.sink") != nil || extraKernelArgs.Get("talos.logging.kernel") != nil {
return errors.New("siderolink kernel arguments are already set, cannot run with --with-siderolink")
}

wgPort := rand.IntN(20000) + 40000
wgHost := gatewayIPs[0].String()
apiPort := strconv.Itoa(wgPort + 1)
sinkPort := strconv.Itoa(wgPort + 2)
logPort := strconv.Itoa(wgPort + 3)

request.SiderolinkRequest.WireguardEndpoint = net.JoinHostPort(wgHost, strconv.Itoa(wgPort))
request.SiderolinkRequest.APIEndpoint = ":" + apiPort
request.SiderolinkRequest.SinkEndpoint = ":" + sinkPort
request.SiderolinkRequest.LogEndpoint = ":" + logPort

apiLink := "grpc://" + net.JoinHostPort(wgHost, apiPort) + "?jointoken=foo"

extraKernelArgs.Append("siderolink.api", apiLink)
extraKernelArgs.Append("talos.events.sink", "[fdae:41e4:649b:9303::1]:"+sinkPort)
extraKernelArgs.Append("talos.logging.kernel", "tcp://[fdae:41e4:649b:9303::1]:"+logPort)
}

// Add talosconfig to provision options, so we'll have it to parse there
provisionOptions = append(provisionOptions, provision.WithTalosConfig(configBundle.TalosConfig()))

var wgNodeGen nodeAddrGenerator

// Create the controlplane nodes.
for i := range controlplanes {
var cfg config.Provider
Expand All @@ -760,6 +793,17 @@ func create(ctx context.Context, flags *pflag.FlagSet) error {

nodeUUID := uuid.New()

if withSiderolinkAgent {
var generated netip.Addr

generated, err = wgNodeGen.GenerateRandomNodeAddr()
if err != nil {
return err
}

request.SiderolinkRequest.AddBind(nodeUUID, generated)
}

nodeReq := provision.NodeRequest{
Name: nodeName(clusterName, "controlplane", i+1, nodeUUID),
Type: machine.TypeControlPlane,
Expand Down Expand Up @@ -820,6 +864,17 @@ func create(ctx context.Context, flags *pflag.FlagSet) error {

nodeUUID := uuid.New()

if withSiderolinkAgent {
var generated netip.Addr

generated, err = wgNodeGen.GenerateRandomNodeAddr()
if err != nil {
return err
}

request.SiderolinkRequest.AddBind(nodeUUID, generated)
}

request.Nodes = append(request.Nodes,
provision.NodeRequest{
Name: nodeName(clusterName, "worker", i, nodeUUID),
Expand Down Expand Up @@ -855,7 +910,7 @@ func create(ctx context.Context, flags *pflag.FlagSet) error {
defer clusterAccess.Close() //nolint:errcheck

if applyConfigEnabled {
err = clusterAccess.ApplyConfig(ctx, request.Nodes, os.Stdout)
err = clusterAccess.ApplyConfig(ctx, request.Nodes, request.SiderolinkRequest, os.Stdout)
if err != nil {
return err
}
Expand Down Expand Up @@ -1153,6 +1208,8 @@ func init() {
createCmd.Flags().IntVar(&bandwidth, "with-network-bandwidth", 0, "specify bandwidth restriction (in kbps) on the bridge interface when creating a qemu cluster")
createCmd.Flags().StringVar(&withFirewall, firewallFlag, "", "inject firewall rules into the cluster, value is default policy - accept/block (QEMU only)")
createCmd.Flags().BoolVar(&withUUIDHostnames, "with-uuid-hostnames", false, "use machine UUIDs as default hostnames (QEMU only)")
createCmd.Flags().BoolVar(&withSiderolinkAgent, "with-siderolink", false, "enables the use of siderolink agent as configuration apply mechanism")
createCmd.Flags().MarkHidden("with-siderolink") //nolint:errcheck

Cmd.AddCommand(createCmd)
}
Expand Down
32 changes: 32 additions & 0 deletions cmd/talosctl/cmd/mgmt/cluster/wg_linux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

//go:build linux

package cluster

import (
"fmt"
"net/netip"

"github.com/siderolabs/gen/value"
"github.com/siderolabs/siderolink/pkg/wireguard"
)

type nodeAddrGenerator struct {
prefix netip.Prefix
}

func (ng *nodeAddrGenerator) GenerateRandomNodeAddr() (netip.Addr, error) {
if value.IsZero(ng.prefix) {
ng.prefix = wireguard.NetworkPrefix("")
}

result, err := wireguard.GenerateRandomNodeAddr(ng.prefix)
if err != nil {
return netip.Addr{}, fmt.Errorf("failed to generate random node address: %w", err)
}

return result.Addr(), nil
}
18 changes: 18 additions & 0 deletions cmd/talosctl/cmd/mgmt/cluster/wg_other.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

//go:build !linux

package cluster

import (
"errors"
"net/netip"
)

type nodeAddrGenerator struct{}

func (ng *nodeAddrGenerator) GenerateRandomNodeAddr() (netip.Addr, error) {
return netip.Addr{}, errors.New("unsupported platform")
}
4 changes: 3 additions & 1 deletion cmd/talosctl/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,14 @@ import (
"fmt"
"os"
"path/filepath"
"slices"
"strings"

"github.com/spf13/cobra"

"github.com/siderolabs/talos/cmd/talosctl/cmd/common"
"github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt"
"github.com/siderolabs/talos/cmd/talosctl/cmd/siderolink"
"github.com/siderolabs/talos/cmd/talosctl/cmd/talos"
"github.com/siderolabs/talos/pkg/cli"
"github.com/siderolabs/talos/pkg/machinery/constants"
Expand Down Expand Up @@ -67,7 +69,7 @@ func Execute() error {
}

func init() {
for _, cmd := range append(talos.Commands, mgmt.Commands...) {
for _, cmd := range slices.Concat(talos.Commands, mgmt.Commands, siderolink.Commands) {
rootCmd.AddCommand(cmd)
}
}
61 changes: 61 additions & 0 deletions cmd/talosctl/cmd/siderolink/agent.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// Package siderolink provides the talosctl siderolink-launch command implementation.
package siderolink

import (
"os"
"os/signal"

"github.com/spf13/cobra"
)

var siderolinkFlags struct {
joinToken string
wireguardEndpoint string
sinkEndpoint string
apiEndpoint string
logEndpoint string
predefinedPairs []string
}

var siderolinkCmd = &cobra.Command{
Use: "siderolink-launch",
Short: "Internal command used by cluster create to launch siderolink agent",
Long: ``,
Args: cobra.NoArgs,
Hidden: true,
RunE: func(cmd *cobra.Command, args []string) error {
ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt)
defer cancel()

return run(ctx)
},
}

func init() {
siderolinkCmd.PersistentFlags().StringVar(&siderolinkFlags.joinToken, "sidero-link-join-token", "", "join token for the cluster")
siderolinkCmd.PersistentFlags().StringVar(&siderolinkFlags.wireguardEndpoint, "sidero-link-wireguard-endpoint", "", "advertised Wireguard endpoint")
siderolinkCmd.PersistentFlags().StringVar(&siderolinkFlags.sinkEndpoint, "event-sink-endpoint", "", "gRPC API endpoint for the Event Sink")
siderolinkCmd.PersistentFlags().StringVar(&siderolinkFlags.apiEndpoint, "sidero-link-api-endpoint", "", "gRPC API endpoint for the SideroLink")
siderolinkCmd.PersistentFlags().StringVar(&siderolinkFlags.logEndpoint, "log-receiver-endpoint", "", "TCP log receiver endpoint")
siderolinkCmd.PersistentFlags().StringArrayVar(&siderolinkFlags.predefinedPairs, "predefined-pair", nil, "predefined pairs of UUID=IPv6 addrs for the nodes")

for _, names := range []string{
"sidero-link-join-token",
"sidero-link-wireguard-endpoint",
"predefined-pair",
"event-sink-endpoint",
"sidero-link-api-endpoint",
"log-receiver-endpoint",
} {
err := siderolinkCmd.PersistentFlags().MarkHidden(names)
if err != nil {
panic(err)
}
}

addCommand(siderolinkCmd)
}
16 changes: 16 additions & 0 deletions cmd/talosctl/cmd/siderolink/root.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package siderolink

import (
"github.com/spf13/cobra"
)

// Commands is a list of commands published by the package.
var Commands []*cobra.Command

func addCommand(cmd *cobra.Command) {
Commands = append(Commands, cmd)
}
61 changes: 61 additions & 0 deletions cmd/talosctl/cmd/siderolink/run_linux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

//go:build linux

package siderolink

import (
"context"
"fmt"

"github.com/siderolabs/siderolink/pkg/agent"
"github.com/siderolabs/siderolink/pkg/wireguard"
"go.uber.org/zap"
"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
)

func run(ctx context.Context) error {
logger, err := zap.NewDevelopment()
if err != nil {
return err
}

logger.Info("starting embedded siderolink agent")
defer logger.Info("stopping embedded siderolink agent")

err = agent.Run(
ctx,
agent.Config{
WireguardEndpoint: siderolinkFlags.wireguardEndpoint,
APIEndpoint: siderolinkFlags.apiEndpoint,
JoinToken: siderolinkFlags.joinToken,
ForceUserspace: true,
SinkEndpoint: siderolinkFlags.sinkEndpoint,
LogEndpoint: siderolinkFlags.logEndpoint,
UUIDIPv6Pairs: siderolinkFlags.predefinedPairs,
},
&handler{l: logger},
logger,
)
if err != nil {
return fmt.Errorf("failed to run siderolink agent: %w", err)
}

return nil
}

type handler struct {
l *zap.Logger
}

func (h *handler) HandlePeerAdded(event wireguard.PeerEvent) error {
h.l.Info("talos agent sees peer added", zap.String("address", event.Address.String()))

return nil
}

func (h *handler) HandlePeerRemoved(wgtypes.Key) error {
return nil
}
16 changes: 16 additions & 0 deletions cmd/talosctl/cmd/siderolink/run_other.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

//go:build !linux

package siderolink

import (
"context"
"errors"
)

func run(context.Context) error {
return errors.New("unsupported platform")
}
12 changes: 7 additions & 5 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,14 @@ go 1.22.1

replace (
// forked coredns so we don't carry caddy and other stuff into the Talos
github.com/coredns/coredns => github.com/siderolabs/coredns v1.11.52
github.com/coredns/coredns => github.com/siderolabs/coredns v1.11.2-0.20240313201213-4df2d63f3a59

// see https://github.com/mdlayher/kobject/pull/5
github.com/mdlayher/kobject => github.com/smira/kobject v0.0.0-20240304111826-49c8d4613389

// siderolink temporarily fork
github.com/siderolabs/siderolink => github.com/DmitriyMV/siderolink v0.1.3-0.20240321174721-930096812155

// Use nested module.
github.com/siderolabs/talos/pkg/machinery => ./pkg/machinery

Expand Down Expand Up @@ -268,7 +271,6 @@ require (
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118 // indirect
github.com/mdlayher/packet v1.1.2 // indirect
github.com/mdlayher/socket v0.5.0 // indirect
Expand Down Expand Up @@ -298,9 +300,9 @@ require (
github.com/pkg/errors v0.9.1 // indirect
github.com/planetscale/vtprotobuf v0.6.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.18.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.45.0 // indirect
github.com/prometheus/client_golang v1.19.0 // indirect
github.com/prometheus/client_model v0.6.0 // indirect
github.com/prometheus/common v0.50.0 // indirect
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
Expand Down
Loading

0 comments on commit c638d32

Please sign in to comment.