Skip to content

Commit

Permalink
Populate srl mgmt0 config with runtime mgmt bridge mtu (#1658)
Browse files Browse the repository at this point in the history 
* populate srl mgmt0 config with runtime mgmt bridge mtu

* reduce cyclomatic complexity

* added IP MTU

---------

Co-authored-by: Roman Dodin <dodin.roman@gmail.com>
  • Loading branch information
@steiler @hellt
steiler and hellt committed Oct 20, 2023
1 parent ce64f6d commit 283aec1
Show file tree
Hide file tree
Showing 5 changed files with 136 additions and 111 deletions.
1 change: 1 addition & 0 deletions docs/manual/kinds/srl.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ Additional configurations that containerlab adds on top of the factory config:
* enabling LLDP
* enabling gNMI/JSON-RPC
* creating tls server certificate
* setting `mgmt0 subinterface 0 ip-mtu` to the MTU value of the underlying container runtime network

A configuration checkpoint named `clab-initial` is generated by containerlab once default and user-provided configs are applied. The checkpoint may be used to quickly revert configuration changes made by a user to a state that was present after the node was started.

Expand Down
20 changes: 16 additions & 4 deletions nodes/srl/srl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@ set / system aaa authentication idle-timeout 7200
{{- if ne .MgmtMTU 0 }}
set / interface mgmt0 mtu {{ .MgmtMTU }}
{{- end }}
{{- if ne .MgmtIPMTU 0 }}
set / interface mgmt0 subinterface 0 ip-mtu {{ .MgmtIPMTU }}
{{- end }}
{{- /* enabling interfaces referenced as endpoints for a node (both e1-2 and e1-3-1 notations) */}}
{{- range $epName, $ep := .IFaces }}
set / interface ethernet-{{ $ep.Slot }}/{{ $ep.Port }} admin-state enable
Expand Down Expand Up @@ -564,6 +567,7 @@ type srlTemplateData struct {
IFaces map[string]tplIFace
SSHPubKeys string
MgmtMTU int
MgmtIPMTU int
DNSServers []string
}

Expand All @@ -582,15 +586,15 @@ func (n *srl) addDefaultConfig(ctx context.Context) error {
return err
}

// struct that holds data used in templating of the default config snippet

// tplData holds data used in templating of the default config snippet
tplData := srlTemplateData{
TLSKey: n.Cfg.TLSKey,
TLSCert: n.Cfg.TLSCert,
TLSAnchor: n.Cfg.TLSAnchor,
Banner: b,
IFaces: map[string]tplIFace{},
MgmtMTU: 0,
MgmtIPMTU: 0,
DNSServers: n.Config().DNS.Servers,
}

Expand All @@ -602,17 +606,25 @@ func (n *srl) addDefaultConfig(ctx context.Context) error {
tplData.SSHPubKeys = catenateKeys(n.sshPubKeys)
}

// set MgmtMTU to the MTU value of the runtime management network
// so that the two MTUs match.
tplData.MgmtIPMTU = n.Runtime.Mgmt().MTU

// prepare the endpoints
for _, e := range n.Endpoints {
ifName := e.GetIfaceName()
if ifName == "mgmt0" {
// skip the mgmt0 interface. This is just for traffic carrying interface

// if the endpoint has a custom MTU set, use it in the template logic
// otherwise we don't set the mtu as srlinux will use the default max value 9232
if m := e.GetLink().GetMTU(); m != links.DefaultLinkMTU {
tplData.MgmtMTU = m
// MgmtMTU seems to be only set when we use macvlan interface
// with network-mode: none. For this super narrow use case
// we setup mgmt port mtu to match the mtu of the macvlan parnet interface
// but then we need to make sure that IP MTU is smaller by 14B
tplData.MgmtIPMTU = m - 14
}
// the rest is just for traffic carrying interfaces
continue
}
// split the interface identifier into their parts
Expand Down
219 changes: 115 additions & 104 deletions runtime/docker/docker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,20 +94,23 @@ func (d *DockerRuntime) WithConfig(cfg *runtime.RuntimeConfig) {
func (d *DockerRuntime) WithMgmtNet(n *types.MgmtNet) {
d.mgmt = n
// return if MTU value was set by a user via config file
if n.MTU != "" {
if n.MTU != 0 {
return
}

// detect default MTU if this config parameter was not provided in the clab file
netRes, err := d.Client.NetworkInspect(context.TODO(), defaultDockerNetwork, dockerTypes.NetworkInspectOptions{})
if err != nil {
d.mgmt.MTU = "1500"
d.mgmt.MTU = 1500
log.Debugf("an error occurred when trying to detect docker default network mtu")
}

if mtu, ok := netRes.Options["com.docker.network.driver.mtu"]; ok {
log.Debugf("detected docker network mtu value - %s", mtu)
d.mgmt.MTU = mtu
d.mgmt.MTU, err = strconv.Atoi(mtu)
if err != nil {
log.Errorf("Error parsing MTU value of %q as int", mtu)
}
}

// if bridge was not set in the topo file, find out the bridge name
Expand Down Expand Up @@ -139,96 +142,10 @@ func (d *DockerRuntime) CreateNet(ctx context.Context) (err error) {
netResource, err := d.Client.NetworkInspect(nctx, d.mgmt.Network, dockerTypes.NetworkInspectOptions{})
switch {
case dockerC.IsErrNotFound(err):
log.Debugf("Network %q does not exist", d.mgmt.Network)
log.Infof("Creating docker network: Name=%q, IPv4Subnet=%q, IPv6Subnet=%q, MTU=%q",
d.mgmt.Network, d.mgmt.IPv4Subnet, d.mgmt.IPv6Subnet, d.mgmt.MTU)

enableIPv6 := false
var ipamConfig []network.IPAMConfig

var v4gw, v6gw string
// check if IPv4/6 addr are assigned to a mgmt bridge
if d.mgmt.Bridge != "" {
v4gw, v6gw, err = utils.FirstLinkIPs(d.mgmt.Bridge)
if err != nil {
// only return error if the error is not about link not found
// we will create the bridge if it doesn't exist
if !errors.As(err, &netlink.LinkNotFoundError{}) {
return err
}
}
log.Debugf("bridge %q has ipv4 addr of %q and ipv6 addr of %q", d.mgmt.Bridge, v4gw, v6gw)
}

if d.mgmt.IPv4Subnet != "" {
if d.mgmt.IPv4Gw != "" {
v4gw = d.mgmt.IPv4Gw
}
ipamCfg := network.IPAMConfig{
Subnet: d.mgmt.IPv4Subnet,
Gateway: v4gw,
}
if d.mgmt.IPv4Range != "" {
ipamCfg.IPRange = d.mgmt.IPv4Range
}
ipamConfig = append(ipamConfig, ipamCfg)
}

if d.mgmt.IPv6Subnet != "" {
if d.mgmt.IPv6Gw != "" {
v6gw = d.mgmt.IPv6Gw
}
ipamCfg := network.IPAMConfig{
Subnet: d.mgmt.IPv6Subnet,
Gateway: v6gw,
}
if d.mgmt.IPv6Range != "" {
ipamCfg.IPRange = d.mgmt.IPv6Range
}
ipamConfig = append(ipamConfig, ipamCfg)
enableIPv6 = true
}

ipam := &network.IPAM{
Driver: "default",
Config: ipamConfig,
}

netwOpts := map[string]string{
"com.docker.network.driver.mtu": d.mgmt.MTU,
}

if bridgeName != "" {
netwOpts["com.docker.network.bridge.name"] = bridgeName
}

opts := dockerTypes.NetworkCreate{
CheckDuplicate: true,
Driver: "bridge",
EnableIPv6: enableIPv6,
IPAM: ipam,
Internal: false,
Attachable: false,
Labels: map[string]string{
"containerlab": "",
},
Options: netwOpts,
}

netCreateResponse, err := d.Client.NetworkCreate(nctx, d.mgmt.Network, opts)
bridgeName, err = d.createMgmtBridge(nctx, bridgeName)
if err != nil {
return err
}

if len(netCreateResponse.ID) < 12 {
return fmt.Errorf("could not get bridge ID")
}
// when bridge is not set by a user explicitly
// we use the 12 chars of docker net as its name
if bridgeName == "" {
bridgeName = "br-" + netCreateResponse.ID[:12]
}

case err == nil:
log.Debugf("network %q was found. Reusing it...", d.mgmt.Network)
if len(netResource.ID) < 12 {
Expand Down Expand Up @@ -256,6 +173,113 @@ func (d *DockerRuntime) CreateNet(ctx context.Context) (err error) {
// get management bridge v4/6 addresses and save it under mgmt struct
// so that nodes can use this information prior to being deployed
// this was added to allow mgmt network gw ip to be available in a startup config templation step (ceos)
d.mgmt.IPv4Gw, d.mgmt.IPv6Gw, err = getMgmtBridgeIPs(bridgeName, netResource)
if err != nil {
return err
}

log.Debugf("Docker network %q, bridge name %q", d.mgmt.Network, bridgeName)

return d.postCreateNetActions()
}

func (d *DockerRuntime) createMgmtBridge(nctx context.Context, bridgeName string) (string, error) {
var err error
log.Debugf("Network %q does not exist", d.mgmt.Network)
log.Infof("Creating docker network: Name=%q, IPv4Subnet=%q, IPv6Subnet=%q, MTU=%q",
d.mgmt.Network, d.mgmt.IPv4Subnet, d.mgmt.IPv6Subnet, d.mgmt.MTU)

enableIPv6 := false
var ipamConfig []network.IPAMConfig

var v4gw, v6gw string
// check if IPv4/6 addr are assigned to a mgmt bridge
if d.mgmt.Bridge != "" {
v4gw, v6gw, err = utils.FirstLinkIPs(d.mgmt.Bridge)
if err != nil {
// only return error if the error is not about link not found
// we will create the bridge if it doesn't exist
if !errors.As(err, &netlink.LinkNotFoundError{}) {
return "", err
}
}
log.Debugf("bridge %q has ipv4 addr of %q and ipv6 addr of %q", d.mgmt.Bridge, v4gw, v6gw)
}

if d.mgmt.IPv4Subnet != "" {
if d.mgmt.IPv4Gw != "" {
v4gw = d.mgmt.IPv4Gw
}
ipamCfg := network.IPAMConfig{
Subnet: d.mgmt.IPv4Subnet,
Gateway: v4gw,
}
if d.mgmt.IPv4Range != "" {
ipamCfg.IPRange = d.mgmt.IPv4Range
}
ipamConfig = append(ipamConfig, ipamCfg)
}

if d.mgmt.IPv6Subnet != "" {
if d.mgmt.IPv6Gw != "" {
v6gw = d.mgmt.IPv6Gw
}
ipamCfg := network.IPAMConfig{
Subnet: d.mgmt.IPv6Subnet,
Gateway: v6gw,
}
if d.mgmt.IPv6Range != "" {
ipamCfg.IPRange = d.mgmt.IPv6Range
}
ipamConfig = append(ipamConfig, ipamCfg)
enableIPv6 = true
}

ipam := &network.IPAM{
Driver: "default",
Config: ipamConfig,
}

netwOpts := map[string]string{
"com.docker.network.driver.mtu": strconv.Itoa(d.mgmt.MTU),
}

if bridgeName != "" {
netwOpts["com.docker.network.bridge.name"] = bridgeName
}

opts := dockerTypes.NetworkCreate{
CheckDuplicate: true,
Driver: "bridge",
EnableIPv6: enableIPv6,
IPAM: ipam,
Internal: false,
Attachable: false,
Labels: map[string]string{
"containerlab": "",
},
Options: netwOpts,
}

netCreateResponse, err := d.Client.NetworkCreate(nctx, d.mgmt.Network, opts)
if err != nil {
return "", err
}

if len(netCreateResponse.ID) < 12 {
return "", fmt.Errorf("could not get bridge ID")
}
// when bridge is not set by a user explicitly
// we use the 12 chars of docker net as its name
if bridgeName == "" {
bridgeName = "br-" + netCreateResponse.ID[:12]
}
return bridgeName, nil
}

// getMgmtBridgeIPs gets the management bridge v4/6 addresses
func getMgmtBridgeIPs(bridgeName string, netResource dockerTypes.NetworkResource) (string, string, error) {
var err error
var v4, v6 string
if v4, v6, err = utils.FirstLinkIPs(bridgeName); err != nil {
log.Warn(
Expand All @@ -278,21 +302,8 @@ func (d *DockerRuntime) CreateNet(ctx context.Context) (err error) {
break
}
}

if v4 == "" && v6 == "" {
// didn't get address in any way -- the case of needing to inspect the docker ipam
// config should be less common (for dind use case basically), so we can just return
// the main error form checking via ip link
return err
}
}

d.mgmt.IPv4Gw = v4
d.mgmt.IPv6Gw = v6

log.Debugf("Docker network %q, bridge name %q", d.mgmt.Network, bridgeName)

return d.postCreateNetActions()
return v4, v6, err
}

// postCreateNetActions performs additional actions after the network has been created.
Expand Down
5 changes: 3 additions & 2 deletions runtime/podman/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"errors"
"fmt"
"net"
"strconv"
"strings"

netTypes "github.com/containers/common/libnetwork/types"
Expand Down Expand Up @@ -435,8 +436,8 @@ func (r *PodmanRuntime) netOpts(_ context.Context) (netTypes.Network, error) {
}

// add custom mtu if defined
if r.mgmt.MTU != "" {
options["mtu"] = r.mgmt.MTU
if r.mgmt.MTU != 0 {
options["mtu"] = strconv.Itoa(r.mgmt.MTU)
}
// compile the resulting struct
toReturn := netTypes.Network{
Expand Down
2 changes: 1 addition & 1 deletion types/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ type MgmtNet struct {
IPv6Subnet string `yaml:"ipv6-subnet,omitempty" json:"ipv6-subnet,omitempty"`
IPv6Gw string `yaml:"ipv6-gw,omitempty" json:"ipv6-gw,omitempty"`
IPv6Range string `yaml:"ipv6-range,omitempty" json:"ipv6-range,omitempty"`
MTU string `yaml:"mtu,omitempty" json:"mtu,omitempty"`
MTU int `yaml:"mtu,omitempty" json:"mtu,omitempty"`
ExternalAccess *bool `yaml:"external-access,omitempty" json:"external-access,omitempty"`
}

Expand Down

0 comments on commit 283aec1

Please sign in to comment.