Merge pull request #462 from srl-labs/keepmgmtnet

Implement --keep-mgmt-net for destroy subcmd

clab/clab.go

@@ -91,6 +91,12 @@ func WithRuntime(name string, d bool, dur time.Duration, gracefulShutdown bool)
 	}
 }
 
+func WithKeepMgmtNet() ClabOption {
+	return func(c *CLab) {
+		c.GlobalRuntime().WithKeepMgmtNet()
+	}
+}
+
 func WithTopoFile(file string) ClabOption {
 	return func(c *CLab) {
 		if file == "" {
@@ -104,7 +110,6 @@ func WithTopoFile(file string) ClabOption {
 		if err != nil {
 			log.Fatalf("failed to init the management network: %s", err)
 		}
-
 	}
 }
 

cmd/destroy.go

@@ -20,8 +20,11 @@ import (
 	"github.com/srl-labs/containerlab/types"
 )
 
-var cleanup bool
-var graceful bool
+var (
+	cleanup     bool
+	graceful    bool
+	keepMgmtNet bool
+)
 
 // destroyCmd represents the destroy command
 var destroyCmd = &cobra.Command{
@@ -42,6 +45,10 @@ var destroyCmd = &cobra.Command{
 			clab.WithRuntime(rt, debug, timeout, graceful),
 		}
 
+		if keepMgmtNet {
+			opts = append(opts, clab.WithKeepMgmtNet())
+		}
+
 		topos := map[string]struct{}{}
 
 		switch {
@@ -107,6 +114,7 @@ func init() {
 	destroyCmd.Flags().BoolVarP(&graceful, "graceful", "", false, "attempt to stop containers before removing")
 	destroyCmd.Flags().BoolVarP(&all, "all", "a", false, "destroy all containerlab labs")
 	destroyCmd.Flags().UintVarP(&maxWorkers, "max-workers", "", 0, "limit the maximum number of workers deleting nodes")
+	destroyCmd.Flags().BoolVarP(&keepMgmtNet, "keep-mgmt-net", "", false, "do not remove the management network")
 }
 
 func deleteEntriesFromHostsFile(containers []types.GenericContainer, bridgeName string) error {
@@ -201,13 +209,13 @@ func destroyLab(ctx context.Context, c *clab.CLab) (err error) {
 	}
 
 	// delete lab management network
-	log.Infof("Deleting network '%s'...", c.Config.Mgmt.Network)
-	if err = c.GlobalRuntime().DeleteNet(ctx); err != nil {
-		// do not log error message if deletion error simply says that such network doesn't exist
-		if err.Error() != fmt.Sprintf("Error: No such network: %s", c.Config.Mgmt.Network) {
-			log.Error(err)
+	if c.Config.Mgmt.Network != "bridge" && !keepMgmtNet {
+		if err = c.GlobalRuntime().DeleteNet(ctx); err != nil {
+			// do not log error message if deletion error simply says that such network doesn't exist
+			if err.Error() != fmt.Sprintf("Error: No such network: %s", c.Config.Mgmt.Network) {
+				log.Error(err)
+			}
 		}
-
 	}
 	// delete container network namespaces symlinks
 	return c.DeleteNetnsSymlinks()

cmd/root.go

@@ -34,7 +34,6 @@ var rootCmd = &cobra.Command{
 		if debug {
 			log.SetLevel(log.DebugLevel)
 		}
-
 	},
 }
 

docs/cmd/destroy.md

@@ -27,6 +27,9 @@ Refer to the [configuration artifacts](../manual/conf-artifacts.md) page to get
 #### graceful
 To make containerlab attempt a graceful shutdown of the running containers, add the `--graceful` flag to destroy cmd. Without it, containers will be removed forcefully without even attempting to stop them.
 
+#### keep-mgmt-net
+Do not try to remove the management network. Usually the management docker network (in case of docker) and the underlaying bridge are being removed. If you have attached additional resources outside of containerlab and you want the bridge to remain intact just add the `--keep-mgmt-net` flag.
+
 #### all
 Destroy command provided with `--all | -a` flag will perform the deletion of all the labs running on the container host. It will not touch containers launched manually.
 

runtime/containerd/containerd.go

@@ -39,8 +39,7 @@ const (
 func init() {
 	runtime.Register(runtimeName, func() runtime.ContainerRuntime {
 		return &ContainerdRuntime{
-			Mgmt: new(types.MgmtNet),
-		}
+			Mgmt: new(types.MgmtNet)}
 	})
 }
 
@@ -91,6 +90,9 @@ func (c *ContainerdRuntime) WithMgmtNet(n *types.MgmtNet) {
 	c.Mgmt = n
 }
 
+func (c *ContainerdRuntime) WithKeepMgmtNet() {
+	c.config.KeepMgmtNet = true
+}
 func (c *ContainerdRuntime) GetName() string               { return runtimeName }
 func (c *ContainerdRuntime) Config() runtime.RuntimeConfig { return c.config }
 
@@ -99,12 +101,31 @@ func (c *ContainerdRuntime) CreateNet(ctx context.Context) error {
 	return nil
 }
 func (c *ContainerdRuntime) DeleteNet(context.Context) error {
-	log.Debug("DeleteNet() - Not yet required with containerd")
-	return nil
+	var err error
+	bridgename := c.Mgmt.Bridge
+	brInUse := true
+	for i := 0; i < 10; i++ {
+		brInUse, err = utils.CheckBrInUse(bridgename)
+		if err != nil {
+			return err
+		}
+		time.Sleep(time.Millisecond * 100)
+		if !brInUse {
+			// Stop early if bridge no longer in use
+			// Need to wait some time, since the earlier veth deletion
+			// triggert from the cotnainer deletion is async and needs
+			// to finish. W'll have a race condition otherwise.
+			break
+		}
+	}
+	if c.config.KeepMgmtNet || brInUse {
+		log.Infof("Skipping deletion of bridge '%s'", bridgename)
+		return nil
+	}
+	return utils.DeleteLinkByName(bridgename)
 }
 
 func (c *ContainerdRuntime) PullImageIfRequired(ctx context.Context, imagename string) error {
-
 	log.Debugf("Looking up %s container image", imagename)
 	ctx = namespaces.WithNamespace(ctx, containerdNamespace)
 	if !strings.Contains(imagename, ":") {
@@ -224,7 +245,6 @@ func (c *ContainerdRuntime) CreateContainer(ctx context.Context, node *types.Nod
 		if len(portmappings) > 0 {
 			cnirc.CapabilityArgs["portMappings"] = portmappings
 		}
-
 	}
 
 	var cOpts []containerd.NewContainerOpts

runtime/docker/docker.go

@@ -61,6 +61,9 @@ func (c *DockerRuntime) Init(opts ...runtime.RuntimeOption) error {
 	return nil
 }
 
+func (c *DockerRuntime) WithKeepMgmtNet() {
+	c.config.KeepMgmtNet = true
+}
 func (c *DockerRuntime) GetName() string               { return runtimeName }
 func (c *DockerRuntime) Config() runtime.RuntimeConfig { return c.config }
 
@@ -201,14 +204,15 @@ func (c *DockerRuntime) CreateNet(ctx context.Context) (err error) {
 
 // DeleteNet deletes a docker bridge
 func (c *DockerRuntime) DeleteNet(ctx context.Context) (err error) {
-	if c.Mgmt.Network == "bridge" {
-		log.Debug("Skipping potential deletion of docker default bridge 'bridge'.")
+	network := c.Mgmt.Network
+	if network == "bridge" || c.config.KeepMgmtNet {
+		log.Debugf("Skipping deletion of '%s' network", network)
 		return nil
 	}
 	nctx, cancel := context.WithTimeout(ctx, c.config.Timeout)
 	defer cancel()
 
-	nres, err := c.Client.NetworkInspect(ctx, c.Mgmt.Network, dockerTypes.NetworkInspectOptions{})
+	nres, err := c.Client.NetworkInspect(ctx, network, dockerTypes.NetworkInspectOptions{})
 	if err != nil {
 		return err
 	}
@@ -217,12 +221,12 @@ func (c *DockerRuntime) DeleteNet(ctx context.Context) (err error) {
 		if c.config.Debug {
 			log.Debugf("network '%s' has %d active endpoints, deletion skipped", c.Mgmt.Network, numEndpoints)
 			for _, endp := range nres.Containers {
-				log.Debugf("'%s' is connected to %s", endp.Name, c.Mgmt.Network)
+				log.Debugf("'%s' is connected to %s", endp.Name, network)
 			}
 		}
 		return nil
 	}
-	err = c.Client.NetworkRemove(nctx, c.Mgmt.Network)
+	err = c.Client.NetworkRemove(nctx, network)
 	if err != nil {
 		return err
 	}

runtime/ignite/iginite.go

@@ -119,6 +119,9 @@ func (c *IgniteRuntime) WithConfig(cfg *runtime.RuntimeConfig) {
 	}
 
 }
+func (c *IgniteRuntime) WithKeepMgmtNet() {
+	c.ctrRuntime.WithKeepMgmtNet()
+}
 
 func (c *IgniteRuntime) WithMgmtNet(n *types.MgmtNet) {
 	c.Mgmt = n

runtime/runtime.go

@@ -24,6 +24,8 @@ type ContainerRuntime interface {
 	WithConfig(*RuntimeConfig)
 	// Set the network management details (generated by the config.go)
 	WithMgmtNet(*types.MgmtNet)
+	// Instructs the runtime not to delete the mgmt network on destroy
+	WithKeepMgmtNet()
 	// Create container (bridge) network
 	CreateNet(context.Context) error
 	// Delete container (bridge) network
@@ -60,6 +62,7 @@ type RuntimeConfig struct {
 	Timeout          time.Duration
 	GracefulShutdown bool
 	Debug            bool
+	KeepMgmtNet      bool
 }
 
 var ContainerRuntimes = map[string]Initializer{}
@@ -79,3 +82,9 @@ func WithMgmtNet(mgmt *types.MgmtNet) RuntimeOption {
 		r.WithMgmtNet(mgmt)
 	}
 }
+
+func WithKeepMgmtNet() RuntimeOption {
+	return func(r ContainerRuntime) {
+		r.WithKeepMgmtNet()
+	}
+}

tests/01-smoke/01-basic-flow.robot

@@ -84,7 +84,7 @@ Verify bind mount in l1 node
 
 Verify port forwarding for node l2
     ${rc}    ${output} =    Run And Return Rc And Output
-    ...    curl localhost:56180
+    ...    curl -m 3 localhost:56180
     Log    ${output}
     Should Be Equal As Integers    ${rc}    0
     Should Contain    ${output}    Thank you for using nginx
@@ -100,6 +100,13 @@ Verify static mgmt addressing for l2
     Log    ${ipv6}
     Should Be Equal As Strings    ${ipv6}    ${n2-ipv6}
 
+Verify l1 environment has MYVAR variable set
+    ${rc}    ${output} =    Run And Return Rc And Output
+    ...    ${runtime-cli-exec-cmd} clab-2-linux-nodes-l1 sh -c "echo \\$MYVAR"
+    Log    ${output}
+    Should Be Equal As Integers    ${rc}    0
+    Should Contain    ${output}    MYVAR is SET
+
 Destroy ${lab-name} lab
     ${rc}    ${output} =    Run And Return Rc And Output
     ...    sudo containerlab --runtime ${runtime} destroy -t ${CURDIR}/01-linux-nodes.clab.yml --cleanup

tests/01-smoke/01-linux-nodes.clab.yml

@@ -9,9 +9,11 @@ topology:
     l1:
       kind: linux
       image: alpine:3
-      cmd: ash -c "sleep 9999"
+      cmd: ash -c "echo $MYVAR > /tmp/var && sleep 9999"
       binds:
         - /tmp/clab-01-test.txt:/01-test.txt
+      env:
+        MYVAR: MYVAR is SET
     l2:
       kind: linux
       image: nginx:stable-alpine

tests/01-smoke/07-keep-mgmt-net.robot

@@ -0,0 +1,53 @@
+*** Comments ***
+This test suite verifies
+- the management bridge is not deleted when --keep-mgmt-net is present and the lab is destroyed
+- the management bridge is deleted by default
+
+*** Settings ***
+Library           OperatingSystem
+Library           String
+Suite Teardown    Run    sudo containerlab --runtime ${runtime} destroy -t ${topo} --cleanup
+
+*** Variables ***
+${lab-name}       7-keep-mgmt-net
+${topo}           ${CURDIR}/07-linux-single-node.clab.yml
+${mgmt-bridge}    01-07-net
+
+*** Test Cases ***
+Deploy ${lab-name} lab
+    Log    ${CURDIR}
+    ${rc}    ${output} =    Run And Return Rc And Output
+    ...    sudo containerlab --runtime ${runtime} deploy -t ${topo}
+    Log    ${output}
+    Should Be Equal As Integers    ${rc}    0
+
+Destroy ${lab-name} lab keep mgmt net
+    ${rc}    ${output} =    Run And Return Rc And Output
+    ...    sudo containerlab --runtime ${runtime} destroy -t ${topo} --keep-mgmt-net
+    Log    ${output}
+    Should Be Equal As Integers    ${rc}    0
+
+Check ${lab-name} mgmt network remains
+    ${rc}    ${output} =    Run And Return Rc And Output
+    ...    sudo ip l show dev ${mgmt-bridge}
+    Log    ${output}
+    Should Be Equal As Integers    ${rc}    0
+
+Deploy ${lab-name} lab again
+    Log    ${CURDIR}
+    ${rc}    ${output} =    Run And Return Rc And Output
+    ...    sudo containerlab --runtime ${runtime} deploy -t ${topo}
+    Log    ${output}
+    Should Be Equal As Integers    ${rc}    0
+
+Destroy ${lab-name} lab dont keep mgmt net
+    ${rc}    ${output} =    Run And Return Rc And Output
+    ...    sudo containerlab --runtime ${runtime} destroy -t ${topo} --cleanup
+    Log    ${output}
+    Should Be Equal As Integers    ${rc}    0
+
+Check ${lab-name} mgmt network is gone
+    ${rc}    ${output} =    Run And Return Rc And Output
+    ...    sudo ip l show dev ${mgmt-bridge}
+    Log    ${output}
+    Should Not Be Equal As Integers    ${rc}    0

tests/01-smoke/07-linux-single-node.clab.yml

@@ -0,0 +1,14 @@
+# Copyright 2020 Nokia
+# Licensed under the BSD 3-Clause License.
+# SPDX-License-Identifier: BSD-3-Clause
+
+name: single-node
+mgmt:
+  bridge: 01-07-net
+
+topology:
+  nodes:
+    l1:
+      kind: linux
+      image: alpine:3
+      cmd: ash -c "sleep 9999"

utils/netlink.go

@@ -50,6 +50,34 @@ func DefaultNetMTU() (string, error) {
 	return fmt.Sprint(b.MTU), nil
 }
 
+func CheckBrInUse(brname string) (bool, error) {
+	InUse := false
+	l, err := netlink.LinkList()
+	if err != nil {
+		return InUse, err
+	}
+	mgmtbr, err := netlink.LinkByName(brname)
+	if err != nil {
+		return InUse, err
+	}
+	mgmtbridx := mgmtbr.Attrs().Index
+	for _, link := range l {
+		if link.Attrs().MasterIndex == mgmtbridx {
+			InUse = true
+			break
+		}
+	}
+	return InUse, nil
+}
+
+func DeleteLinkByName(name string) error {
+	l, err := netlink.LinkByName(name)
+	if err != nil {
+		return err
+	}
+	return netlink.LinkDel(l)
+}
+
 // GenMac generates a random MAC address for a given OUI
 func GenMac(oui string) string {
 	buf := make([]byte, 3)