topology: Added support for VPP running in docker container

Makefile

@@ -102,6 +102,7 @@ WITH_OPENCONTRAIL?=true
 WITH_LIBVIRT_GO?=true
 WITH_EBPF_DOCKER_BUILDER?=false
 WITH_VPP?=false
+WITH_DOCKER_VPP?=false
 
 export PATH:=$(BUILD_TOOLS):$(PATH)
 
@@ -203,6 +204,10 @@ ifeq ($(WITH_VPP), true)
   AGENT_TEST_EXTRA_PROBES+=vpp
 endif
 
+ifeq ($(WITH_DOCKER_VPP), true)
+  BUILD_TAGS+=docker_vpp
+endif
+
 ifeq (${DEBUG}, true)
 GOFLAGS=-gcflags='-N -l'
 GO_BINDATA_FLAGS+=-debug

agent/probes.go

@@ -26,6 +26,8 @@ import (
 	"github.com/skydive-project/skydive/logging"
 	"github.com/skydive-project/skydive/probe"
 	"github.com/skydive-project/skydive/topology/probes/docker"
+	"github.com/skydive-project/skydive/topology/probes/docker/subprobes"
+	docker_vpp "github.com/skydive-project/skydive/topology/probes/docker/subprobes/vpp"
 	"github.com/skydive-project/skydive/topology/probes/libvirt"
 	"github.com/skydive-project/skydive/topology/probes/lldp"
 	"github.com/skydive-project/skydive/topology/probes/lxd"
@@ -86,7 +88,8 @@ func NewTopologyProbeBundleFromConfig(g *graph.Graph, hostNode *graph.Node) (*pr
 		case "docker":
 			dockerURL := config.GetString("agent.topology.docker.url")
 			netnsRunPath := config.GetString("agent.topology.docker.netns.run_path")
-			dockerProbe, err := docker.NewProbe(nsProbe, dockerURL, netnsRunPath)
+			subprobes := dockerSubprobes(nsProbe)
+			dockerProbe, err := docker.NewProbe(nsProbe, dockerURL, netnsRunPath, subprobes)
 			if err != nil {
 				return nil, fmt.Errorf("Failed to initialize Docker probe: %s", err)
 			}
@@ -137,3 +140,14 @@ func NewTopologyProbeBundleFromConfig(g *graph.Graph, hostNode *graph.Node) (*pr
 
 	return bundle, nil
 }
+
+// dockerSubprobes create all docker related subprobes
+func dockerSubprobes(nsProbe *netns.Probe) []subprobes.Subprobe {
+	subprobes := make([]subprobes.Subprobe, 0)
+	if vpp, err := docker_vpp.NewSubprobe(nsProbe); err != nil {
+		logging.GetLogger().Warningf("VPP subprobe in docker probe will be disabled because its creation failed: %v", err)
+	} else {
+		subprobes = append(subprobes, vpp)
+	}
+	return subprobes
+}

config/config.go

@@ -91,6 +91,8 @@ func init() {
 	cfg.SetDefault("agent.topology.probes", []string{"ovsdb"})
 	cfg.SetDefault("agent.topology.docker.url", "unix:///var/run/docker.sock")
 	cfg.SetDefault("agent.topology.docker.netns.run_path", "/var/run/docker/netns")
+	cfg.SetDefault("agent.topology.docker.vpp.cliconnect", "")
+	cfg.SetDefault("agent.topology.docker.vpp.probeinterval", 5) // interval is in seconds
 	cfg.SetDefault("agent.topology.netlink.metrics_update", 30)
 	cfg.SetDefault("agent.topology.netns.run_path", "/var/run/netns")
 	cfg.SetDefault("agent.topology.neutron.domain_name", "Default")

etc/skydive.yml.default

@@ -211,6 +211,20 @@ agent:
       netns:
         # allow to specify where the docker probe is watching network namespaces
         # run_path: /var/run/docker/netns
+      vpp:
+        # configuration for vpp subprobe
+        cliconnect:
+          # Path to VPP CLI socket file or VPP CLI connection IP address with port.
+          # Default "" is using default VPP CLI socket file (/run/vpp/cli.sock). This
+          # setting is used for retrieving information from VPP in docker, i.e. to get
+          # VPP version, docker client executes in given container command
+          # "vppctl -s <cliconnect> show version"
+        probeinterval:
+          # Interval (in seconds) between regular probing in all docker containers for
+          # VPP topology changes. Default value is 5. Minimal values is 1 (lower values
+          # will be changed to 1). The probing can get easily CPU-intensive when
+          # count of docker containers increases. You can mitigate CPU problems by setting
+          # longer probe intervals.
 
     netlink:
       # delay in seconds between two metric updates

scripts/ci/jobs/jobs.yml

@@ -368,7 +368,7 @@
     builders:
       - skydive-cleanup
       - skydive-test:
-          test: BACKEND=orientdb scripts/ci/run-functional-tests.sh
+          test: BACKEND=orientdb WITH_DOCKER_VPP=true scripts/ci/run-functional-tests.sh
     publishers:
       - junit:
           results: tests.xml

scripts/ci/run-compile-tests.sh

@@ -27,7 +27,7 @@ make
 make test.functionals.compile TAGS=${TAGS}
 
 # Compile with all build options supported enabled
-make WITH_DPDK=true WITH_EBPF=true WITH_VPP=true WITH_EBPF_DOCKER_BUILDER=true WITH_K8S=true WITH_ISTIO=true WITH_HELM=true VERBOSE=true
+make WITH_DPDK=true WITH_EBPF=true WITH_VPP=true WITH_DOCKER_VPP=true WITH_EBPF_DOCKER_BUILDER=true WITH_K8S=true WITH_ISTIO=true WITH_HELM=true VERBOSE=true
 
 # Compile Skydive for Windows
 GOOS=windows GOARCH=amd64 govendor build github.com/skydive-project/skydive
@@ -39,7 +39,7 @@ GOOS=darwin GOARCH=amd64 govendor build github.com/skydive-project/skydive
 make WITH_PROF=true VERBOSE=true
 
 # Compile all tests
-make test.functionals.compile TAGS=${TAGS} WITH_NEUTRON=true WITH_SELENIUM=true WITH_CDD=true WITH_SCALE=true WITH_EBPF=true WITH_VPP=true WITH_K8S=true WITH_ISTIO=true WITH_HELM=true WITH_EBPF_DOCKER_BUILDER=true
+make test.functionals.compile TAGS=${TAGS} WITH_NEUTRON=true WITH_SELENIUM=true WITH_CDD=true WITH_SCALE=true WITH_EBPF=true WITH_VPP=true WITH_DOCKER_VPP=true WITH_K8S=true WITH_ISTIO=true WITH_HELM=true WITH_EBPF_DOCKER_BUILDER=true
 
 # Compile static
 make static

tests/docker_vpp_test.go

@@ -0,0 +1,145 @@
+// +build docker_vpp
+
+// Copyright (c) 2019 PANTHEON.tech s.r.o.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tests
+
+import (
+	"fmt"
+	"github.com/skydive-project/skydive/gremlin"
+	"testing"
+)
+
+const (
+	dockerImageWithRunningVPP = "ligato/vpp-base:19.04"
+	vppWaitScript             = "sh -c 'retry=%d;until docker exec %s vppctl sh version || [ $retry -eq 0 ]; do retry=$(( retry-1 ));sleep 0.5s;echo \"VPP not ready-retries left \"$retry;done'"
+)
+
+func TestRunningVPPInDocker(t *testing.T) {
+	test := &Test{
+		setupCmds: []Cmd{
+			{fmt.Sprintf("docker run -d -t -i --name test-skydive-docker-running-vpp %s", dockerImageWithRunningVPP), false},
+			{fmt.Sprintf(vppWaitScript, 10, "test-skydive-docker-running-vpp"), true},
+		},
+
+		tearDownCmds: []Cmd{
+			{"docker rm -f test-skydive-docker-running-vpp", false},
+		},
+
+		mode: Replay,
+
+		checks: []CheckFunction{func(c *CheckContext) error {
+			return assertOneEndNode(c, c.gremlin.V().Has("Type", "netns", "Manager", "docker").
+				Out("Type", "vpp", "Manager", "docker"))
+		}},
+	}
+
+	RunTest(t, test)
+}
+
+func TestDockerVPPConnectingToVeth(t *testing.T) {
+	test := &Test{
+		setupCmds: []Cmd{
+			{fmt.Sprintf("docker run -d -t -i --privileged --name test-skydive-docker-vpp-to-veth %s", dockerImageWithRunningVPP), false},
+			{fmt.Sprintf(vppWaitScript, 40, "test-skydive-docker-vpp-to-veth"), true},
+			{"docker exec test-skydive-docker-vpp-to-veth ip link add name veth-container type veth peer name veth-host", true}, // creating veth tunnel (that can be used to tunnel docker container and docker host)
+			{"docker exec test-skydive-docker-vpp-to-veth ip link set dev veth-container up", true},
+			{"docker exec test-skydive-docker-vpp-to-veth ip link set dev veth-host up", true},                     // no need for this test to actually push veth-host to network namespace of docker host OS
+			{"docker exec test-skydive-docker-vpp-to-veth vppctl create host-interface name veth-container", true}, // grabbing and using veth-container end of tunnel in VPP
+			{"docker exec test-skydive-docker-vpp-to-veth vppctl set int state host-veth-container up", true},
+		},
+
+		tearDownCmds: []Cmd{
+			{"docker rm -f test-skydive-docker-vpp-to-veth", false},
+		},
+
+		mode: Replay,
+
+		checks: []CheckFunction{func(c *CheckContext) error {
+			return assertOneEndNode(c, c.gremlin.V().Has("Type", "vpp", "Manager", "docker", "Container", "test-skydive-docker-vpp-to-veth").
+				Out("Type", "veth", "Name", "veth-container"))
+		}},
+	}
+
+	RunTest(t, test)
+}
+
+func TestTwoVPPsConnectedUsingMemifTunnel(t *testing.T) {
+	vpp1Container := "test-skydive-docker-vpp1-with-memif-tunnel"
+	vpp2Container := "test-skydive-docker-vpp2-with-memif-tunnel"
+	test := &Test{
+		setupCmds: []Cmd{
+			// prepare container-shared folder (docker would create it automatically, but creating it now and with user that is running test resolves permission problems in teardown)
+			{"mkdir /tmp/skydivetests-dockervpp-sockets", false},
+
+			// starting docker contrainers
+			{fmt.Sprintf("docker run -d -t -i -v /tmp/skydivetests-dockervpp-sockets/:/run/othersockets/ --name %s %s", vpp1Container, dockerImageWithRunningVPP), false},
+			{fmt.Sprintf("docker run -d -t -i -v /tmp/skydivetests-dockervpp-sockets/:/run/othersockets/ --name %s %s", vpp2Container, dockerImageWithRunningVPP), false},
+
+			// waiting for VPPs to start inside containers
+			{fmt.Sprintf(vppWaitScript, 10, vpp1Container), true},
+			{fmt.Sprintf(vppWaitScript, 10, vpp2Container), true},
+
+			// creating memif tunnel
+			{fmt.Sprintf("docker exec %s vppctl create memif socket id 1 filename /run/othersockets/another-memif.sock", vpp1Container), true},
+			{fmt.Sprintf("docker exec %s vppctl create interface memif socket-id 1 id 0 master", vpp1Container), true},
+			{fmt.Sprintf("docker exec %s vppctl set int state memif1/0 up", vpp1Container), true},
+			{fmt.Sprintf("docker exec %s vppctl create memif socket id 1 filename /run/othersockets/another-memif.sock", vpp2Container), true},
+			{fmt.Sprintf("docker exec %s vppctl create interface memif socket-id 1 id 0 slave", vpp2Container), true},
+			{fmt.Sprintf("docker exec %s vppctl set int state memif1/0 up", vpp2Container), true},
+		},
+
+		tearDownCmds: []Cmd{
+			// removing memif socket file (it was created by VPP,but removing it from VPP doesn't remove the physical
+			// file->removing reference from VPPs and removing it on docker container level to prevent permission problems)
+			{fmt.Sprintf("docker exec %s vppctl delete interface memif memif1/0", vpp1Container), true},
+			{fmt.Sprintf("docker exec %s vppctl delete interface memif memif1/0", vpp2Container), true},
+			{fmt.Sprintf("docker exec %s vppctl delete memif socket id 1", vpp1Container), true},
+			{fmt.Sprintf("docker exec %s vppctl delete memif socket id 1", vpp2Container), true},
+			{fmt.Sprintf("docker exec %s rm -rf /run/othersockets/another-memif.sock", vpp1Container), true},
+
+			// removing docker containers
+			{fmt.Sprintf("docker rm -f %s", vpp1Container), false},
+			{fmt.Sprintf("docker rm -f %s", vpp2Container), false},
+
+			// removing container-shared folder for memif socket file
+			{"rm -rf /tmp/skydivetests-dockervpp-sockets", true},
+		},
+
+		mode: Replay,
+
+		checks: []CheckFunction{func(c *CheckContext) error {
+			return assertOneEndNode(c, c.gremlin.V().Has("Type", "vpp", "Manager", "docker", "Container", vpp1Container).
+				Out("Type", "intf", "Name", "memif1/0", "Manager", "docker").
+				Both("Type", "intf", "Name", "memif1/0", "Manager", "docker").
+				In("Type", "vpp", "Manager", "docker", "Container", vpp2Container))
+		}},
+	}
+
+	RunTest(t, test)
+}
+
+func assertOneEndNode(c *CheckContext, queryString gremlin.QueryString) error {
+	nodes, err := c.gh.GetNodes(queryString)
+	if err != nil {
+		return err
+	}
+
+	if len(nodes) != 1 {
+		return fmt.Errorf("expected 1 end node, got %+v", nodes)
+	}
+
+	return nil
+}

topology/probes/docker/docker.go

@@ -36,6 +36,7 @@ import (
 	"github.com/skydive-project/skydive/graffiti/graph"
 	"github.com/skydive-project/skydive/logging"
 	"github.com/skydive-project/skydive/topology"
+	"github.com/skydive-project/skydive/topology/probes/docker/subprobes"
 	ns "github.com/skydive-project/skydive/topology/probes/netns"
 	sversion "github.com/skydive-project/skydive/version"
 )
@@ -60,6 +61,7 @@ type Probe struct {
 	wg           sync.WaitGroup
 	hostNs       netns.NsHandle
 	containerMap map[string]containerInfo
+	subprobes    []subprobes.Subprobe
 }
 
 func (probe *Probe) containerNamespace(pid int) string {
@@ -144,6 +146,17 @@ func (probe *Probe) registerContainer(id string) {
 		Pid:  info.State.Pid,
 		Node: containerNode,
 	}
+
+	regData := &subprobes.ContainerRegistrationData{
+		Info:     info,
+		Node:     containerNode,
+		NSRootID: n.ID,
+	}
+	for _, sp := range probe.subprobes {
+		if err := sp.RegisterContainer(regData); err != nil {
+			logging.GetLogger().Errorf("Subprobe %T failed by container registration: %v", sp, err)
+		}
+	}
 }
 
 func (probe *Probe) unregisterContainer(id string) {
@@ -155,6 +168,15 @@ func (probe *Probe) unregisterContainer(id string) {
 		return
 	}
 
+	unregData := &subprobes.ContainerUnregistrationData{
+		Node: infos.Node,
+	}
+	for _, sp := range probe.subprobes {
+		if err := sp.UnregisterContainer(unregData); err != nil {
+			logging.GetLogger().Errorf("Subprobe %T failed by container unregistration: %v", sp, err)
+		}
+	}
+
 	probe.Graph.Lock()
 	if err := probe.Graph.DelNode(infos.Node); err != nil {
 		probe.Graph.Unlock()
@@ -269,6 +291,10 @@ func (probe *Probe) Start() {
 			probe.wg.Wait()
 		}
 	}()
+
+	for _, sp := range probe.subprobes {
+		sp.Start()
+	}
 }
 
 // Stop the probe
@@ -281,17 +307,21 @@ func (probe *Probe) Stop() {
 		probe.cancel()
 		probe.wg.Wait()
 	}
+	for _, sp := range probe.subprobes {
+		sp.Stop()
+	}
 
 	atomic.StoreInt64(&probe.state, common.StoppedState)
 }
 
 // NewProbe creates a new topology Docker probe
-func NewProbe(nsProbe *ns.Probe, dockerURL, netnsRunPath string) (*Probe, error) {
+func NewProbe(nsProbe *ns.Probe, dockerURL, netnsRunPath string, subprobes []subprobes.Subprobe) (*Probe, error) {
 	probe := &Probe{
 		Probe:        nsProbe,
 		url:          dockerURL,
 		containerMap: make(map[string]containerInfo),
 		state:        common.StoppedState,
+		subprobes:    subprobes,
 	}
 
 	if netnsRunPath != "" {

topology/probes/docker/no_docker.go

@@ -21,6 +21,7 @@ package docker
 
 import (
 	"github.com/skydive-project/skydive/common"
+	"github.com/skydive-project/skydive/topology/probes/docker/subprobes"
 	ns "github.com/skydive-project/skydive/topology/probes/netns"
 )
 
@@ -37,6 +38,6 @@ func (probe *Probe) Stop() {
 }
 
 // NewProbe creates a new topology Docker probe
-func NewProbe(nsProbe *ns.Probe, dockerURL, netnsRunPath string) (*Probe, error) {
+func NewProbe(nsProbe *ns.Probe, dockerURL, netnsRunPath string, subprobes []subprobes.Subprobe) (*Probe, error) {
 	return nil, common.ErrNotImplemented
 }