From d825508891ab470e1e7ea2eda5dc7bc39f67b951 Mon Sep 17 00:00:00 2001 From: Antonin Bas Date: Fri, 22 Nov 2019 10:42:37 -0800 Subject: [PATCH] Fix Kind support (Linux hosts only) The following changes were required: * Disable TX HW checksum offload in containers. This is done in the Antrea CNI server when setting-up Pod networking, using an ioctl ethtool system call. * Disable TX HW checksum offload in the Linux host for the veth interface of each Kind Node. This must be done by invoking an additional script (hack/kind_linux.sh) after creating the Kind cluster. * Create a secondary br-phy bridge on each Node, as required by OVS userspace tunneling. * Use a new version of start_ovs (start_ovs_netdev) which modifies the ovs-ctl script in-place to avoid loading the kernel module. Refer to #14 for the rationale for all the above bullet points. A new test "provider" was added to the e2e test framework so that all the e2e tests can be run on Kind clusters. As part of this, some changes to the framework had to be performed. For example it is impractical to run SSH commands on Kind Nodes - as they do not have an SSH server - so instead we use "docker exec". Fixes #14 Fixes #13 --- .github/workflows/kind.yml | 42 +++++++++++ build/images/ethtool/Dockerfile | 8 ++ build/images/ethtool/README.md | 16 ++++ build/images/scripts/start_ovs_netdev | 96 ++++++++++++++++++++++++ build/yamls/patches/kind/startOvs.yml | 10 +++ ci/kind/config.yml | 8 ++ cmd/antrea-agent/agent.go | 1 + docs/kind.md | 19 ++++- hack/generate-manifest.sh | 3 + hack/kind-linux.sh | 30 ++++++++ pkg/agent/cniserver/pod_configuration.go | 41 ++++++---- pkg/agent/cniserver/server.go | 3 +- pkg/agent/util/ethtool/ethtool_linux.go | 74 ++++++++++++++++++ test/e2e/README.md | 12 +++ test/e2e/basic_test.go | 2 +- test/e2e/fixtures.go | 7 +- test/e2e/framework.go | 25 ++---- test/e2e/providers/exec/docker.go | 61 +++++++++++++++ test/e2e/{ => providers/exec}/ssh.go | 8 +- test/e2e/providers/interface.go | 6 +- test/e2e/providers/kind.go | 66 ++++++++++++++++ test/e2e/providers/vagrant.go | 18 ++++- 22 files changed, 506 insertions(+), 50 deletions(-) create mode 100644 .github/workflows/kind.yml create mode 100644 build/images/ethtool/Dockerfile create mode 100644 build/images/ethtool/README.md create mode 100755 build/images/scripts/start_ovs_netdev create mode 100644 build/yamls/patches/kind/startOvs.yml create mode 100644 ci/kind/config.yml create mode 100755 hack/kind-linux.sh create mode 100644 pkg/agent/util/ethtool/ethtool_linux.go create mode 100644 test/e2e/providers/exec/docker.go rename test/e2e/{ => providers/exec}/ssh.go (85%) create mode 100644 test/e2e/providers/kind.go diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml new file mode 100644 index 00000000000..f5dd3190a40 --- /dev/null +++ b/.github/workflows/kind.yml @@ -0,0 +1,42 @@ +name: Kind +on: + pull_request: + branches: + - master + - release-* +jobs: + test-unit: + name: E2e tests on a Kind cluster on Linux + runs-on: [ubuntu-18.04] + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-go@v1 + with: + go-version: 1.12 + - name: Build Antrea image + run: make + - name: Install Kind + env: + KIND_VERSION: v0.6.0 + run: | + curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 + chmod +x ./kind + sudo mv kind /usr/local/bin + - name: Create Kind cluster + run: | + kind create cluster --config ci/kind/config.yml + kind get nodes | xargs ./hack/kind-linux.sh + - name: Deploy Antrea + # kubectl is installed on the Github Ubuntu 18.04 worker + run: | + kind load docker-image antrea/antrea-ubuntu:latest + ./hack/generate-manifest.sh --kind | kubectl apply -f - + - name: Printing some debug information + run: | + sleep 30 + kubectl get -A all + kubectl -n kube-system logs --all-containers -l app=antrea + - name: Run e2e tests + run: | + ./hack/generate-manifest.sh --kind | docker exec -i kind-control-plane dd of=/root/antrea.yml + go test github.com/vmware-tanzu/antrea/test/e2e -provider=kind diff --git a/build/images/ethtool/Dockerfile b/build/images/ethtool/Dockerfile new file mode 100644 index 00000000000..929289a0035 --- /dev/null +++ b/build/images/ethtool/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:18.04 + +LABEL maintainer="Antrea " +LABEL description="A Docker image based on Ubuntu 18.04 which includes ethtool and ip tools." + +RUN apt-get update && \ + apt-get install -y --no-install-recommends ethtool iproute2 && \ + rm -rf /var/cache/apt/* /var/lib/apt/lists/* diff --git a/build/images/ethtool/README.md b/build/images/ethtool/README.md new file mode 100644 index 00000000000..4e9f2f76ccf --- /dev/null +++ b/build/images/ethtool/README.md @@ -0,0 +1,16 @@ +# images/ethtool + +This Docker image is a very lightweight image based on Ubuntu 18.04 which +includes ethtool and the ip tools. + +If you need to build a new version of the image and push it to Dockerhub, you +can run the following: + +```bash +cd build/images/ethtool +docker build -t antrea/ethtool:latest . +docker push antrea/ethtool:latest +``` + +The `docker push` command will fail if you do not have permission to push to the +`antrea` Dockerhub repository. diff --git a/build/images/scripts/start_ovs_netdev b/build/images/scripts/start_ovs_netdev new file mode 100755 index 00000000000..2f55812aae3 --- /dev/null +++ b/build/images/scripts/start_ovs_netdev @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +source logging +source daemon_status + +CONTAINER_NAME="antrea-ovs" +OVS_DB_FILE="/var/run/openvswitch/conf.db" + +set -euo pipefail + +hwaddr=$(ip link show eth0 | grep link/ether | awk '{print $2}') +inet=$(ip addr show eth0 | grep "inet " | awk '{ print $2 }') +gw=$(ip route | grep default | awk '{ print $3 }') + +# Modify ovs-ctl so that the kernel module is no longer loaded since it is not +# needed when using OVS in userspace mode. It also enables running OVS with the +# netdev datapath type on platforms which do not have the OVS kernel module. +# This is easier than starting daemons manually... +function fix_ovs_ctl { + sed -i 's/\(\w*\)\(insert_mod_if_required || return 1\)/\1# \2/' /usr/share/openvswitch/scripts/ovs-ctl +} + +# See http://docs.openvswitch.org/en/latest/howto/userspace-tunneling/ +function add_br_phy { + log_info $CONTAINER_NAME "Creating OVS br-phy bridge for netdev datapath type" + ovs-vsctl --may-exist add-br br-phy \ + -- set Bridge br-phy datapath_type=netdev \ + -- br-set-external-id br-phy bridge-id br-phy \ + -- set bridge br-phy fail-mode=standalone \ + other_config:hwaddr="$hwaddr" + + ovs-vsctl --timeout 10 add-port br-phy eth0 + ip addr add "$inet" dev br-phy + ip link set br-phy up + ip addr flush dev eth0 2>/dev/null + ip link set eth0 up + ip route add default via "$gw" dev br-phy +} + +function del_br_phy { + log_info $CONTAINER_NAME "Deleting OVS br-phy bridge" + ovs-vsctl del-port br-phy eth0 + ovs-vsctl del-br br-phy + ip addr add "$inet" dev eth0 + ip link set eth0 up + ip route add default via "$gw" dev eth0 +} + +function start_ovs { + log_info $CONTAINER_NAME "Starting OVS" + /usr/share/openvswitch/scripts/ovs-ctl --system-id=random start --db-file=$OVS_DB_FILE +} + +function stop_ovs { + log_info $CONTAINER_NAME "Stopping OVS" + /usr/share/openvswitch/scripts/ovs-ctl stop +} + +SLEEP_PID= + +function quit { + log_info $CONTAINER_NAME "Stopping OVS before quit" + # delete the bridge and move IP address back to eth0 to restore connectivity + # when OVS is stopped. + del_br_phy + stop_ovs + # kill background sleep process + if [ "$SLEEP_PID" != "" ]; then kill $SLEEP_PID > /dev/null 2>&1 || true; fi + exit 0 +} + +# Do not trap EXIT as it would then ignore the "exit 0" statement in quit and +# exit with code 128 + SIGNAL +trap "quit" INT TERM + +fix_ovs_ctl + +start_ovs +add_br_phy + +log_info $CONTAINER_NAME "Started the loop that checks OVS status every 30 seconds" +while true; do + # we run sleep in the background so that we can immediately exit when we + # receive SIGINT / SIGTERM + # see https://stackoverflow.com/questions/32041674/linux-how-to-kill-sleep + sleep 30 & + SLEEP_PID=$! + wait $SLEEP_PID + + if ! check_ovs_status ; then + # OVS was stopped in the container. + log_warning $CONTAINER_NAME "OVS was stopped. Starting it again" + + start_ovs + fi +done diff --git a/build/yamls/patches/kind/startOvs.yml b/build/yamls/patches/kind/startOvs.yml new file mode 100644 index 00000000000..631f71ae4e6 --- /dev/null +++ b/build/yamls/patches/kind/startOvs.yml @@ -0,0 +1,10 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: antrea-agent +spec: + template: + spec: + containers: + - name: antrea-ovs + command: ["start_ovs_netdev"] diff --git a/ci/kind/config.yml b/ci/kind/config.yml new file mode 100644 index 00000000000..9d4e28bf9ee --- /dev/null +++ b/ci/kind/config.yml @@ -0,0 +1,8 @@ +kind: Cluster +apiVersion: kind.sigs.k8s.io/v1alpha3 +networking: + disableDefaultCNI: true + podSubnet: 10.10.0.0/16 +nodes: +- role: control-plane +- role: worker diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 55b06314b02..5c3df0f8603 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -99,6 +99,7 @@ func run(o *Options) error { o.config.CNISocket, o.config.HostProcPathPrefix, o.config.DefaultMTU, + o.config.OVSDatapathType, nodeConfig, ovsBridgeClient, ofClient, diff --git a/docs/kind.md b/docs/kind.md index 440c7d84aa0..d33098457d1 100644 --- a/docs/kind.md +++ b/docs/kind.md @@ -1,5 +1,8 @@ # Deploying Antrea on a Kind cluster +At the moment Kind is only supported on Linux hosts. We are working on +supporting macOS hosts as well. + ## Create a Kind cluster and deploy Antrea in a few seconds ### Create a Kind cluster @@ -31,6 +34,8 @@ These instructions assume that you have built the `antrea/antrea-ubuntu` Docker image locally (e.g. by running `make` from the root of the repository). ```bash +# "fix" the host's veth interfaces (for the different Kind Nodes) +kind get nodes | xargs ./hack/kind-linux.sh # load the Antrea Docker image in the Nodes kind load docker-image antrea/antrea-ubuntu:latest # deploy Antrea @@ -39,7 +44,7 @@ kind load docker-image antrea/antrea-ubuntu:latest ### Check that everything is working -After a few seconds you sould be able to observe the following when running +After a few seconds you should be able to observe the following when running `kubectl get -n kube-system pods -l app=antrea`: ```bash NAME READY STATUS RESTARTS AGE @@ -65,3 +70,15 @@ requires some changes to the way Antrea is deployed. Most notably: (`netdev`) OVS datapath type is used * the Antrea agent's Init Container no longer needs to load the `openvswitch` kernel module + * the `start_ovs` script used by the `antrea-ovs` container needs to be + replaced with the `start_ovs_netdev` script, which creates an additional + bridge (`br-phy`) as required for [OVS userspace + tunneling](http://docs.openvswitch.org/en/latest/howto/userspace-tunneling/) + +### Why do I need to run the `hack/kind-linux.sh` script on my host? + +The script is required for Antrea to work properly in a Kind cluster on +Linux. It takes care of disabling TX hardware checksum offload for the veth +interface (in the host's network namespace) of each Kind Node. This is required +when using OVS in userspace mode. Refer to this [Antrea Github issue +#14](https://github.com/vmware-tanzu/antrea/issues/14) for more information. diff --git a/hack/generate-manifest.sh b/hack/generate-manifest.sh index 515e2d88041..1bcc44a2059 100755 --- a/hack/generate-manifest.sh +++ b/hack/generate-manifest.sh @@ -131,6 +131,9 @@ if $KIND; then $KUSTOMIZE edit add patch tunDevice.yml # edit antrea Agent configuration to use the netdev datapath $KUSTOMIZE edit add patch ovsDatapath.yml + # antrea-ovs should use start_ovs_netdev instead of start_ovs to ensure that the br_phy bridge + # is created. + $KUSTOMIZE edit add patch startOvs.yml # change initContainer script and remove SYS_MODULE capability $KUSTOMIZE edit add patch installCni.yml fi diff --git a/hack/kind-linux.sh b/hack/kind-linux.sh new file mode 100755 index 00000000000..c4a94a01b91 --- /dev/null +++ b/hack/kind-linux.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Copyright 2019 Antrea Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is required for Antrea to work properly in a Kind cluster on Linux. It takes care of +# disabling TX hardware checksum offload for the veth interface (in the host's network namespace) of +# each Kind Node. This is required when using OVS in userspace mode. Refer to +# https://github.com/vmware-tanzu/antrea/issues/14 for more information. + +# The script uses the antrea/ethtool Docker image (so that ethtool does not need to be installed on +# the Linux host). + +for node in "$@"; do + peerIdx=$(docker exec "$node" ip link | grep eth0 | awk -F[@:] '{ print $3 }' | cut -c 3-) + peerName=$(docker run --net=host antrea/ethtool:latest ip link | grep "$peerIdx": | awk -F[:@] '{ print $2 }' | cut -c 2-) + echo "Disabling TX checksum offload for node $node ($peerName)" + docker run --net=host --privileged antrea/ethtool:latest ethtool -K "$peerName" tx off +done diff --git a/pkg/agent/cniserver/pod_configuration.go b/pkg/agent/cniserver/pod_configuration.go index 76190d2e333..81a3e85389b 100644 --- a/pkg/agent/cniserver/pod_configuration.go +++ b/pkg/agent/cniserver/pod_configuration.go @@ -33,6 +33,7 @@ import ( "github.com/vmware-tanzu/antrea/pkg/agent/interfacestore" "github.com/vmware-tanzu/antrea/pkg/agent/openflow" "github.com/vmware-tanzu/antrea/pkg/agent/util" + "github.com/vmware-tanzu/antrea/pkg/agent/util/ethtool" "github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig" ) @@ -57,6 +58,24 @@ const ( ovsExternalIDPodNamespace = "pod-namespace" ) +type podConfigurator struct { + ovsBridgeClient ovsconfig.OVSBridgeClient + ofClient openflow.Client + ifaceStore interfacestore.InterfaceStore + gatewayMAC net.HardwareAddr + ovsDatapathType string +} + +func newPodConfigurator( + ovsBridgeClient ovsconfig.OVSBridgeClient, + ofClient openflow.Client, + ifaceStore interfacestore.InterfaceStore, + gatewayMAC net.HardwareAddr, + ovsDatapathType string, +) *podConfigurator { + return &podConfigurator{ovsBridgeClient, ofClient, ifaceStore, gatewayMAC, ovsDatapathType} +} + // setupInterfaces creates a veth pair: containerIface is in the container // network namespace and hostIface is in the host network namespace. func (pc *podConfigurator) setupInterfaces( @@ -78,6 +97,13 @@ func (pc *podConfigurator) setupInterfaces( containerIface.Sandbox = netns.Path() hostIface.Name = hostVeth.Name hostIface.Mac = hostVeth.HardwareAddr.String() + // OVS netdev datapath doesn't support TX checksum offloading, i.e. if packet + // arrives with bad/no checksum it will be sent to the output port with same bad/no checksum. + if pc.ovsDatapathType == ovsconfig.OVSDatapathNetdev { + if err := ethtool.EthtoolTXHWCsumOff(containerVeth.Name); err != nil { + return fmt.Errorf("error when disabling TX checksum offload on container veth: %v", err) + } + } return nil }); err != nil { return nil, nil, err @@ -264,21 +290,6 @@ func ParseOVSPortInterfaceConfig(portData *ovsconfig.OVSPortData, portConfig *in PodNamespace: podNamespace} } -type podConfigurator struct { - ovsBridgeClient ovsconfig.OVSBridgeClient - ofClient openflow.Client - ifaceStore interfacestore.InterfaceStore - gatewayMAC net.HardwareAddr -} - -func newPodConfigurator( - ovsBridgeClient ovsconfig.OVSBridgeClient, - ofClient openflow.Client, - ifaceStore interfacestore.InterfaceStore, - gatewayMAC net.HardwareAddr) *podConfigurator { - return &podConfigurator{ovsBridgeClient, ofClient, ifaceStore, gatewayMAC} -} - func (pc *podConfigurator) configureInterface( podName string, podNameSpace string, diff --git a/pkg/agent/cniserver/server.go b/pkg/agent/cniserver/server.go index 5c45ebdb222..b17d2a6124e 100644 --- a/pkg/agent/cniserver/server.go +++ b/pkg/agent/cniserver/server.go @@ -481,6 +481,7 @@ func (s *CNIServer) CmdCheck(ctx context.Context, request *cnipb.CniCmdRequest) func New( cniSocket, hostProcPathPrefix string, defaultMTU int, + ovsDatapathType string, nodeConfig *types.NodeConfig, ovsBridgeClient ovsconfig.OVSBridgeClient, ofClient openflow.Client, @@ -496,7 +497,7 @@ func New( defaultMTU: defaultMTU, kubeClient: kubeClient, containerAccess: newContainerAccessArbitrator(), - podConfigurator: newPodConfigurator(ovsBridgeClient, ofClient, ifaceStore, nodeConfig.GatewayConfig.MAC), + podConfigurator: newPodConfigurator(ovsBridgeClient, ofClient, ifaceStore, nodeConfig.GatewayConfig.MAC, ovsDatapathType), } } diff --git a/pkg/agent/util/ethtool/ethtool_linux.go b/pkg/agent/util/ethtool/ethtool_linux.go new file mode 100644 index 00000000000..f97bfed048e --- /dev/null +++ b/pkg/agent/util/ethtool/ethtool_linux.go @@ -0,0 +1,74 @@ +// Copyright 2019 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ethtool + +import ( + "fmt" + "syscall" + "unsafe" +) + +const ( + IFNAMSIZ = 16 // defined in linux/if.h + SIOCETHTOOL = 0x8946 // ethtool interface, defined in linux/sockios.h + ETHTOOL_STXCSUM = 0x00000017 // set TX hw csum enable, defined in linux/ethtool.h +) + +// defined in linux/if.h (struct ifreq) +type ifReq struct { + Name [IFNAMSIZ]byte + Data uintptr +} + +// defined in linux/ethtool.h (struct ethtool_value) +type ethtoolValue struct { + Cmd uint32 + Data uint32 +} + +// EthtoolTXHWCsumOff disables TX checksum offload on the specified interface. +func EthtoolTXHWCsumOff(name string) error { + if len(name)+1 > IFNAMSIZ { + return fmt.Errorf("name '%s' exceeds IFNAMSIZ (%d)", name, IFNAMSIZ) + } + + fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP) + if err != nil { + return fmt.Errorf("error when opening socket: %v", err) + } + defer syscall.Close(fd) + + value := ethtoolValue{ + Cmd: ETHTOOL_STXCSUM, + Data: 0, + } + request := ifReq{ + Data: uintptr(unsafe.Pointer(&value)), + } + copy(request.Name[:], []byte(name)) + + // We perform the call unconditionally: if TX checksum offload is already disabled the call + // will be a no-op and there will be no error. + if _, _, errno := syscall.RawSyscall( + syscall.SYS_IOCTL, + uintptr(fd), + uintptr(SIOCETHTOOL), + uintptr(unsafe.Pointer(&request)), + ); errno != 0 { + return fmt.Errorf("ioctl call failed: %v", errno) + } + + return nil +} diff --git a/test/e2e/README.md b/test/e2e/README.md index 13bbe33fe79..84a5b427340 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -98,6 +98,18 @@ By default the description and logs for Antrea Pods are only written to disk if test fails. You can choose to dump this information unconditionally with `--logs-export-on-success`. +## Running the e2e tests on a Kind cluster + +Refer to this [document](/docs/antrea.md) for instructions on how to create a +Kind cluster and use Antrea as the CNI. You need at least one control-plane +(master) Node and one worker Node. Before running the Go e2e tests, you will +also need to copy the Antrea manifest to the master Docker container: + +```bash +./hack/generate-manifest.sh --kind | docker exec -i kind-control-plane dd of=/root/antrea.yml +go test -v github.com/vmware-tanzu/antrea/test/e2e -provider=kind +``` + ## Tests to be added * Network policy tests diff --git a/test/e2e/basic_test.go b/test/e2e/basic_test.go index 2865823d9ca..5ab140957fa 100644 --- a/test/e2e/basic_test.go +++ b/test/e2e/basic_test.go @@ -97,7 +97,7 @@ func TestDeletePod(t *testing.T) { doesInterfaceExist := func() bool { cmd := fmt.Sprintf("ip link show %s", ifName) - if rc, _, _, err := RunSSHCommandOnNode(nodeName, cmd); err != nil { + if rc, _, _, err := RunCommandOnNode(nodeName, cmd); err != nil { t.Fatalf("Error when running ip command on Node '%s': %v", nodeName, err) } else { return rc == 0 diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go index 11c2e673b82..2849f6e300c 100644 --- a/test/e2e/fixtures.go +++ b/test/e2e/fixtures.go @@ -109,7 +109,7 @@ func exportLogs(t *testing.T, data *TestData) { // runKubectl runs the provided kubectl command on the master Node and returns the // output. It returns an empty string in case of error. runKubectl := func(cmd string) string { - rc, stdout, _, err := RunSSHCommandOnNode(masterNodeName(), cmd) + rc, stdout, _, err := RunCommandOnNode(masterNodeName(), cmd) if err != nil || rc != 0 { t.Errorf("Error when running this kubectl command on master Node: %s", cmd) return "" @@ -153,8 +153,9 @@ func exportLogs(t *testing.T, data *TestData) { // print a log message. If kubelet is not run with systemd, the log file will be empty. if err := forAllNodes(func(nodeName string) error { const numLines = 100 - cmd := fmt.Sprintf("journalctl -u kubelet -n %d", numLines) - rc, stdout, _, err := RunSSHCommandOnNode(nodeName, cmd) + // --no-pager ensures the command does not hang. + cmd := fmt.Sprintf("journalctl -u kubelet -n %d --no-pager", numLines) + rc, stdout, _, err := RunCommandOnNode(nodeName, cmd) if err != nil || rc != 0 { // return an error and skip subsequent Nodes return fmt.Errorf("error when running journalctl on Node '%s', is it available?", nodeName) diff --git a/test/e2e/framework.go b/test/e2e/framework.go index ed2d6c6689c..46da0173ec2 100755 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -113,6 +113,7 @@ func nodeName(idx int) string { func initProvider() error { providerFactory := map[string]func(string) (providers.ProviderInterface, error){ "vagrant": providers.NewVagrantProvider, + "kind": providers.NewKindProvider, } if fn, ok := providerFactory[testOptions.providerName]; ok { if newProvider, err := fn(testOptions.providerConfigPath); err != nil { @@ -126,14 +127,9 @@ func initProvider() error { return nil } -// A convenience wrapper around RunSSHCommand which runs the provided command on the node with name -// nodeName. -func RunSSHCommandOnNode(nodeName string, cmd string) (code int, stdout string, stderr string, err error) { - host, config, err := provider.GetSSHConfig(nodeName) - if err != nil { - return 0, "", "", fmt.Errorf("error when retrieving SSH config for node '%s': %v", nodeName, err) - } - return RunSSHCommand(host, config, cmd) +// RunCommandOnNode is a convenience wrapper around the Provider interface RunCommandOnNode method. +func RunCommandOnNode(nodeName string, cmd string) (code int, stdout string, stderr string, err error) { + return provider.RunCommandOnNode(nodeName, cmd) } func collectClusterInfo() error { @@ -180,9 +176,9 @@ func collectClusterInfo() error { // retrieve cluster CIDR if err := func() error { cmd := "kubectl cluster-info dump | grep cluster-cidr" - rc, stdout, _, err := RunSSHCommandOnNode(clusterInfo.masterNodeName, cmd) + rc, stdout, _, err := RunCommandOnNode(clusterInfo.masterNodeName, cmd) if err != nil || rc != 0 { - return fmt.Errorf("error when running the following command on master Node: %s", cmd) + return fmt.Errorf("error when running the following command on master Node: %s", stdout) } re := regexp.MustCompile(`cluster-cidr=([^"]+)`) if matches := re.FindStringSubmatch(stdout); len(matches) == 0 { @@ -249,16 +245,11 @@ func (data *TestData) deleteTestNamespace(timeout time.Duration) error { return err } -// deployAntrea deploys the Antrea DaemonSet using kubectl through an SSH session to the master node. +// deployAntrea deploys the Antrea DaemonSet using kubectl on the master node. func (data *TestData) deployAntrea() error { // TODO: use the K8s apiserver when server side apply is available? // See https://kubernetes.io/docs/reference/using-api/api-concepts/#server-side-apply - host, config, err := provider.GetSSHConfig(masterNodeName()) - if err != nil { - return fmt.Errorf("error when retrieving SSH config for master: %v", err) - } - cmd := fmt.Sprintf("kubectl apply -f ~/antrea.yml") - rc, _, _, err := RunSSHCommand(host, config, cmd) + rc, _, _, err := provider.RunCommandOnNode(masterNodeName(), "kubectl apply -f antrea.yml") if err != nil || rc != 0 { return fmt.Errorf("error when deploying Antrea; is antrea.yml available on the master Node?") } diff --git a/test/e2e/providers/exec/docker.go b/test/e2e/providers/exec/docker.go new file mode 100644 index 00000000000..d180396c8b0 --- /dev/null +++ b/test/e2e/providers/exec/docker.go @@ -0,0 +1,61 @@ +// Copyright 2019 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package exec + +import ( + "fmt" + "io/ioutil" + "os/exec" + "strings" +) + +// TODO: we could use the Docker Go SDK for this, but it seems like a big dependency to pull in just +// to run some "docker exec" commands. + +// RunDockerExecCommand runs the provided command on the specified host using "docker exec". Returns +// the exit code of the command, along with the contents of stdout and stderr as strings. Note that +// if the command returns a non-zero error code, this function does not report it as an error. +func RunDockerExecCommand(container string, cmd string, workdir string) ( + code int, stdout string, stderr string, err error, +) { + args := make([]string, 0) + args = append(args, "exec", "-w", workdir, "-t", container) + args = append(args, strings.Fields(cmd)...) + dockerCmd := exec.Command("docker", args...) + stdoutPipe, err := dockerCmd.StdoutPipe() + if err != nil { + return 0, "", "", fmt.Errorf("error when connecting to stdout: %v", err) + } + stderrPipe, err := dockerCmd.StderrPipe() + if err != nil { + return 0, "", "", fmt.Errorf("error when connecting to stderr: %v", err) + } + if err := dockerCmd.Start(); err != nil { + return 0, "", "", fmt.Errorf("error when starting command: %v", err) + } + + stdoutBytes, _ := ioutil.ReadAll(stdoutPipe) + stderrBytes, _ := ioutil.ReadAll(stderrPipe) + + if err := dockerCmd.Wait(); err != nil { + if e, ok := err.(*exec.ExitError); ok { + return e.ExitCode(), string(stdoutBytes), string(stderrBytes), nil + } + return 0, "", "", err + } + + // command is succesful + return 0, string(stdoutBytes), string(stderrBytes), nil +} diff --git a/test/e2e/ssh.go b/test/e2e/providers/exec/ssh.go similarity index 85% rename from test/e2e/ssh.go rename to test/e2e/providers/exec/ssh.go index 4a87eb6ed8f..d6b128a4ab4 100644 --- a/test/e2e/ssh.go +++ b/test/e2e/providers/exec/ssh.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package e2e +package exec import ( "bytes" @@ -21,9 +21,9 @@ import ( "golang.org/x/crypto/ssh" ) -// Run the provided SSH command on the specified host. Returns the exit code of the command, along -// with the contents of stdout and stderr as strings. Note that if the command returns a non-zero -// error code, this function does not report it as an error. +// RunSSHCommand runs the provided SSH command on the specified host. Returns the exit code of the +// command, along with the contents of stdout and stderr as strings. Note that if the command +// returns a non-zero error code, this function does not report it as an error. func RunSSHCommand(host string, config *ssh.ClientConfig, cmd string) (code int, stdout string, stderr string, err error) { client, err := ssh.Dial("tcp", host, config) if err != nil { diff --git a/test/e2e/providers/interface.go b/test/e2e/providers/interface.go index 0f30dd44f2c..7ca4224dd8f 100644 --- a/test/e2e/providers/interface.go +++ b/test/e2e/providers/interface.go @@ -14,13 +14,9 @@ package providers -import ( - "golang.org/x/crypto/ssh" -) - // Hides away specific characteristics of the K8s cluster. This should enable the same tests to be // run on a variety of providers. type ProviderInterface interface { - GetSSHConfig(name string) (string, *ssh.ClientConfig, error) + RunCommandOnNode(nodeName string, cmd string) (code int, stdout string, stderr string, err error) GetKubeconfigPath() (string, error) } diff --git a/test/e2e/providers/kind.go b/test/e2e/providers/kind.go new file mode 100644 index 00000000000..d287f4fc65e --- /dev/null +++ b/test/e2e/providers/kind.go @@ -0,0 +1,66 @@ +// Copyright 2019 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package providers + +import ( + "fmt" + "os" + "path" + + "github.com/vmware-tanzu/antrea/test/e2e/providers/exec" +) + +type KindProvider struct{} + +func (provider *KindProvider) RunCommandOnNode(nodeName string, cmd string) ( + code int, stdout string, stderr string, err error, +) { + return exec.RunDockerExecCommand(nodeName, cmd, "/root") +} + +func (provider *KindProvider) GetKubeconfigPath() (string, error) { + homeDir, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("error when retrieving user home directory: %v", err) + } + kubeconfigPath := path.Join(homeDir, ".kube", "config") + if _, err := os.Stat(kubeconfigPath); os.IsNotExist(err) { + return "", fmt.Errorf("Kubeconfig file not found at expected location '%s'", kubeconfigPath) + } + return kubeconfigPath, nil +} + +// enableKubectlOnMaster copies the Kubeconfig file on the Kind control-plane / master Node to the +// default location, in order to make sure that we can run kubectl on the Node. +func (provider *KindProvider) enableKubectlOnMaster() error { + // TODO: try not to hardcode this name if possible. + nodeName := "kind-control-plane" + rc, stdout, _, err := provider.RunCommandOnNode(nodeName, "cp /etc/kubernetes/admin.conf /root/.kube/config") + if err != nil || rc != 0 { + return fmt.Errorf("error when copying Kubeconfig file to /root/.kube/config on '%s': %s", nodeName, stdout) + } + return nil +} + +// NewKindProvider returns an implementation of ProviderInterface which is suitable for a +// Kubernetes test cluster created with Kind. +// configPath is unused for the kind provider +func NewKindProvider(configPath string) (ProviderInterface, error) { + provider := &KindProvider{} + if err := provider.enableKubectlOnMaster(); err != nil { + return nil, err + } + return provider, nil +} diff --git a/test/e2e/providers/vagrant.go b/test/e2e/providers/vagrant.go index eadde905ca2..f6ecde6e3c1 100644 --- a/test/e2e/providers/vagrant.go +++ b/test/e2e/providers/vagrant.go @@ -23,6 +23,8 @@ import ( "github.com/kevinburke/ssh_config" "golang.org/x/crypto/ssh" + + "github.com/vmware-tanzu/antrea/test/e2e/providers/exec" ) func vagrantPath() (string, error) { @@ -99,9 +101,7 @@ func convertConfig(inConfig *ssh_config.Config, name string) (string, *ssh.Clien return host, config, nil } -type VagrantProvider struct{} - -func (provider *VagrantProvider) GetSSHConfig(name string) (string, *ssh.ClientConfig, error) { +func GetSSHConfig(name string) (string, *ssh.ClientConfig, error) { // We convert the Vagrant ssh config file (generated by "vagrant ssh-config"), which is an // OpenSSH config file, to an instance of ssh.ClientConfig. sshConfig, err := importConfig() @@ -111,6 +111,18 @@ func (provider *VagrantProvider) GetSSHConfig(name string) (string, *ssh.ClientC return convertConfig(sshConfig, name) } +type VagrantProvider struct{} + +func (provider *VagrantProvider) RunCommandOnNode(nodeName string, cmd string) ( + code int, stdout string, stderr string, err error, +) { + host, config, err := GetSSHConfig(nodeName) + if err != nil { + return 0, "", "", fmt.Errorf("error when retrieving SSH config for node '%s': %v", nodeName, err) + } + return exec.RunSSHCommand(host, config, cmd) +} + func (provider *VagrantProvider) GetKubeconfigPath() (string, error) { vagrantPath, err := vagrantPath() if err != nil {