diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml new file mode 100644 index 00000000000..f5dd3190a40 --- /dev/null +++ b/.github/workflows/kind.yml @@ -0,0 +1,42 @@ +name: Kind +on: + pull_request: + branches: + - master + - release-* +jobs: + test-unit: + name: E2e tests on a Kind cluster on Linux + runs-on: [ubuntu-18.04] + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-go@v1 + with: + go-version: 1.12 + - name: Build Antrea image + run: make + - name: Install Kind + env: + KIND_VERSION: v0.6.0 + run: | + curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 + chmod +x ./kind + sudo mv kind /usr/local/bin + - name: Create Kind cluster + run: | + kind create cluster --config ci/kind/config.yml + kind get nodes | xargs ./hack/kind-linux.sh + - name: Deploy Antrea + # kubectl is installed on the Github Ubuntu 18.04 worker + run: | + kind load docker-image antrea/antrea-ubuntu:latest + ./hack/generate-manifest.sh --kind | kubectl apply -f - + - name: Printing some debug information + run: | + sleep 30 + kubectl get -A all + kubectl -n kube-system logs --all-containers -l app=antrea + - name: Run e2e tests + run: | + ./hack/generate-manifest.sh --kind | docker exec -i kind-control-plane dd of=/root/antrea.yml + go test github.com/vmware-tanzu/antrea/test/e2e -provider=kind diff --git a/build/images/ethtool/Dockerfile b/build/images/ethtool/Dockerfile new file mode 100644 index 00000000000..929289a0035 --- /dev/null +++ b/build/images/ethtool/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:18.04 + +LABEL maintainer="Antrea " +LABEL description="A Docker image based on Ubuntu 18.04 which includes ethtool and ip tools." + +RUN apt-get update && \ + apt-get install -y --no-install-recommends ethtool iproute2 && \ + rm -rf /var/cache/apt/* /var/lib/apt/lists/* diff --git a/build/images/ethtool/README.md b/build/images/ethtool/README.md new file mode 100644 index 00000000000..4e9f2f76ccf --- /dev/null +++ b/build/images/ethtool/README.md @@ -0,0 +1,16 @@ +# images/ethtool + +This Docker image is a very lightweight image based on Ubuntu 18.04 which +includes ethtool and the ip tools. + +If you need to build a new version of the image and push it to Dockerhub, you +can run the following: + +```bash +cd build/images/ethtool +docker build -t antrea/ethtool:latest . +docker push antrea/ethtool:latest +``` + +The `docker push` command will fail if you do not have permission to push to the +`antrea` Dockerhub repository. diff --git a/build/images/scripts/start_ovs_netdev b/build/images/scripts/start_ovs_netdev new file mode 100755 index 00000000000..2f55812aae3 --- /dev/null +++ b/build/images/scripts/start_ovs_netdev @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +source logging +source daemon_status + +CONTAINER_NAME="antrea-ovs" +OVS_DB_FILE="/var/run/openvswitch/conf.db" + +set -euo pipefail + +hwaddr=$(ip link show eth0 | grep link/ether | awk '{print $2}') +inet=$(ip addr show eth0 | grep "inet " | awk '{ print $2 }') +gw=$(ip route | grep default | awk '{ print $3 }') + +# Modify ovs-ctl so that the kernel module is no longer loaded since it is not +# needed when using OVS in userspace mode. It also enables running OVS with the +# netdev datapath type on platforms which do not have the OVS kernel module. +# This is easier than starting daemons manually... +function fix_ovs_ctl { + sed -i 's/\(\w*\)\(insert_mod_if_required || return 1\)/\1# \2/' /usr/share/openvswitch/scripts/ovs-ctl +} + +# See http://docs.openvswitch.org/en/latest/howto/userspace-tunneling/ +function add_br_phy { + log_info $CONTAINER_NAME "Creating OVS br-phy bridge for netdev datapath type" + ovs-vsctl --may-exist add-br br-phy \ + -- set Bridge br-phy datapath_type=netdev \ + -- br-set-external-id br-phy bridge-id br-phy \ + -- set bridge br-phy fail-mode=standalone \ + other_config:hwaddr="$hwaddr" + + ovs-vsctl --timeout 10 add-port br-phy eth0 + ip addr add "$inet" dev br-phy + ip link set br-phy up + ip addr flush dev eth0 2>/dev/null + ip link set eth0 up + ip route add default via "$gw" dev br-phy +} + +function del_br_phy { + log_info $CONTAINER_NAME "Deleting OVS br-phy bridge" + ovs-vsctl del-port br-phy eth0 + ovs-vsctl del-br br-phy + ip addr add "$inet" dev eth0 + ip link set eth0 up + ip route add default via "$gw" dev eth0 +} + +function start_ovs { + log_info $CONTAINER_NAME "Starting OVS" + /usr/share/openvswitch/scripts/ovs-ctl --system-id=random start --db-file=$OVS_DB_FILE +} + +function stop_ovs { + log_info $CONTAINER_NAME "Stopping OVS" + /usr/share/openvswitch/scripts/ovs-ctl stop +} + +SLEEP_PID= + +function quit { + log_info $CONTAINER_NAME "Stopping OVS before quit" + # delete the bridge and move IP address back to eth0 to restore connectivity + # when OVS is stopped. + del_br_phy + stop_ovs + # kill background sleep process + if [ "$SLEEP_PID" != "" ]; then kill $SLEEP_PID > /dev/null 2>&1 || true; fi + exit 0 +} + +# Do not trap EXIT as it would then ignore the "exit 0" statement in quit and +# exit with code 128 + SIGNAL +trap "quit" INT TERM + +fix_ovs_ctl + +start_ovs +add_br_phy + +log_info $CONTAINER_NAME "Started the loop that checks OVS status every 30 seconds" +while true; do + # we run sleep in the background so that we can immediately exit when we + # receive SIGINT / SIGTERM + # see https://stackoverflow.com/questions/32041674/linux-how-to-kill-sleep + sleep 30 & + SLEEP_PID=$! + wait $SLEEP_PID + + if ! check_ovs_status ; then + # OVS was stopped in the container. + log_warning $CONTAINER_NAME "OVS was stopped. Starting it again" + + start_ovs + fi +done diff --git a/build/yamls/patches/kind/startOvs.yml b/build/yamls/patches/kind/startOvs.yml new file mode 100644 index 00000000000..631f71ae4e6 --- /dev/null +++ b/build/yamls/patches/kind/startOvs.yml @@ -0,0 +1,10 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: antrea-agent +spec: + template: + spec: + containers: + - name: antrea-ovs + command: ["start_ovs_netdev"] diff --git a/ci/kind/config.yml b/ci/kind/config.yml new file mode 100644 index 00000000000..9d4e28bf9ee --- /dev/null +++ b/ci/kind/config.yml @@ -0,0 +1,8 @@ +kind: Cluster +apiVersion: kind.sigs.k8s.io/v1alpha3 +networking: + disableDefaultCNI: true + podSubnet: 10.10.0.0/16 +nodes: +- role: control-plane +- role: worker diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index e551b0e3c27..4bb9f667560 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -99,6 +99,7 @@ func run(o *Options) error { o.config.CNISocket, o.config.HostProcPathPrefix, o.config.DefaultMTU, + o.config.OVSDatapathType, nodeConfig, ovsBridgeClient, ofClient, diff --git a/docs/kind.md b/docs/kind.md index 440c7d84aa0..d33098457d1 100644 --- a/docs/kind.md +++ b/docs/kind.md @@ -1,5 +1,8 @@ # Deploying Antrea on a Kind cluster +At the moment Kind is only supported on Linux hosts. We are working on +supporting macOS hosts as well. + ## Create a Kind cluster and deploy Antrea in a few seconds ### Create a Kind cluster @@ -31,6 +34,8 @@ These instructions assume that you have built the `antrea/antrea-ubuntu` Docker image locally (e.g. by running `make` from the root of the repository). ```bash +# "fix" the host's veth interfaces (for the different Kind Nodes) +kind get nodes | xargs ./hack/kind-linux.sh # load the Antrea Docker image in the Nodes kind load docker-image antrea/antrea-ubuntu:latest # deploy Antrea @@ -39,7 +44,7 @@ kind load docker-image antrea/antrea-ubuntu:latest ### Check that everything is working -After a few seconds you sould be able to observe the following when running +After a few seconds you should be able to observe the following when running `kubectl get -n kube-system pods -l app=antrea`: ```bash NAME READY STATUS RESTARTS AGE @@ -65,3 +70,15 @@ requires some changes to the way Antrea is deployed. Most notably: (`netdev`) OVS datapath type is used * the Antrea agent's Init Container no longer needs to load the `openvswitch` kernel module + * the `start_ovs` script used by the `antrea-ovs` container needs to be + replaced with the `start_ovs_netdev` script, which creates an additional + bridge (`br-phy`) as required for [OVS userspace + tunneling](http://docs.openvswitch.org/en/latest/howto/userspace-tunneling/) + +### Why do I need to run the `hack/kind-linux.sh` script on my host? + +The script is required for Antrea to work properly in a Kind cluster on +Linux. It takes care of disabling TX hardware checksum offload for the veth +interface (in the host's network namespace) of each Kind Node. This is required +when using OVS in userspace mode. Refer to this [Antrea Github issue +#14](https://github.com/vmware-tanzu/antrea/issues/14) for more information. diff --git a/hack/generate-manifest.sh b/hack/generate-manifest.sh index 515e2d88041..1bcc44a2059 100755 --- a/hack/generate-manifest.sh +++ b/hack/generate-manifest.sh @@ -131,6 +131,9 @@ if $KIND; then $KUSTOMIZE edit add patch tunDevice.yml # edit antrea Agent configuration to use the netdev datapath $KUSTOMIZE edit add patch ovsDatapath.yml + # antrea-ovs should use start_ovs_netdev instead of start_ovs to ensure that the br_phy bridge + # is created. + $KUSTOMIZE edit add patch startOvs.yml # change initContainer script and remove SYS_MODULE capability $KUSTOMIZE edit add patch installCni.yml fi diff --git a/hack/kind-linux.sh b/hack/kind-linux.sh new file mode 100755 index 00000000000..c4a94a01b91 --- /dev/null +++ b/hack/kind-linux.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Copyright 2019 Antrea Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is required for Antrea to work properly in a Kind cluster on Linux. It takes care of +# disabling TX hardware checksum offload for the veth interface (in the host's network namespace) of +# each Kind Node. This is required when using OVS in userspace mode. Refer to +# https://github.com/vmware-tanzu/antrea/issues/14 for more information. + +# The script uses the antrea/ethtool Docker image (so that ethtool does not need to be installed on +# the Linux host). + +for node in "$@"; do + peerIdx=$(docker exec "$node" ip link | grep eth0 | awk -F[@:] '{ print $3 }' | cut -c 3-) + peerName=$(docker run --net=host antrea/ethtool:latest ip link | grep "$peerIdx": | awk -F[:@] '{ print $2 }' | cut -c 2-) + echo "Disabling TX checksum offload for node $node ($peerName)" + docker run --net=host --privileged antrea/ethtool:latest ethtool -K "$peerName" tx off +done diff --git a/pkg/agent/cniserver/pod_configuration.go b/pkg/agent/cniserver/pod_configuration.go index 76190d2e333..81a3e85389b 100644 --- a/pkg/agent/cniserver/pod_configuration.go +++ b/pkg/agent/cniserver/pod_configuration.go @@ -33,6 +33,7 @@ import ( "github.com/vmware-tanzu/antrea/pkg/agent/interfacestore" "github.com/vmware-tanzu/antrea/pkg/agent/openflow" "github.com/vmware-tanzu/antrea/pkg/agent/util" + "github.com/vmware-tanzu/antrea/pkg/agent/util/ethtool" "github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig" ) @@ -57,6 +58,24 @@ const ( ovsExternalIDPodNamespace = "pod-namespace" ) +type podConfigurator struct { + ovsBridgeClient ovsconfig.OVSBridgeClient + ofClient openflow.Client + ifaceStore interfacestore.InterfaceStore + gatewayMAC net.HardwareAddr + ovsDatapathType string +} + +func newPodConfigurator( + ovsBridgeClient ovsconfig.OVSBridgeClient, + ofClient openflow.Client, + ifaceStore interfacestore.InterfaceStore, + gatewayMAC net.HardwareAddr, + ovsDatapathType string, +) *podConfigurator { + return &podConfigurator{ovsBridgeClient, ofClient, ifaceStore, gatewayMAC, ovsDatapathType} +} + // setupInterfaces creates a veth pair: containerIface is in the container // network namespace and hostIface is in the host network namespace. func (pc *podConfigurator) setupInterfaces( @@ -78,6 +97,13 @@ func (pc *podConfigurator) setupInterfaces( containerIface.Sandbox = netns.Path() hostIface.Name = hostVeth.Name hostIface.Mac = hostVeth.HardwareAddr.String() + // OVS netdev datapath doesn't support TX checksum offloading, i.e. if packet + // arrives with bad/no checksum it will be sent to the output port with same bad/no checksum. + if pc.ovsDatapathType == ovsconfig.OVSDatapathNetdev { + if err := ethtool.EthtoolTXHWCsumOff(containerVeth.Name); err != nil { + return fmt.Errorf("error when disabling TX checksum offload on container veth: %v", err) + } + } return nil }); err != nil { return nil, nil, err @@ -264,21 +290,6 @@ func ParseOVSPortInterfaceConfig(portData *ovsconfig.OVSPortData, portConfig *in PodNamespace: podNamespace} } -type podConfigurator struct { - ovsBridgeClient ovsconfig.OVSBridgeClient - ofClient openflow.Client - ifaceStore interfacestore.InterfaceStore - gatewayMAC net.HardwareAddr -} - -func newPodConfigurator( - ovsBridgeClient ovsconfig.OVSBridgeClient, - ofClient openflow.Client, - ifaceStore interfacestore.InterfaceStore, - gatewayMAC net.HardwareAddr) *podConfigurator { - return &podConfigurator{ovsBridgeClient, ofClient, ifaceStore, gatewayMAC} -} - func (pc *podConfigurator) configureInterface( podName string, podNameSpace string, diff --git a/pkg/agent/cniserver/server.go b/pkg/agent/cniserver/server.go index 5c45ebdb222..b17d2a6124e 100644 --- a/pkg/agent/cniserver/server.go +++ b/pkg/agent/cniserver/server.go @@ -481,6 +481,7 @@ func (s *CNIServer) CmdCheck(ctx context.Context, request *cnipb.CniCmdRequest) func New( cniSocket, hostProcPathPrefix string, defaultMTU int, + ovsDatapathType string, nodeConfig *types.NodeConfig, ovsBridgeClient ovsconfig.OVSBridgeClient, ofClient openflow.Client, @@ -496,7 +497,7 @@ func New( defaultMTU: defaultMTU, kubeClient: kubeClient, containerAccess: newContainerAccessArbitrator(), - podConfigurator: newPodConfigurator(ovsBridgeClient, ofClient, ifaceStore, nodeConfig.GatewayConfig.MAC), + podConfigurator: newPodConfigurator(ovsBridgeClient, ofClient, ifaceStore, nodeConfig.GatewayConfig.MAC, ovsDatapathType), } } diff --git a/pkg/agent/util/ethtool/ethtool_linux.go b/pkg/agent/util/ethtool/ethtool_linux.go new file mode 100644 index 00000000000..f97bfed048e --- /dev/null +++ b/pkg/agent/util/ethtool/ethtool_linux.go @@ -0,0 +1,74 @@ +// Copyright 2019 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ethtool + +import ( + "fmt" + "syscall" + "unsafe" +) + +const ( + IFNAMSIZ = 16 // defined in linux/if.h + SIOCETHTOOL = 0x8946 // ethtool interface, defined in linux/sockios.h + ETHTOOL_STXCSUM = 0x00000017 // set TX hw csum enable, defined in linux/ethtool.h +) + +// defined in linux/if.h (struct ifreq) +type ifReq struct { + Name [IFNAMSIZ]byte + Data uintptr +} + +// defined in linux/ethtool.h (struct ethtool_value) +type ethtoolValue struct { + Cmd uint32 + Data uint32 +} + +// EthtoolTXHWCsumOff disables TX checksum offload on the specified interface. +func EthtoolTXHWCsumOff(name string) error { + if len(name)+1 > IFNAMSIZ { + return fmt.Errorf("name '%s' exceeds IFNAMSIZ (%d)", name, IFNAMSIZ) + } + + fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP) + if err != nil { + return fmt.Errorf("error when opening socket: %v", err) + } + defer syscall.Close(fd) + + value := ethtoolValue{ + Cmd: ETHTOOL_STXCSUM, + Data: 0, + } + request := ifReq{ + Data: uintptr(unsafe.Pointer(&value)), + } + copy(request.Name[:], []byte(name)) + + // We perform the call unconditionally: if TX checksum offload is already disabled the call + // will be a no-op and there will be no error. + if _, _, errno := syscall.RawSyscall( + syscall.SYS_IOCTL, + uintptr(fd), + uintptr(SIOCETHTOOL), + uintptr(unsafe.Pointer(&request)), + ); errno != 0 { + return fmt.Errorf("ioctl call failed: %v", errno) + } + + return nil +} diff --git a/test/e2e/README.md b/test/e2e/README.md index 13bbe33fe79..84a5b427340 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -98,6 +98,18 @@ By default the description and logs for Antrea Pods are only written to disk if test fails. You can choose to dump this information unconditionally with `--logs-export-on-success`. +## Running the e2e tests on a Kind cluster + +Refer to this [document](/docs/antrea.md) for instructions on how to create a +Kind cluster and use Antrea as the CNI. You need at least one control-plane +(master) Node and one worker Node. Before running the Go e2e tests, you will +also need to copy the Antrea manifest to the master Docker container: + +```bash +./hack/generate-manifest.sh --kind | docker exec -i kind-control-plane dd of=/root/antrea.yml +go test -v github.com/vmware-tanzu/antrea/test/e2e -provider=kind +``` + ## Tests to be added * Network policy tests diff --git a/test/e2e/basic_test.go b/test/e2e/basic_test.go index 2865823d9ca..5ab140957fa 100644 --- a/test/e2e/basic_test.go +++ b/test/e2e/basic_test.go @@ -97,7 +97,7 @@ func TestDeletePod(t *testing.T) { doesInterfaceExist := func() bool { cmd := fmt.Sprintf("ip link show %s", ifName) - if rc, _, _, err := RunSSHCommandOnNode(nodeName, cmd); err != nil { + if rc, _, _, err := RunCommandOnNode(nodeName, cmd); err != nil { t.Fatalf("Error when running ip command on Node '%s': %v", nodeName, err) } else { return rc == 0 diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go index 11c2e673b82..2849f6e300c 100644 --- a/test/e2e/fixtures.go +++ b/test/e2e/fixtures.go @@ -109,7 +109,7 @@ func exportLogs(t *testing.T, data *TestData) { // runKubectl runs the provided kubectl command on the master Node and returns the // output. It returns an empty string in case of error. runKubectl := func(cmd string) string { - rc, stdout, _, err := RunSSHCommandOnNode(masterNodeName(), cmd) + rc, stdout, _, err := RunCommandOnNode(masterNodeName(), cmd) if err != nil || rc != 0 { t.Errorf("Error when running this kubectl command on master Node: %s", cmd) return "" @@ -153,8 +153,9 @@ func exportLogs(t *testing.T, data *TestData) { // print a log message. If kubelet is not run with systemd, the log file will be empty. if err := forAllNodes(func(nodeName string) error { const numLines = 100 - cmd := fmt.Sprintf("journalctl -u kubelet -n %d", numLines) - rc, stdout, _, err := RunSSHCommandOnNode(nodeName, cmd) + // --no-pager ensures the command does not hang. + cmd := fmt.Sprintf("journalctl -u kubelet -n %d --no-pager", numLines) + rc, stdout, _, err := RunCommandOnNode(nodeName, cmd) if err != nil || rc != 0 { // return an error and skip subsequent Nodes return fmt.Errorf("error when running journalctl on Node '%s', is it available?", nodeName) diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 58f33f3b036..ec899b4bdd2 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -113,6 +113,7 @@ func nodeName(idx int) string { func initProvider() error { providerFactory := map[string]func(string) (providers.ProviderInterface, error){ "vagrant": providers.NewVagrantProvider, + "kind": providers.NewKindProvider, } if fn, ok := providerFactory[testOptions.providerName]; ok { if newProvider, err := fn(testOptions.providerConfigPath); err != nil { @@ -126,14 +127,9 @@ func initProvider() error { return nil } -// A convenience wrapper around RunSSHCommand which runs the provided command on the node with name -// nodeName. -func RunSSHCommandOnNode(nodeName string, cmd string) (code int, stdout string, stderr string, err error) { - host, config, err := provider.GetSSHConfig(nodeName) - if err != nil { - return 0, "", "", fmt.Errorf("error when retrieving SSH config for node '%s': %v", nodeName, err) - } - return RunSSHCommand(host, config, cmd) +// RunCommandOnNode is a convenience wrapper around the Provider interface RunCommandOnNode method. +func RunCommandOnNode(nodeName string, cmd string) (code int, stdout string, stderr string, err error) { + return provider.RunCommandOnNode(nodeName, cmd) } func collectClusterInfo() error { @@ -180,9 +176,9 @@ func collectClusterInfo() error { // retrieve cluster CIDR if err := func() error { cmd := "kubectl cluster-info dump | grep cluster-cidr" - rc, stdout, _, err := RunSSHCommandOnNode(clusterInfo.masterNodeName, cmd) + rc, stdout, _, err := RunCommandOnNode(clusterInfo.masterNodeName, cmd) if err != nil || rc != 0 { - return fmt.Errorf("error when running the following command on master Node: %s", cmd) + return fmt.Errorf("error when running the following command on master Node: %s", stdout) } re := regexp.MustCompile(`cluster-cidr=([^"]+)`) if matches := re.FindStringSubmatch(stdout); len(matches) == 0 { @@ -249,16 +245,11 @@ func (data *TestData) deleteTestNamespace(timeout time.Duration) error { return err } -// deployAntrea deploys the Antrea DaemonSet using kubectl through an SSH session to the master node. +// deployAntrea deploys the Antrea DaemonSet using kubectl on the master node. func (data *TestData) deployAntrea() error { // TODO: use the K8s apiserver when server side apply is available? // See https://kubernetes.io/docs/reference/using-api/api-concepts/#server-side-apply - host, config, err := provider.GetSSHConfig(masterNodeName()) - if err != nil { - return fmt.Errorf("error when retrieving SSH config for master: %v", err) - } - cmd := fmt.Sprintf("kubectl apply -f ~/antrea.yml") - rc, _, _, err := RunSSHCommand(host, config, cmd) + rc, _, _, err := provider.RunCommandOnNode(masterNodeName(), "kubectl apply -f antrea.yml") if err != nil || rc != 0 { return fmt.Errorf("error when deploying Antrea; is antrea.yml available on the master Node?") } diff --git a/test/e2e/providers/exec/docker.go b/test/e2e/providers/exec/docker.go new file mode 100644 index 00000000000..d180396c8b0 --- /dev/null +++ b/test/e2e/providers/exec/docker.go @@ -0,0 +1,61 @@ +// Copyright 2019 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package exec + +import ( + "fmt" + "io/ioutil" + "os/exec" + "strings" +) + +// TODO: we could use the Docker Go SDK for this, but it seems like a big dependency to pull in just +// to run some "docker exec" commands. + +// RunDockerExecCommand runs the provided command on the specified host using "docker exec". Returns +// the exit code of the command, along with the contents of stdout and stderr as strings. Note that +// if the command returns a non-zero error code, this function does not report it as an error. +func RunDockerExecCommand(container string, cmd string, workdir string) ( + code int, stdout string, stderr string, err error, +) { + args := make([]string, 0) + args = append(args, "exec", "-w", workdir, "-t", container) + args = append(args, strings.Fields(cmd)...) + dockerCmd := exec.Command("docker", args...) + stdoutPipe, err := dockerCmd.StdoutPipe() + if err != nil { + return 0, "", "", fmt.Errorf("error when connecting to stdout: %v", err) + } + stderrPipe, err := dockerCmd.StderrPipe() + if err != nil { + return 0, "", "", fmt.Errorf("error when connecting to stderr: %v", err) + } + if err := dockerCmd.Start(); err != nil { + return 0, "", "", fmt.Errorf("error when starting command: %v", err) + } + + stdoutBytes, _ := ioutil.ReadAll(stdoutPipe) + stderrBytes, _ := ioutil.ReadAll(stderrPipe) + + if err := dockerCmd.Wait(); err != nil { + if e, ok := err.(*exec.ExitError); ok { + return e.ExitCode(), string(stdoutBytes), string(stderrBytes), nil + } + return 0, "", "", err + } + + // command is succesful + return 0, string(stdoutBytes), string(stderrBytes), nil +} diff --git a/test/e2e/ssh.go b/test/e2e/providers/exec/ssh.go similarity index 85% rename from test/e2e/ssh.go rename to test/e2e/providers/exec/ssh.go index f22d5d933e8..50df2755d68 100644 --- a/test/e2e/ssh.go +++ b/test/e2e/providers/exec/ssh.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package e2e +package exec import ( "bytes" @@ -21,9 +21,9 @@ import ( "golang.org/x/crypto/ssh" ) -// Run the provided SSH command on the specified host. Returns the exit code of the command, along -// with the contents of stdout and stderr as strings. Note that if the command returns a non-zero -// error code, this function does not report it as an error. +// RunSSHCommand runs the provided SSH command on the specified host. Returns the exit code of the +// command, along with the contents of stdout and stderr as strings. Note that if the command +// returns a non-zero error code, this function does not report it as an error. func RunSSHCommand(host string, config *ssh.ClientConfig, cmd string) (code int, stdout string, stderr string, err error) { client, err := ssh.Dial("tcp", host, config) if err != nil { diff --git a/test/e2e/providers/interface.go b/test/e2e/providers/interface.go index 0f30dd44f2c..7ca4224dd8f 100644 --- a/test/e2e/providers/interface.go +++ b/test/e2e/providers/interface.go @@ -14,13 +14,9 @@ package providers -import ( - "golang.org/x/crypto/ssh" -) - // Hides away specific characteristics of the K8s cluster. This should enable the same tests to be // run on a variety of providers. type ProviderInterface interface { - GetSSHConfig(name string) (string, *ssh.ClientConfig, error) + RunCommandOnNode(nodeName string, cmd string) (code int, stdout string, stderr string, err error) GetKubeconfigPath() (string, error) } diff --git a/test/e2e/providers/kind.go b/test/e2e/providers/kind.go new file mode 100644 index 00000000000..d287f4fc65e --- /dev/null +++ b/test/e2e/providers/kind.go @@ -0,0 +1,66 @@ +// Copyright 2019 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package providers + +import ( + "fmt" + "os" + "path" + + "github.com/vmware-tanzu/antrea/test/e2e/providers/exec" +) + +type KindProvider struct{} + +func (provider *KindProvider) RunCommandOnNode(nodeName string, cmd string) ( + code int, stdout string, stderr string, err error, +) { + return exec.RunDockerExecCommand(nodeName, cmd, "/root") +} + +func (provider *KindProvider) GetKubeconfigPath() (string, error) { + homeDir, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("error when retrieving user home directory: %v", err) + } + kubeconfigPath := path.Join(homeDir, ".kube", "config") + if _, err := os.Stat(kubeconfigPath); os.IsNotExist(err) { + return "", fmt.Errorf("Kubeconfig file not found at expected location '%s'", kubeconfigPath) + } + return kubeconfigPath, nil +} + +// enableKubectlOnMaster copies the Kubeconfig file on the Kind control-plane / master Node to the +// default location, in order to make sure that we can run kubectl on the Node. +func (provider *KindProvider) enableKubectlOnMaster() error { + // TODO: try not to hardcode this name if possible. + nodeName := "kind-control-plane" + rc, stdout, _, err := provider.RunCommandOnNode(nodeName, "cp /etc/kubernetes/admin.conf /root/.kube/config") + if err != nil || rc != 0 { + return fmt.Errorf("error when copying Kubeconfig file to /root/.kube/config on '%s': %s", nodeName, stdout) + } + return nil +} + +// NewKindProvider returns an implementation of ProviderInterface which is suitable for a +// Kubernetes test cluster created with Kind. +// configPath is unused for the kind provider +func NewKindProvider(configPath string) (ProviderInterface, error) { + provider := &KindProvider{} + if err := provider.enableKubectlOnMaster(); err != nil { + return nil, err + } + return provider, nil +} diff --git a/test/e2e/providers/vagrant.go b/test/e2e/providers/vagrant.go index eadde905ca2..f6ecde6e3c1 100644 --- a/test/e2e/providers/vagrant.go +++ b/test/e2e/providers/vagrant.go @@ -23,6 +23,8 @@ import ( "github.com/kevinburke/ssh_config" "golang.org/x/crypto/ssh" + + "github.com/vmware-tanzu/antrea/test/e2e/providers/exec" ) func vagrantPath() (string, error) { @@ -99,9 +101,7 @@ func convertConfig(inConfig *ssh_config.Config, name string) (string, *ssh.Clien return host, config, nil } -type VagrantProvider struct{} - -func (provider *VagrantProvider) GetSSHConfig(name string) (string, *ssh.ClientConfig, error) { +func GetSSHConfig(name string) (string, *ssh.ClientConfig, error) { // We convert the Vagrant ssh config file (generated by "vagrant ssh-config"), which is an // OpenSSH config file, to an instance of ssh.ClientConfig. sshConfig, err := importConfig() @@ -111,6 +111,18 @@ func (provider *VagrantProvider) GetSSHConfig(name string) (string, *ssh.ClientC return convertConfig(sshConfig, name) } +type VagrantProvider struct{} + +func (provider *VagrantProvider) RunCommandOnNode(nodeName string, cmd string) ( + code int, stdout string, stderr string, err error, +) { + host, config, err := GetSSHConfig(nodeName) + if err != nil { + return 0, "", "", fmt.Errorf("error when retrieving SSH config for node '%s': %v", nodeName, err) + } + return exec.RunSSHCommand(host, config, cmd) +} + func (provider *VagrantProvider) GetKubeconfigPath() (string, error) { vagrantPath, err := vagrantPath() if err != nil {