CI: integrate hubble-relay and add initial hubble-relay tests

Makefile

@@ -386,6 +386,7 @@ docker-cilium-builder-manifest:
 docker-hubble-relay-image:
 	$(QUIET)$(CONTAINER_ENGINE) build \
 		--build-arg NOSTRIP=${NOSTRIP} \
+		--build-arg CILIUM_SHA=$(firstword $(GIT_VERSION)) \
 		-f hubble-relay.Dockerfile \
 		-t "cilium/hubble-relay:$(DOCKER_IMAGE_TAG)" .
 	$(QUIET)$(CONTAINER_ENGINE) tag cilium/hubble-relay:$(DOCKER_IMAGE_TAG) cilium/hubble-relay:$(DOCKER_IMAGE_TAG)-${GOARCH}

hubble-relay.Dockerfile

@@ -1,13 +1,19 @@
 FROM docker.io/library/golang:1.14.3 as builder
+ARG CILIUM_SHA=""
+LABEL cilium-sha=${CILIUM_SHA}
 ADD . /go/src/github.com/cilium/cilium
 WORKDIR /go/src/github.com/cilium/cilium/hubble-relay
 ARG NOSTRIP
 RUN make NOSTRIP=$NOSTRIP
 
 FROM docker.io/library/alpine:3.11 as certs
+ARG CILIUM_SHA=""
+LABEL cilium-sha=${CILIUM_SHA}
 RUN apk --update add ca-certificates
 
 FROM docker.io/library/golang:1.14.3 as gops
+ARG CILIUM_SHA=""
+LABEL cilium-sha=${CILIUM_SHA}
 RUN go get -d github.com/google/gops && \
     cd /go/src/github.com/google/gops && \
     git checkout -b v0.3.6 v0.3.6 && \
@@ -17,6 +23,8 @@ RUN go get -d github.com/google/gops && \
     strip /go/bin/gops
 
 FROM scratch
+ARG CILIUM_SHA=""
+LABEL cilium-sha=${CILIUM_SHA}
 LABEL maintainer="maintainer@cilium.io"
 COPY --from=builder /go/src/github.com/cilium/cilium/hubble-relay/hubble-relay /usr/bin/hubble-relay
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

test/Vagrantfile

@@ -21,6 +21,7 @@ $CONTAINER_RUNTIME=(ENV['CONTAINER_RUNTIME'] || "docker")
 $CNI_INTEGRATION=(ENV['CNI_INTEGRATION'] || "")
 $CILIUM_IMAGE = ENV['CILIUM_IMAGE'] || ""
 $CILIUM_OPERATOR_IMAGE = ENV['CILIUM_OPERATOR_IMAGE'] || ""
+$HUBBLE_RELAY_IMAGE = ENV['HUBBLE_RELAY_IMAGE'] || ""
 $CILIUM_REGISTRY = ENV['CILIUM_REGISTRY'] || ""
 $PRELOAD_VM = ENV['PRELOAD_VM'] || "false"
 $SKIP_K8S_PROVISION = ENV['SKIP_K8S_PROVISION'] || "false"
@@ -134,6 +135,7 @@ Vagrant.configure("2") do |config|
                     "#{$IPv6}", "#{$CONTAINER_RUNTIME}", "#{$CNI_INTEGRATION}"]
                 sh.env = {"CILIUM_IMAGE" => "#{$CILIUM_IMAGE}",
                           "CILIUM_OPERATOR_IMAGE" => "#{$CILIUM_OPERATOR_IMAGE}",
+                          "HUBBLE_RELAY_IMAGE" => "#{$HUBBLE_RELAY_IMAGE}",
                           "CILIUM_REGISTRY" => "#{$CILIUM_REGISTRY}",
                           "PRELOAD_VM" => "#{$PRELOAD_VM}",
                           "SKIP_K8S_PROVISION" => "#{$SKIP_K8S_PROVISION}",

test/clean-local-registry-tag.sh

@@ -11,13 +11,14 @@ fi
 
 BUSYBOX_VERSION=1.31.1
 
-docker pull docker.io/library/busybox:$BUSYBOX_VERSION
+docker pull "docker.io/library/busybox:$BUSYBOX_VERSION"
 
-docker tag busybox:$BUSYBOX_VERSION $1/cilium/cilium:$2
-docker tag busybox:$BUSYBOX_VERSION $1/cilium/cilium-dev:$2
-docker tag busybox:$BUSYBOX_VERSION $1/cilium/operator:$2
-
-docker push $1/cilium/cilium:$2
-docker push $1/cilium/cilium-dev:$2
-docker push $1/cilium/operator:$2
+docker tag "busybox:${BUSYBOX_VERSION}" "$1/cilium/cilium:$2"
+docker tag "busybox:${BUSYBOX_VERSION}" "$1/cilium/cilium-dev:$2"
+docker tag "busybox:${BUSYBOX_VERSION}" "$1/cilium/operator:$2"
+docker tag "busybox:${BUSYBOX_VERSION}" "$1/cilium/hubble-relay:$2"
 
+docker push "$1/cilium/cilium:$2"
+docker push "$1/cilium/cilium-dev:$2"
+docker push "$1/cilium/operator:$2"
+docker push "$1/cilium/hubble-relay:$2"

test/config/config.go

@@ -34,6 +34,7 @@ type CiliumTestConfigType struct {
 	SkipLogGathering    bool
 	CiliumImage         string
 	CiliumOperatorImage string
+	HubbleRelayImage    string
 	ProvisionK8s        bool
 	Timeout             time.Duration
 	Kubeconfig          string
@@ -65,6 +66,8 @@ func (c *CiliumTestConfigType) ParseFlags() {
 		"Specifies which image of cilium to use during tests")
 	flag.StringVar(&c.CiliumOperatorImage, "cilium.operator-image", "",
 		"Specifies which image of cilium-operator to use during tests")
+	flag.StringVar(&c.HubbleRelayImage, "cilium.hubble-relay-image", "",
+		"Specifies which image of hubble-relay to use during tests")
 	flag.BoolVar(&c.ProvisionK8s, "cilium.provision-k8s", true,
 		"Specifies whether Kubernetes should be deployed and installed via kubeadm or not")
 	flag.DurationVar(&c.Timeout, "cilium.timeout", 24*time.Hour,

test/helpers/kubectl.go

@@ -187,6 +187,10 @@ func Init() {
 		os.Setenv("CILIUM_OPERATOR_IMAGE", config.CiliumTestConfig.CiliumOperatorImage)
 	}
 
+	if config.CiliumTestConfig.HubbleRelayImage != "" {
+		os.Setenv("HUBBLE_RELAY_IMAGE", config.CiliumTestConfig.HubbleRelayImage)
+	}
+
 	if config.CiliumTestConfig.Registry != "" {
 		os.Setenv("CILIUM_REGISTRY", config.CiliumTestConfig.Registry)
 	}

test/k8sT/assertionHelpers.go

@@ -56,6 +56,22 @@ func ExpectCiliumOperatorReady(vm *helpers.Kubectl) {
 	ExpectWithOffset(1, err).Should(BeNil(), "Cilium operator was not able to get into ready state")
 }
 
+// ExpectHubbleCLIReady is a wrapper around helpers/WaitForPods. It asserts
+// that the error returned by that function is nil.
+func ExpectHubbleCLIReady(vm *helpers.Kubectl, ns string) {
+	By("Waiting for hubble-cli to be ready")
+	err := vm.WaitforPods(ns, "-l k8s-app=hubble-cli", longTimeout)
+	ExpectWithOffset(1, err).Should(BeNil(), "hubble-cli was not able to get into ready state")
+}
+
+// ExpectHubbleRelayReady is a wrapper around helpers/WaitForPods. It asserts
+// that the error returned by that function is nil.
+func ExpectHubbleRelayReady(vm *helpers.Kubectl, ns string) {
+	By("Waiting for hubble-relay to be ready")
+	err := vm.WaitforPods(ns, "-l k8s-app=hubble-relay", longTimeout)
+	ExpectWithOffset(1, err).Should(BeNil(), "hubble-relay was not able to get into ready state")
+}
+
 // ExpectAllPodsTerminated is a wrapper around helpers/WaitCleanAllTerminatingPods.
 // It asserts that the error returned by that function is nil.
 func ExpectAllPodsTerminated(vm *helpers.Kubectl) {

test/k8sT/hubble.go

@@ -18,8 +18,10 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"strconv"
 	"strings"
 
+	"github.com/asaskevich/govalidator"
 	"github.com/cilium/cilium/pkg/annotation"
 	. "github.com/cilium/cilium/test/ginkgo-ext"
 	"github.com/cilium/cilium/test/helpers"
@@ -35,8 +37,9 @@ var _ = Describe("K8sHubbleTest", func() {
 
 		hubblePodK8s1 string
 
-		hubbleSelector  = "-l k8s-app=hubble-cli"
-		hubbleNamespace = helpers.CiliumNamespace
+		hubbleNamespace    = helpers.CiliumNamespace
+		hubbleRelayService = "hubble-relay"
+		hubbleRelayAddress string
 
 		demoPath string
 
@@ -86,16 +89,22 @@ var _ = Describe("K8sHubbleTest", func() {
 		demoPath = helpers.ManifestGet(kubectl.BasePath(), "demo.yaml")
 
 		DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, map[string]string{
-			"global.hubble.cli.enabled":     "true",
 			"global.hubble.metricsServer":   fmt.Sprintf(":%s", prometheusPort),
 			"global.hubble.metrics.enabled": `"{dns:query;ignoreAAAA,drop,tcp,flow,port-distribution,icmp,http}"`,
+			"global.hubble.cli.enabled":     "true",
+			"global.hubble.relay.enabled":   "true",
 		})
 
-		err := kubectl.WaitforPods(hubbleNamespace, hubbleSelector, helpers.HelperTimeout)
-		Expect(err).Should(BeNil(), "hubble-cli pods did not become ready")
-
+		var err error
+		ExpectHubbleCLIReady(kubectl, hubbleNamespace)
 		hubblePodK8s1, err = kubectl.GetHubbleClientPodOnNodeWithLabel(hubbleNamespace, helpers.K8s1)
 		Expect(err).Should(BeNil(), "unable to find hubble-cli pod on %s", helpers.K8s1)
+
+		ExpectHubbleRelayReady(kubectl, hubbleNamespace)
+		hubbleRelayIP, hubbleRelayPort, err := kubectl.GetServiceHostPort(hubbleNamespace, hubbleRelayService)
+		Expect(err).Should(BeNil(), "Cannot get service %s", hubbleRelayService)
+		Expect(govalidator.IsIP(hubbleRelayIP)).Should(BeTrue(), "hubbleRelayIP is not an IP")
+		hubbleRelayAddress = net.JoinHostPort(hubbleRelayIP, strconv.Itoa(hubbleRelayPort))
 	})
 
 	AfterFailed(func() {
@@ -169,6 +178,21 @@ var _ = Describe("K8sHubbleTest", func() {
 			res.ExpectContains(`hubble_flows_processed_total{subtype="to-endpoint",type="Trace",verdict="FORWARDED"}`)
 		})
 
+		It("Test L3/L4 Flow with hubble-relay", func() {
+			ctx, cancel := context.WithTimeout(context.Background(), helpers.MidCommandTimeout)
+			defer cancel()
+			follow := kubectl.HubbleObserveFollow(ctx, hubbleNamespace, hubblePodK8s1, fmt.Sprintf(
+				"--server %s --last 1 --type trace --from-pod %s/%s --to-namespace %s --to-label %s --to-port %d",
+				hubbleRelayAddress, namespaceForTest, appPods[helpers.App2], namespaceForTest, app1Labels, app1Port))
+
+			res := kubectl.ExecPodCmd(namespaceForTest, appPods[helpers.App2],
+				helpers.CurlFail(fmt.Sprintf("http://%s/public", app1ClusterIP)))
+			res.ExpectSuccess("%q cannot curl clusterIP %q", appPods[helpers.App2], app1ClusterIP)
+
+			err := follow.WaitUntilMatchFilterLineTimeout(`{$.Type}`, "L3_L4", helpers.ShortCommandTimeout)
+			Expect(err).To(BeNil(), fmt.Sprintf("hubble observe query timed out on %q", follow.OutputPrettyPrint()))
+		})
+
 		It("Test L7 Flow", func() {
 			addVisibilityAnnotation(namespaceForTest, app1Labels, "Ingress", "80", "TCP", "HTTP")
 			defer removeVisbilityAnnotation(namespaceForTest, app1Labels)
@@ -193,5 +217,23 @@ var _ = Describe("K8sHubbleTest", func() {
 			res.ExpectSuccess("%s/%s cannot curl metrics %q", hubbleNamespace, hubblePodK8s1, app1ClusterIP)
 			res.ExpectContains(`hubble_flows_processed_total{subtype="HTTP",type="L7",verdict="FORWARDED"}`)
 		})
+
+		It("Test L7 Flow with hubble-relay", func() {
+			addVisibilityAnnotation(namespaceForTest, app1Labels, "Ingress", "80", "TCP", "HTTP")
+			defer removeVisbilityAnnotation(namespaceForTest, app1Labels)
+
+			ctx, cancel := context.WithTimeout(context.Background(), helpers.MidCommandTimeout)
+			defer cancel()
+			follow := kubectl.HubbleObserveFollow(ctx, hubbleNamespace, hubblePodK8s1, fmt.Sprintf(
+				"--server %s --last 1 --type l7 --from-pod %s/%s --to-namespace %s --to-label %s --protocol http",
+				hubbleRelayAddress, namespaceForTest, appPods[helpers.App2], namespaceForTest, app1Labels))
+
+			res := kubectl.ExecPodCmd(namespaceForTest, appPods[helpers.App2],
+				helpers.CurlFail(fmt.Sprintf("http://%s/public", app1ClusterIP)))
+			res.ExpectSuccess("%q cannot curl clusterIP %q", appPods[helpers.App2], app1ClusterIP)
+
+			err := follow.WaitUntilMatchFilterLineTimeout(`{$.Type}`, "L7", helpers.ShortCommandTimeout)
+			Expect(err).To(BeNil(), fmt.Sprintf("hubble observe query timed out on %q", follow.OutputPrettyPrint()))
+		})
 	})
 })

test/make-images-push-to-local-registry.sh

@@ -3,15 +3,17 @@
 set -e
 
 cd ..
-make docker-image DOCKER_IMAGE_TAG=$2
+make docker-image DOCKER_IMAGE_TAG="$2"
 
-docker tag cilium/cilium:$2 $1/cilium/cilium:$2
-docker tag cilium/cilium:$2 $1/cilium/cilium-dev:$2
-docker tag cilium/operator:$2 $1/cilium/operator:$2
+docker tag "cilium/cilium:$2" "$1/cilium/cilium:$2"
+docker tag "cilium/cilium:$2" "$1/cilium/cilium-dev:$2"
+docker tag "cilium/operator:$2" "$1/cilium/operator:$2"
+docker tag "cilium/hubble-relay:$2" "$1/cilium/hubble-relay:$2"
 
-docker push $1/cilium/cilium:$2
-docker push $1/cilium/cilium-dev:$2
-docker push $1/cilium/operator:$2
+docker push "$1/cilium/cilium:$2"
+docker push "$1/cilium/cilium-dev:$2"
+docker push "$1/cilium/operator:$2"
+docker push "$1/cilium/hubble-relay:$2"
 
 cilium_git_version="$(cat GIT_VERSION)"
 

test/provision/compile.sh

@@ -9,6 +9,7 @@ GOPATH="/home/vagrant/go"
 REGISTRY="k8s1:5000"
 CILIUM_TAG="cilium/cilium-dev"
 CILIUM_OPERATOR_TAG="cilium/operator"
+HUBBLE_RELAY_TAG="cilium/hubble-relay"
 
 DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 
@@ -28,44 +29,57 @@ then
     # Only need to build on one host, since we can pull from the other host.
     if [[ "$(hostname)" == "k8s1" && "${CILIUM_REGISTRY}" == "" ]]; then
       ./test/provision/container-images.sh cilium_images .
-      if [[ "${CILIUM_IMAGE}" == "" && "${CILIUM_OPERATOR_IMAGE}" == "" ]]; then
-        echo "building cilium/cilium container image..."
+
+      if [[ "${CILIUM_IMAGE}" == "" ]]; then
+        echo "building cilium container image..."
         make LOCKDEBUG=1 docker-image-no-clean
+        echo "tagging cilium image..."
+        docker tag cilium/cilium "${REGISTRY}/${CILIUM_TAG}"
+        echo "pushing cilium image to ${REGISTRY}/${CILIUM_TAG}..."
+        docker push "${REGISTRY}/${CILIUM_TAG}"
+        echo "removing local cilium image..."
+        docker rmi cilium/cilium:latest
+      else
+        pull_image_and_push_to_local_registry "${CILIUM_IMAGE}" "${REGISTRY}" "${CILIUM_TAG}"
+      fi
 
-        echo "building cilium/operator container image..."
-        make LOCKDEBUG=1 docker-operator-image&
-        export OPERATORPID=$!
+      if [[ "${CILIUM_OPERATOR_IMAGE}" == "" ]]; then
+        echo "building cilium-operator image..."
+        make LOCKDEBUG=1 docker-operator-image
+        echo "tagging cilium-operator image..."
+        docker tag "${CILIUM_OPERATOR_TAG}" "${REGISTRY}/${CILIUM_OPERATOR_TAG}"
+        echo "pushing cilium/operator image to ${REGISTRY}/${CILIUM_OPERATOR_TAG}..."
+        docker push "${REGISTRY}/${CILIUM_OPERATOR_TAG}"
+        echo "removing local cilium-operator image..."
+        docker rmi "${CILIUM_OPERATOR_TAG}:latest"
+      else
+        pull_image_and_push_to_local_registry "${CILIUM_OPERATOR_IMAGE}" "${REGISTRY}" "${CILIUM_OPERATOR_TAG}"
+      fi
 
-        echo "pushing cilium/cilium image to k8s1:5000/cilium/cilium-dev..."
-        docker tag cilium/cilium k8s1:5000/cilium/cilium-dev
-        docker rmi cilium/cilium:latest
-        docker push k8s1:5000/cilium/cilium-dev
+      delete_cilium_pods
 
-        wait $OPERATORPID
-        echo "pushing cilium/operator image to k8s1:5000/cilium/operator..."
-        docker tag cilium/operator k8s1:5000/cilium/operator
-        docker push k8s1:5000/cilium/operator
-        delete_cilium_pods
-      elif [[ "${CILIUM_IMAGE}" != "" && "${CILIUM_OPERATOR_IMAGE}" == "" ]]; then
-        pull_image_and_push_to_local_registry ${CILIUM_IMAGE} ${REGISTRY} ${CILIUM_TAG}
-        build_operator_image
-        delete_cilium_pods
-      elif [[ "${CILIUM_IMAGE}" == "" && "${CILIUM_OPERATOR_IMAGE}" != "" ]]; then
-        pull_image_and_push_to_local_registry ${CILIUM_OPERATOR_IMAGE} ${REGISTRY} ${CILIUM_OPERATOR_TAG}
-        build_cilium_image
-        delete_cilium_pods
+      if [[ "${HUBBLE_RELAY_IMAGE}" == "" ]]; then
+        echo "building hubble-relay image..."
+        make LOCKDEBUG=1 docker-hubble-relay-image
+        echo "tagging hubble-relay image..."
+        docker tag ${HUBBLE_RELAY_TAG} ${REGISTRY}/${HUBBLE_RELAY_TAG}
+        echo "pushing hubble-relay image to ${REGISTRY}/${HUBBLE_RELAY_TAG}..."
+        docker push ${REGISTRY}/${HUBBLE_RELAY_TAG}
+        echo "removing local hubble-relay image..."
+        docker rmi "${HUBBLE_RELAY_TAG}:latest"
       else
-        pull_image_and_push_to_local_registry ${CILIUM_IMAGE} ${REGISTRY} ${CILIUM_TAG}
-        pull_image_and_push_to_local_registry ${CILIUM_OPERATOR_IMAGE} ${REGISTRY} ${CILIUM_OPERATOR_TAG}
-        delete_cilium_pods
+        pull_image_and_push_to_local_registry "${HUBBLE_RELAY_IMAGE}" "${REGISTRY}" "${HUBBLE_RELAY_TAG}"
       fi
 
     elif [[ "$(hostname)" == "k8s1" && "${CILIUM_REGISTRY}" != "" ]]; then
 		if [[ ${CILIUM_IMAGE} != "" ]]; then
-			pull_image_and_push_to_local_registry ${CILIUM_REGISTRY}/${CILIUM_IMAGE} ${REGISTRY} ${CILIUM_TAG}
+			pull_image_and_push_to_local_registry "${CILIUM_REGISTRY}/${CILIUM_IMAGE}" "${REGISTRY}" "${CILIUM_TAG}"
 		fi
 		if [[ ${CILIUM_OPERATOR_IMAGE} != "" ]]; then
-			pull_image_and_push_to_local_registry ${CILIUM_REGISTRY}/${CILIUM_OPERATOR_IMAGE} ${REGISTRY} ${CILIUM_OPERATOR_TAG}
+			pull_image_and_push_to_local_registry "${CILIUM_REGISTRY}/${CILIUM_OPERATOR_IMAGE}" "${REGISTRY}" "${CILIUM_OPERATOR_TAG}"
+		fi
+		if [[ ${HUBBLE_RELAY_IMAGE} != "" ]]; then
+			pull_image_and_push_to_local_registry "${CILIUM_REGISTRY}/${HUBBLE_RELAY_IMAGE}" "${REGISTRY}" "${HUBBLE_RELAY_TAG}"
 		fi
     else
         echo "Not on master K8S node; no need to compile Cilium container"

test/provision/helpers.bash

@@ -85,29 +85,10 @@ function pull_image_and_push_to_local_registry {
   echo "done pulling ${IMG}"
 
   echo "tagging ${IMG} with tag ${TAG_WITH_REG}"
-  docker tag "${IMG}" ${TAG_WITH_REG}
+  docker tag "${IMG}" "${TAG_WITH_REG}"
   echo "done tagging ${IMG} with tag ${TAG_WITH_REG}"
 
   echo "pushing ${TAG_WITH_REG}"
-  docker push ${TAG_WITH_REG}
+  docker push "${TAG_WITH_REG}"
   echo "done pushing ${TAG_WITH_REG}"
 }
-
-function build_cilium_image {
-  echo "building cilium image..."
-  make LOCKDEBUG=1 docker-image-no-clean
-  echo "tagging cilium image..."
-  docker tag cilium/cilium k8s1:5000/cilium/cilium-dev
-  echo "pushing cilium image..."
-  docker push k8s1:5000/cilium/cilium-dev
-}
-
-function build_operator_image {
-  # build cilium-operator image
-  echo "building cilium-operator image..."
-  make LOCKDEBUG=1 docker-operator-image
-  echo "tagging cilium-operator image..."
-  docker tag cilium/operator k8s1:5000/cilium/operator
-  echo "pushing cilium-operator image..."
-  docker push k8s1:5000/cilium/operator
-}