diff --git a/Makefile b/Makefile index 56e26f576627..2bab544ab826 100644 --- a/Makefile +++ b/Makefile @@ -386,6 +386,7 @@ docker-cilium-builder-manifest: docker-hubble-relay-image: $(QUIET)$(CONTAINER_ENGINE) build \ --build-arg NOSTRIP=${NOSTRIP} \ + --build-arg CILIUM_SHA=$(firstword $(GIT_VERSION)) \ -f hubble-relay.Dockerfile \ -t "cilium/hubble-relay:$(DOCKER_IMAGE_TAG)" . $(QUIET)$(CONTAINER_ENGINE) tag cilium/hubble-relay:$(DOCKER_IMAGE_TAG) cilium/hubble-relay:$(DOCKER_IMAGE_TAG)-${GOARCH} diff --git a/hubble-relay.Dockerfile b/hubble-relay.Dockerfile index 3597bf1f9917..8e5c795d9a07 100644 --- a/hubble-relay.Dockerfile +++ b/hubble-relay.Dockerfile @@ -1,13 +1,19 @@ FROM docker.io/library/golang:1.14.3 as builder +ARG CILIUM_SHA="" +LABEL cilium-sha=${CILIUM_SHA} ADD . /go/src/github.com/cilium/cilium WORKDIR /go/src/github.com/cilium/cilium/hubble-relay ARG NOSTRIP RUN make NOSTRIP=$NOSTRIP FROM docker.io/library/alpine:3.11 as certs +ARG CILIUM_SHA="" +LABEL cilium-sha=${CILIUM_SHA} RUN apk --update add ca-certificates FROM docker.io/library/golang:1.14.3 as gops +ARG CILIUM_SHA="" +LABEL cilium-sha=${CILIUM_SHA} RUN go get -d github.com/google/gops && \ cd /go/src/github.com/google/gops && \ git checkout -b v0.3.6 v0.3.6 && \ @@ -17,6 +23,8 @@ RUN go get -d github.com/google/gops && \ strip /go/bin/gops FROM scratch +ARG CILIUM_SHA="" +LABEL cilium-sha=${CILIUM_SHA} LABEL maintainer="maintainer@cilium.io" COPY --from=builder /go/src/github.com/cilium/cilium/hubble-relay/hubble-relay /usr/bin/hubble-relay COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt diff --git a/test/Vagrantfile b/test/Vagrantfile index 7f7b1fe67323..8fb6c33cdf1b 100644 --- a/test/Vagrantfile +++ b/test/Vagrantfile @@ -21,6 +21,7 @@ $CONTAINER_RUNTIME=(ENV['CONTAINER_RUNTIME'] || "docker") $CNI_INTEGRATION=(ENV['CNI_INTEGRATION'] || "") $CILIUM_IMAGE = ENV['CILIUM_IMAGE'] || "" $CILIUM_OPERATOR_IMAGE = ENV['CILIUM_OPERATOR_IMAGE'] || "" +$HUBBLE_RELAY_IMAGE = ENV['HUBBLE_RELAY_IMAGE'] || "" $CILIUM_REGISTRY = ENV['CILIUM_REGISTRY'] || "" $PRELOAD_VM = ENV['PRELOAD_VM'] || "false" $SKIP_K8S_PROVISION = ENV['SKIP_K8S_PROVISION'] || "false" @@ -134,6 +135,7 @@ Vagrant.configure("2") do |config| "#{$IPv6}", "#{$CONTAINER_RUNTIME}", "#{$CNI_INTEGRATION}"] sh.env = {"CILIUM_IMAGE" => "#{$CILIUM_IMAGE}", "CILIUM_OPERATOR_IMAGE" => "#{$CILIUM_OPERATOR_IMAGE}", + "HUBBLE_RELAY_IMAGE" => "#{$HUBBLE_RELAY_IMAGE}", "CILIUM_REGISTRY" => "#{$CILIUM_REGISTRY}", "PRELOAD_VM" => "#{$PRELOAD_VM}", "SKIP_K8S_PROVISION" => "#{$SKIP_K8S_PROVISION}", diff --git a/test/clean-local-registry-tag.sh b/test/clean-local-registry-tag.sh index 23e728817083..cf569994a55f 100755 --- a/test/clean-local-registry-tag.sh +++ b/test/clean-local-registry-tag.sh @@ -11,13 +11,14 @@ fi BUSYBOX_VERSION=1.31.1 -docker pull docker.io/library/busybox:$BUSYBOX_VERSION +docker pull "docker.io/library/busybox:$BUSYBOX_VERSION" -docker tag busybox:$BUSYBOX_VERSION $1/cilium/cilium:$2 -docker tag busybox:$BUSYBOX_VERSION $1/cilium/cilium-dev:$2 -docker tag busybox:$BUSYBOX_VERSION $1/cilium/operator:$2 - -docker push $1/cilium/cilium:$2 -docker push $1/cilium/cilium-dev:$2 -docker push $1/cilium/operator:$2 +docker tag "busybox:${BUSYBOX_VERSION}" "$1/cilium/cilium:$2" +docker tag "busybox:${BUSYBOX_VERSION}" "$1/cilium/cilium-dev:$2" +docker tag "busybox:${BUSYBOX_VERSION}" "$1/cilium/operator:$2" +docker tag "busybox:${BUSYBOX_VERSION}" "$1/cilium/hubble-relay:$2" +docker push "$1/cilium/cilium:$2" +docker push "$1/cilium/cilium-dev:$2" +docker push "$1/cilium/operator:$2" +docker push "$1/cilium/hubble-relay:$2" diff --git a/test/config/config.go b/test/config/config.go index 68059bb8a032..779e1815f0ea 100644 --- a/test/config/config.go +++ b/test/config/config.go @@ -34,6 +34,7 @@ type CiliumTestConfigType struct { SkipLogGathering bool CiliumImage string CiliumOperatorImage string + HubbleRelayImage string ProvisionK8s bool Timeout time.Duration Kubeconfig string @@ -65,6 +66,8 @@ func (c *CiliumTestConfigType) ParseFlags() { "Specifies which image of cilium to use during tests") flag.StringVar(&c.CiliumOperatorImage, "cilium.operator-image", "", "Specifies which image of cilium-operator to use during tests") + flag.StringVar(&c.HubbleRelayImage, "cilium.hubble-relay-image", "", + "Specifies which image of hubble-relay to use during tests") flag.BoolVar(&c.ProvisionK8s, "cilium.provision-k8s", true, "Specifies whether Kubernetes should be deployed and installed via kubeadm or not") flag.DurationVar(&c.Timeout, "cilium.timeout", 24*time.Hour, diff --git a/test/helpers/kubectl.go b/test/helpers/kubectl.go index 6a8f03c97153..ee9b9526f005 100644 --- a/test/helpers/kubectl.go +++ b/test/helpers/kubectl.go @@ -187,6 +187,10 @@ func Init() { os.Setenv("CILIUM_OPERATOR_IMAGE", config.CiliumTestConfig.CiliumOperatorImage) } + if config.CiliumTestConfig.HubbleRelayImage != "" { + os.Setenv("HUBBLE_RELAY_IMAGE", config.CiliumTestConfig.HubbleRelayImage) + } + if config.CiliumTestConfig.Registry != "" { os.Setenv("CILIUM_REGISTRY", config.CiliumTestConfig.Registry) } diff --git a/test/k8sT/assertionHelpers.go b/test/k8sT/assertionHelpers.go index 373b9b26250f..a1541f0e7e6d 100644 --- a/test/k8sT/assertionHelpers.go +++ b/test/k8sT/assertionHelpers.go @@ -56,6 +56,22 @@ func ExpectCiliumOperatorReady(vm *helpers.Kubectl) { ExpectWithOffset(1, err).Should(BeNil(), "Cilium operator was not able to get into ready state") } +// ExpectHubbleCLIReady is a wrapper around helpers/WaitForPods. It asserts +// that the error returned by that function is nil. +func ExpectHubbleCLIReady(vm *helpers.Kubectl, ns string) { + By("Waiting for hubble-cli to be ready") + err := vm.WaitforPods(ns, "-l k8s-app=hubble-cli", longTimeout) + ExpectWithOffset(1, err).Should(BeNil(), "hubble-cli was not able to get into ready state") +} + +// ExpectHubbleRelayReady is a wrapper around helpers/WaitForPods. It asserts +// that the error returned by that function is nil. +func ExpectHubbleRelayReady(vm *helpers.Kubectl, ns string) { + By("Waiting for hubble-relay to be ready") + err := vm.WaitforPods(ns, "-l k8s-app=hubble-relay", longTimeout) + ExpectWithOffset(1, err).Should(BeNil(), "hubble-relay was not able to get into ready state") +} + // ExpectAllPodsTerminated is a wrapper around helpers/WaitCleanAllTerminatingPods. // It asserts that the error returned by that function is nil. func ExpectAllPodsTerminated(vm *helpers.Kubectl) { diff --git a/test/k8sT/hubble.go b/test/k8sT/hubble.go index 2048ea6af8e1..455b70a3ef18 100644 --- a/test/k8sT/hubble.go +++ b/test/k8sT/hubble.go @@ -18,8 +18,10 @@ import ( "context" "fmt" "net" + "strconv" "strings" + "github.com/asaskevich/govalidator" "github.com/cilium/cilium/pkg/annotation" . "github.com/cilium/cilium/test/ginkgo-ext" "github.com/cilium/cilium/test/helpers" @@ -35,8 +37,9 @@ var _ = Describe("K8sHubbleTest", func() { hubblePodK8s1 string - hubbleSelector = "-l k8s-app=hubble-cli" - hubbleNamespace = helpers.CiliumNamespace + hubbleNamespace = helpers.CiliumNamespace + hubbleRelayService = "hubble-relay" + hubbleRelayAddress string demoPath string @@ -86,16 +89,22 @@ var _ = Describe("K8sHubbleTest", func() { demoPath = helpers.ManifestGet(kubectl.BasePath(), "demo.yaml") DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, map[string]string{ - "global.hubble.cli.enabled": "true", "global.hubble.metricsServer": fmt.Sprintf(":%s", prometheusPort), "global.hubble.metrics.enabled": `"{dns:query;ignoreAAAA,drop,tcp,flow,port-distribution,icmp,http}"`, + "global.hubble.cli.enabled": "true", + "global.hubble.relay.enabled": "true", }) - err := kubectl.WaitforPods(hubbleNamespace, hubbleSelector, helpers.HelperTimeout) - Expect(err).Should(BeNil(), "hubble-cli pods did not become ready") - + var err error + ExpectHubbleCLIReady(kubectl, hubbleNamespace) hubblePodK8s1, err = kubectl.GetHubbleClientPodOnNodeWithLabel(hubbleNamespace, helpers.K8s1) Expect(err).Should(BeNil(), "unable to find hubble-cli pod on %s", helpers.K8s1) + + ExpectHubbleRelayReady(kubectl, hubbleNamespace) + hubbleRelayIP, hubbleRelayPort, err := kubectl.GetServiceHostPort(hubbleNamespace, hubbleRelayService) + Expect(err).Should(BeNil(), "Cannot get service %s", hubbleRelayService) + Expect(govalidator.IsIP(hubbleRelayIP)).Should(BeTrue(), "hubbleRelayIP is not an IP") + hubbleRelayAddress = net.JoinHostPort(hubbleRelayIP, strconv.Itoa(hubbleRelayPort)) }) AfterFailed(func() { @@ -169,6 +178,21 @@ var _ = Describe("K8sHubbleTest", func() { res.ExpectContains(`hubble_flows_processed_total{subtype="to-endpoint",type="Trace",verdict="FORWARDED"}`) }) + It("Test L3/L4 Flow with hubble-relay", func() { + ctx, cancel := context.WithTimeout(context.Background(), helpers.MidCommandTimeout) + defer cancel() + follow := kubectl.HubbleObserveFollow(ctx, hubbleNamespace, hubblePodK8s1, fmt.Sprintf( + "--server %s --last 1 --type trace --from-pod %s/%s --to-namespace %s --to-label %s --to-port %d", + hubbleRelayAddress, namespaceForTest, appPods[helpers.App2], namespaceForTest, app1Labels, app1Port)) + + res := kubectl.ExecPodCmd(namespaceForTest, appPods[helpers.App2], + helpers.CurlFail(fmt.Sprintf("http://%s/public", app1ClusterIP))) + res.ExpectSuccess("%q cannot curl clusterIP %q", appPods[helpers.App2], app1ClusterIP) + + err := follow.WaitUntilMatchFilterLineTimeout(`{$.Type}`, "L3_L4", helpers.ShortCommandTimeout) + Expect(err).To(BeNil(), fmt.Sprintf("hubble observe query timed out on %q", follow.OutputPrettyPrint())) + }) + It("Test L7 Flow", func() { addVisibilityAnnotation(namespaceForTest, app1Labels, "Ingress", "80", "TCP", "HTTP") defer removeVisbilityAnnotation(namespaceForTest, app1Labels) @@ -193,5 +217,23 @@ var _ = Describe("K8sHubbleTest", func() { res.ExpectSuccess("%s/%s cannot curl metrics %q", hubbleNamespace, hubblePodK8s1, app1ClusterIP) res.ExpectContains(`hubble_flows_processed_total{subtype="HTTP",type="L7",verdict="FORWARDED"}`) }) + + It("Test L7 Flow with hubble-relay", func() { + addVisibilityAnnotation(namespaceForTest, app1Labels, "Ingress", "80", "TCP", "HTTP") + defer removeVisbilityAnnotation(namespaceForTest, app1Labels) + + ctx, cancel := context.WithTimeout(context.Background(), helpers.MidCommandTimeout) + defer cancel() + follow := kubectl.HubbleObserveFollow(ctx, hubbleNamespace, hubblePodK8s1, fmt.Sprintf( + "--server %s --last 1 --type l7 --from-pod %s/%s --to-namespace %s --to-label %s --protocol http", + hubbleRelayAddress, namespaceForTest, appPods[helpers.App2], namespaceForTest, app1Labels)) + + res := kubectl.ExecPodCmd(namespaceForTest, appPods[helpers.App2], + helpers.CurlFail(fmt.Sprintf("http://%s/public", app1ClusterIP))) + res.ExpectSuccess("%q cannot curl clusterIP %q", appPods[helpers.App2], app1ClusterIP) + + err := follow.WaitUntilMatchFilterLineTimeout(`{$.Type}`, "L7", helpers.ShortCommandTimeout) + Expect(err).To(BeNil(), fmt.Sprintf("hubble observe query timed out on %q", follow.OutputPrettyPrint())) + }) }) }) diff --git a/test/make-images-push-to-local-registry.sh b/test/make-images-push-to-local-registry.sh index 86ba2e5c5af6..a80e525f82af 100755 --- a/test/make-images-push-to-local-registry.sh +++ b/test/make-images-push-to-local-registry.sh @@ -3,15 +3,17 @@ set -e cd .. -make docker-image DOCKER_IMAGE_TAG=$2 +make docker-image DOCKER_IMAGE_TAG="$2" -docker tag cilium/cilium:$2 $1/cilium/cilium:$2 -docker tag cilium/cilium:$2 $1/cilium/cilium-dev:$2 -docker tag cilium/operator:$2 $1/cilium/operator:$2 +docker tag "cilium/cilium:$2" "$1/cilium/cilium:$2" +docker tag "cilium/cilium:$2" "$1/cilium/cilium-dev:$2" +docker tag "cilium/operator:$2" "$1/cilium/operator:$2" +docker tag "cilium/hubble-relay:$2" "$1/cilium/hubble-relay:$2" -docker push $1/cilium/cilium:$2 -docker push $1/cilium/cilium-dev:$2 -docker push $1/cilium/operator:$2 +docker push "$1/cilium/cilium:$2" +docker push "$1/cilium/cilium-dev:$2" +docker push "$1/cilium/operator:$2" +docker push "$1/cilium/hubble-relay:$2" cilium_git_version="$(cat GIT_VERSION)" diff --git a/test/provision/compile.sh b/test/provision/compile.sh index 5d4ef28e1b8c..96c9b1e4b7b7 100755 --- a/test/provision/compile.sh +++ b/test/provision/compile.sh @@ -9,6 +9,7 @@ GOPATH="/home/vagrant/go" REGISTRY="k8s1:5000" CILIUM_TAG="cilium/cilium-dev" CILIUM_OPERATOR_TAG="cilium/operator" +HUBBLE_RELAY_TAG="cilium/hubble-relay" DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) @@ -28,44 +29,57 @@ then # Only need to build on one host, since we can pull from the other host. if [[ "$(hostname)" == "k8s1" && "${CILIUM_REGISTRY}" == "" ]]; then ./test/provision/container-images.sh cilium_images . - if [[ "${CILIUM_IMAGE}" == "" && "${CILIUM_OPERATOR_IMAGE}" == "" ]]; then - echo "building cilium/cilium container image..." + + if [[ "${CILIUM_IMAGE}" == "" ]]; then + echo "building cilium container image..." make LOCKDEBUG=1 docker-image-no-clean + echo "tagging cilium image..." + docker tag cilium/cilium "${REGISTRY}/${CILIUM_TAG}" + echo "pushing cilium image to ${REGISTRY}/${CILIUM_TAG}..." + docker push "${REGISTRY}/${CILIUM_TAG}" + echo "removing local cilium image..." + docker rmi cilium/cilium:latest + else + pull_image_and_push_to_local_registry "${CILIUM_IMAGE}" "${REGISTRY}" "${CILIUM_TAG}" + fi - echo "building cilium/operator container image..." - make LOCKDEBUG=1 docker-operator-image& - export OPERATORPID=$! + if [[ "${CILIUM_OPERATOR_IMAGE}" == "" ]]; then + echo "building cilium-operator image..." + make LOCKDEBUG=1 docker-operator-image + echo "tagging cilium-operator image..." + docker tag "${CILIUM_OPERATOR_TAG}" "${REGISTRY}/${CILIUM_OPERATOR_TAG}" + echo "pushing cilium/operator image to ${REGISTRY}/${CILIUM_OPERATOR_TAG}..." + docker push "${REGISTRY}/${CILIUM_OPERATOR_TAG}" + echo "removing local cilium-operator image..." + docker rmi "${CILIUM_OPERATOR_TAG}:latest" + else + pull_image_and_push_to_local_registry "${CILIUM_OPERATOR_IMAGE}" "${REGISTRY}" "${CILIUM_OPERATOR_TAG}" + fi - echo "pushing cilium/cilium image to k8s1:5000/cilium/cilium-dev..." - docker tag cilium/cilium k8s1:5000/cilium/cilium-dev - docker rmi cilium/cilium:latest - docker push k8s1:5000/cilium/cilium-dev + delete_cilium_pods - wait $OPERATORPID - echo "pushing cilium/operator image to k8s1:5000/cilium/operator..." - docker tag cilium/operator k8s1:5000/cilium/operator - docker push k8s1:5000/cilium/operator - delete_cilium_pods - elif [[ "${CILIUM_IMAGE}" != "" && "${CILIUM_OPERATOR_IMAGE}" == "" ]]; then - pull_image_and_push_to_local_registry ${CILIUM_IMAGE} ${REGISTRY} ${CILIUM_TAG} - build_operator_image - delete_cilium_pods - elif [[ "${CILIUM_IMAGE}" == "" && "${CILIUM_OPERATOR_IMAGE}" != "" ]]; then - pull_image_and_push_to_local_registry ${CILIUM_OPERATOR_IMAGE} ${REGISTRY} ${CILIUM_OPERATOR_TAG} - build_cilium_image - delete_cilium_pods + if [[ "${HUBBLE_RELAY_IMAGE}" == "" ]]; then + echo "building hubble-relay image..." + make LOCKDEBUG=1 docker-hubble-relay-image + echo "tagging hubble-relay image..." + docker tag ${HUBBLE_RELAY_TAG} ${REGISTRY}/${HUBBLE_RELAY_TAG} + echo "pushing hubble-relay image to ${REGISTRY}/${HUBBLE_RELAY_TAG}..." + docker push ${REGISTRY}/${HUBBLE_RELAY_TAG} + echo "removing local hubble-relay image..." + docker rmi "${HUBBLE_RELAY_TAG}:latest" else - pull_image_and_push_to_local_registry ${CILIUM_IMAGE} ${REGISTRY} ${CILIUM_TAG} - pull_image_and_push_to_local_registry ${CILIUM_OPERATOR_IMAGE} ${REGISTRY} ${CILIUM_OPERATOR_TAG} - delete_cilium_pods + pull_image_and_push_to_local_registry "${HUBBLE_RELAY_IMAGE}" "${REGISTRY}" "${HUBBLE_RELAY_TAG}" fi elif [[ "$(hostname)" == "k8s1" && "${CILIUM_REGISTRY}" != "" ]]; then if [[ ${CILIUM_IMAGE} != "" ]]; then - pull_image_and_push_to_local_registry ${CILIUM_REGISTRY}/${CILIUM_IMAGE} ${REGISTRY} ${CILIUM_TAG} + pull_image_and_push_to_local_registry "${CILIUM_REGISTRY}/${CILIUM_IMAGE}" "${REGISTRY}" "${CILIUM_TAG}" fi if [[ ${CILIUM_OPERATOR_IMAGE} != "" ]]; then - pull_image_and_push_to_local_registry ${CILIUM_REGISTRY}/${CILIUM_OPERATOR_IMAGE} ${REGISTRY} ${CILIUM_OPERATOR_TAG} + pull_image_and_push_to_local_registry "${CILIUM_REGISTRY}/${CILIUM_OPERATOR_IMAGE}" "${REGISTRY}" "${CILIUM_OPERATOR_TAG}" + fi + if [[ ${HUBBLE_RELAY_IMAGE} != "" ]]; then + pull_image_and_push_to_local_registry "${CILIUM_REGISTRY}/${HUBBLE_RELAY_IMAGE}" "${REGISTRY}" "${HUBBLE_RELAY_TAG}" fi else echo "Not on master K8S node; no need to compile Cilium container" diff --git a/test/provision/helpers.bash b/test/provision/helpers.bash index a3d4a74c748b..6f63ded0b3e6 100644 --- a/test/provision/helpers.bash +++ b/test/provision/helpers.bash @@ -85,29 +85,10 @@ function pull_image_and_push_to_local_registry { echo "done pulling ${IMG}" echo "tagging ${IMG} with tag ${TAG_WITH_REG}" - docker tag "${IMG}" ${TAG_WITH_REG} + docker tag "${IMG}" "${TAG_WITH_REG}" echo "done tagging ${IMG} with tag ${TAG_WITH_REG}" echo "pushing ${TAG_WITH_REG}" - docker push ${TAG_WITH_REG} + docker push "${TAG_WITH_REG}" echo "done pushing ${TAG_WITH_REG}" } - -function build_cilium_image { - echo "building cilium image..." - make LOCKDEBUG=1 docker-image-no-clean - echo "tagging cilium image..." - docker tag cilium/cilium k8s1:5000/cilium/cilium-dev - echo "pushing cilium image..." - docker push k8s1:5000/cilium/cilium-dev -} - -function build_operator_image { - # build cilium-operator image - echo "building cilium-operator image..." - make LOCKDEBUG=1 docker-operator-image - echo "tagging cilium-operator image..." - docker tag cilium/operator k8s1:5000/cilium/operator - echo "pushing cilium-operator image..." - docker push k8s1:5000/cilium/operator -}