[Flow Visibility] Add e2e tests for Flow Aggregator with ClickHouse (#…

…3673) * Add ClickHouse client e2e test Signed-off-by: Shawn Wang <wshaoquan@vmware.com> * Run Flow Aggregator e2e tests separately This change introduces the following changes to the e2e tests: - only run Flow Aggregator e2e tests if requested explicitly by flag - setup antrea-agent with proper config as part of manifest generation - update Github workflow for kind tests Signed-off-by: Shawn Wang <wshaoquan@vmware.com>
antrea-io · May 4, 2022 · 8f451f7 · 8f451f7
1 parent f08aecd
commit 8f451f7
Show file tree

Hide file tree

Showing 12 changed files with 609 additions and 65 deletions.
diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -67,7 +67,7 @@ jobs:
 
   test-e2e-encap:
     name: E2e tests on a Kind cluster on Linux
-    needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image]
+    needs: [build-antrea-coverage-image]
     runs-on: [ubuntu-latest]
     steps:
     - name: Free disk space
@@ -84,7 +84,6 @@ jobs:
     - name: Load Antrea image
       run:  |
         docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
-        docker load -i flow-aggregator-cov/flow-aggregator.tar
     - name: Install Kind
       run: |
         curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -124,7 +123,7 @@ jobs:
 
   test-e2e-encap-no-proxy:
     name: E2e tests on a Kind cluster on Linux with AntreaProxy disabled
-    needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image]
+    needs: [build-antrea-coverage-image]
     runs-on: [ubuntu-latest]
     steps:
     - name: Free disk space
@@ -141,7 +140,6 @@ jobs:
     - name: Load Antrea image
       run: |
         docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
-        docker load -i flow-aggregator-cov/flow-aggregator.tar
     - name: Install Kind
       run: |
         curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -181,7 +179,7 @@ jobs:
 
   test-e2e-encap-proxy-all:
     name: E2e tests on a Kind cluster on Linux with AntreaProxy all Service support
-    needs: [ build-antrea-coverage-image, build-flow-aggregator-coverage-image ]
+    needs: [ build-antrea-coverage-image ]
     runs-on: [ ubuntu-latest ]
     steps:
       - name: Free disk space
@@ -198,7 +196,6 @@ jobs:
       - name: Load Antrea image
         run: |
           docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
-          docker load -i flow-aggregator-cov/flow-aggregator.tar
       - name: Install Kind
         run: |
           curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -238,7 +235,7 @@ jobs:
 
   test-e2e-noencap:
     name: E2e tests on a Kind cluster on Linux (noEncap)
-    needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image]
+    needs: [build-antrea-coverage-image]
     runs-on: [ubuntu-latest]
     steps:
     - name: Free disk space
@@ -255,7 +252,6 @@ jobs:
     - name: Load Antrea image
       run: |
         docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
-        docker load -i flow-aggregator-cov/flow-aggregator.tar
     - name: Install Kind
       run: |
         curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -295,7 +291,7 @@ jobs:
 
   test-e2e-hybrid:
     name: E2e tests on a Kind cluster on Linux (hybrid)
-    needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image]
+    needs: [build-antrea-coverage-image]
     runs-on: [ubuntu-latest]
     steps:
     - name: Free disk space
@@ -312,7 +308,6 @@ jobs:
     - name: Load Antrea image
       run: |
         docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
-        docker load -i flow-aggregator-cov/flow-aggregator.tar
     - name: Install Kind
       run: |
         curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -355,7 +350,7 @@ jobs:
   # test uses a Geneve overlay.
   test-e2e-encap-no-np:
     name: E2e tests on a Kind cluster on Linux with Antrea-native policies disabled
-    needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image]
+    needs: [build-antrea-coverage-image]
     runs-on: [ubuntu-latest]
     steps:
     - name: Free disk space
@@ -372,7 +367,6 @@ jobs:
     - name: Load Antrea image
       run: |
         docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
-        docker load -i flow-aggregator-cov/flow-aggregator.tar
     - name: Install Kind
       run: |
         curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -410,6 +404,63 @@ jobs:
         path: log.tar.gz
         retention-days: 30
 
+  test-e2e-flow-visibility:
+    name: E2e tests on a Kind cluster on Linux for Flow Visibility
+    needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image]
+    runs-on: [ubuntu-latest]
+    steps:
+      - name: Free disk space
+        # https://github.com/actions/virtual-environments/issues/709
+        run: |
+          sudo apt-get clean
+          df -h
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.17
+      - name: Download Antrea images from previous jobs
+        uses: actions/download-artifact@v3
+      - name: Load Antrea image
+        run:  |
+          docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
+          docker load -i flow-aggregator-cov/flow-aggregator.tar
+      - name: Install Kind
+        run: |
+          curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
+          chmod +x ./kind
+          sudo mv kind /usr/local/bin
+      - name: Run e2e tests
+        run: |
+          mkdir log
+          mkdir test-e2e-fa-coverage
+          ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-fa-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --coverage --flow-visibility
+      - name: Tar coverage files
+        run: tar -czf test-e2e-fa-coverage.tar.gz test-e2e-fa-coverage
+      - name: Upload coverage for test-e2e-fa-coverage
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-e2e-fa-coverage
+          path: test-e2e-fa-coverage.tar.gz
+          retention-days: 30
+      - name: Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: '*.cov.out*'
+          flags: kind-e2e-tests
+          name: codecov-test-e2e-fa
+          directory: test-e2e-fa-coverage
+      - name: Tar log files
+        if: ${{ failure() }}
+        run: tar -czf log.tar.gz log
+      - name: Upload test log
+        uses: actions/upload-artifact@v3
+        if: ${{ failure() }}
+        with:
+          name: e2e-kind-fa.tar.gz
+          path: log.tar.gz
+          retention-days: 30
+
   test-netpol-tmp:
     name: Run experimental network policy tests (netpol) on Kind cluster
     needs: build-antrea-coverage-image

diff --git a/build/yamls/flow-visibility/base/kustomization-e2e.yml b/build/yamls/flow-visibility/base/kustomization-e2e.yml
@@ -0,0 +1,26 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: flow-visibility
+
+resources:
+  - clickhouse.yml
+
+configMapGenerator:
+  - name: clickhouse-mounted-configmap
+    namespace: flow-visibility
+    files:
+      - provisioning/datasources/create_table.sh
+
+# CLICKHOUSE_CONFIG_MAP_NAME exports the value in `metadata.name` from `ConfigMap` named `clickhouse-mounted-configmap`,
+# which is used for inserting the value to a CRD for an object of kind `ClickHouseInstallation`
+vars:
+  - name: CLICKHOUSE_CONFIG_MAP_NAME
+    objref:
+      kind: ConfigMap
+      name: clickhouse-mounted-configmap
+      apiVersion: v1
+    fieldref:
+      fieldpath: metadata.name
+
+configurations:
+  - kustomize-config.yml
diff --git a/build/yamls/flow-visibility/patches/e2e/imagePullPolicyClickhouse.yml b/build/yamls/flow-visibility/patches/e2e/imagePullPolicyClickhouse.yml
@@ -0,0 +1,6 @@
+- op: add
+  path: /spec/templates/podTemplates/0/spec/containers/0/imagePullPolicy
+  value: IfNotPresent
+- op: add
+  path: /spec/templates/podTemplates/0/spec/containers/1/imagePullPolicy
+  value: IfNotPresent
diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh
@@ -29,6 +29,7 @@ _usage="Usage: $0 [--encap-mode <mode>] [--ip-family <v4|v6>] [--no-proxy] [--np
         --proxy-all                   Enables Antrea proxy with all Service support.
         --endpointslice               Enables Antrea proxy and EndpointSlice support.
         --no-np                       Disables Antrea-native policies.
+        --flow-visibility             Only run flow visibility related e2e tests.
         --skip                        A comma-separated list of keywords, with which tests should be skipped.
         --coverage                    Enables measure Antrea code coverage when run e2e tests on kind.
         --setup-only                  Only perform setting up the cluster and run test.
@@ -45,6 +46,9 @@ function print_usage {
 TESTBED_CMD=$(dirname $0)"/kind-setup.sh"
 YML_CMD=$(dirname $0)"/../../hack/generate-manifest.sh"
 FLOWAGGREGATOR_YML_CMD=$(dirname $0)"/../../hack/generate-manifest-flow-aggregator.sh"
+FLOW_VISIBILITY_CMD=$(dirname $0)"/../../hack/generate-manifest-flow-visibility.sh --mode e2e"
+FLOW_VISIBILITY_HELM_VALUES=$(dirname $0)"/values-flow-exporter.yml"
+CH_OPERATOR_YML=$(dirname $0)"/../../build/yamls/clickhouse-operator-install-bundle.yml"
 
 function quit {
   result=$?
@@ -61,6 +65,7 @@ proxy=true
 proxy_all=false
 endpointslice=false
 np=true
+flow_visibility=false
 coverage=false
 skiplist=""
 setup_only=false
@@ -91,6 +96,10 @@ case $key in
     np=false
     shift
     ;;
+    --flow-visibility)
+    flow_visibility=true
+    shift
+    ;;
     --skip)
     skiplist="$2"
     shift 2
@@ -150,18 +159,20 @@ fi
 if ! $np; then
     manifest_args="$manifest_args --no-np"
 fi
+if $flow_visibility; then
+    manifest_args="$manifest_args --flow-exporter --extra-helm-values-file $FLOW_VISIBILITY_HELM_VALUES"
+fi
 
 COMMON_IMAGES_LIST=("k8s.gcr.io/e2e-test-images/agnhost:2.29" \
                     "projects.registry.vmware.com/library/busybox"  \
                     "projects.registry.vmware.com/antrea/nginx:1.21.6-alpine" \
-                    "projects.registry.vmware.com/antrea/perftool" \
-                    "projects.registry.vmware.com/antrea/ipfix-collector:v0.5.12")
-for image in "${COMMON_IMAGES_LIST[@]}"; do
-    for i in `seq 3`; do
-        docker pull $image && break
-        sleep 1
-    done
-done
+                    "projects.registry.vmware.com/antrea/perftool")
+
+FLOW_VISIBILITY_IMAGE_LIST=("projects.registry.vmware.com/antrea/ipfix-collector:v0.5.12" \
+                            "projects.registry.vmware.com/antrea/flow-visibility-clickhouse-operator:0.18.2" \
+                            "projects.registry.vmware.com/antrea/flow-visibility-metrics-exporter:0.18.2" \
+                            "projects.registry.vmware.com/antrea/flow-visibility-clickhouse-server:21.11" \
+                            "projects.registry.vmware.com/antrea/flow-visibility-clickhouse-monitor:latest")
 if $coverage; then
     manifest_args="$manifest_args --coverage"
     COMMON_IMAGES_LIST+=("antrea/antrea-ubuntu-coverage:latest")
@@ -173,6 +184,20 @@ fi
 if $proxy_all; then
     COMMON_IMAGES_LIST+=("k8s.gcr.io/echoserver:1.10")
 fi
+if $flow_visibility; then
+    COMMON_IMAGES_LIST+=("${FLOW_VISIBILITY_IMAGE_LIST[@]}")
+    if $coverage; then
+        COMMON_IMAGES_LIST+=("antrea/flow-aggregator-coverage:latest")
+    else
+        COMMON_IMAGES_LIST+=("projects.registry.vmware.com/antrea/flow-aggregator:latest")
+    fi
+fi
+for image in "${COMMON_IMAGES_LIST[@]}"; do
+    for i in `seq 3`; do
+        docker pull $image && break
+        sleep 1
+    done
+done
 
 printf -v COMMON_IMAGES "%s " "${COMMON_IMAGES_LIST[@]}"
 
@@ -195,16 +220,32 @@ function setup_cluster {
 
 function run_test {
   current_mode=$1
+  coverage_args=""
+  flow_visibility_args=""
 
   if $coverage; then
       $YML_CMD --encap-mode $current_mode $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea-coverage.yml
       $YML_CMD --ipsec $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea-ipsec-coverage.yml
-      $FLOWAGGREGATOR_YML_CMD --coverage | docker exec -i kind-control-plane dd of=/root/flow-aggregator-coverage.yml
+      timeout="80m"
+      coverage_args="--coverage --coverage-dir $ANTREA_COV_DIR"
   else
       $YML_CMD --encap-mode $current_mode $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea.yml
       $YML_CMD --ipsec $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea-ipsec.yml
-      $FLOWAGGREGATOR_YML_CMD | docker exec -i kind-control-plane dd of=/root/flow-aggregator.yml
+      timeout="75m"
   fi
+
+  if $flow_visibility; then
+      timeout="10m"
+      flow_visibility_args="-run=TestFlowAggregator --flow-visibility"
+      if $coverage; then
+          $FLOWAGGREGATOR_YML_CMD --coverage | docker exec -i kind-control-plane dd of=/root/flow-aggregator-coverage.yml
+      else
+          $FLOWAGGREGATOR_YML_CMD | docker exec -i kind-control-plane dd of=/root/flow-aggregator.yml
+      fi
+      $FLOW_VISIBILITY_CMD | docker exec -i kind-control-plane dd of=/root/flow-visibility.yml
+      cat $CH_OPERATOR_YML | docker exec -i kind-control-plane dd of=/root/clickhouse-operator-install-bundle.yml
+  fi
+
   if $proxy_all; then
       apiserver=$(docker exec -i kind-control-plane kubectl get endpoints kubernetes --no-headers | awk '{print $2}')
       if $coverage; then
@@ -215,11 +256,7 @@ function run_test {
   fi
   sleep 1
 
-  if $coverage; then
-      go test -v -timeout=80m antrea.io/antrea/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR --coverage --coverage-dir $ANTREA_COV_DIR --skip=$skiplist
-  else
-      go test -v -timeout=75m antrea.io/antrea/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR --skip=$skiplist
-  fi
+  go test -v -timeout=$timeout antrea.io/antrea/test/e2e $flow_visibility_args -provider=kind --logs-export-dir=$ANTREA_LOG_DIR --skip=$skiplist $coverage_args
 }
 
 if [[ "$mode" == "" ]] || [[ "$mode" == "encap" ]]; then

diff --git a/ci/kind/values-flow-exporter.yml b/ci/kind/values-flow-exporter.yml
@@ -0,0 +1,4 @@
+flowCollector:
+  flowPollInterval: "1s"
+  activeFlowExportTimeout: "2s"
+  idleFlowExportTimeout: "1s"