Implement E2E test framework for testing ApplicationSets against live…

… Kubernetes/Argo CD instance (argoproj#65)
jgwest · Jan 8, 2021 · f98bf7e · f98bf7e
1 parent 1e1b453
commit f98bf7e
Show file tree

Hide file tree

Showing 12 changed files with 1,028 additions and 3 deletions.
diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml
@@ -96,3 +96,117 @@ jobs:
         with:
           name: test-results
           path: test-results/
+
+  test-e2e:
+    name: Run end-to-end tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        k3s-version: [v1.19.2]
+        # k3s-version: [v1.19.2, v1.18.9, v1.17.11, v1.16.15]
+    # needs: 
+      # - build-go
+    env:
+      GOPATH: /home/runner/go
+      ARGOCD_FAKE_IN_CLUSTER: "true"
+      ARGOCD_SSH_DATA_PATH: "/tmp/argo-e2e/app/config/ssh"
+      ARGOCD_TLS_DATA_PATH: "/tmp/argo-e2e/app/config/tls"
+      ARGOCD_E2E_SSH_KNOWN_HOSTS: "../fixture/certs/ssh_known_hosts"
+      ARGOCD_E2E_K3S: "true"
+      ARGOCD_IN_CI: "true"
+      ARGOCD_E2E_APISERVER_PORT: "8088"
+      ARGOCD_SERVER: "127.0.0.1:8088"
+    steps:
+      - name: Checkout latest Argo CD code
+        uses: actions/checkout@v2
+        with:
+          repository: argoproj/argo-cd
+          ref: v1.8.0
+          # Pin a specific commit to prevent Argo CD regressions from impacting us
+      - name: Setup Golang  
+        uses: actions/setup-go@v1
+        with:
+          go-version: '1.14.2'
+      - name: Install K3S
+        env:
+          INSTALL_K3S_VERSION: ${{ matrix.k3s-version }}+k3s1
+        run: |
+          set -x
+          curl -sfL https://get.k3s.io | sh -
+          sudo chmod -R a+rw /etc/rancher/k3s
+          sudo mkdir -p $HOME/.kube && sudo chown -R runner $HOME/.kube
+          sudo k3s kubectl config view --raw > $HOME/.kube/config
+          sudo chown runner $HOME/.kube/config
+          kubectl version
+      - name: Restore go build cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/go-build
+          key: ${{ runner.os }}-go-build-v1-${{ github.run_id }}
+      - name: Add ~/go/bin to PATH
+        run: |
+          echo "/home/runner/go/bin" >> $GITHUB_PATH
+      - name: Add /usr/local/bin to PATH
+        run: |
+          echo "/usr/local/bin" >> $GITHUB_PATH
+      - name: Download Go dependencies
+        run: |
+          go mod download
+          go get github.com/mattn/goreman
+      - name: Install all tools required for building & testing
+        run: |
+          make install-test-tools-local
+      - name: Setup git username and email
+        run: |
+          git config --global user.name "John Doe"
+          git config --global user.email "john.doe@example.com"
+      - name: Pull Docker image required for tests
+        run: |
+          docker pull quay.io/dexidp/dex:v2.25.0
+          docker pull argoproj/argo-cd-ci-builder:v1.0.0
+          docker pull redis:5.0.10-alpine
+      - name: Create target directory for binaries in the build-process
+        run: |
+          mkdir -p dist
+          chown runner dist
+      - name: Run Argo CD E2E server and wait for it being available
+        timeout-minutes: 30
+        run: |
+          set -x
+          # Add 8081 as an exposed port to Argo CD test container, to allow us to access repo server for git generators
+          # (this should be upstreamed, to prevent it from breaking here)
+          sed -i 's/4000:4000/4000:4000 -p 8081:8081/' Makefile
+          # Something is weird in GH runners -- there's a phantom listener for
+          # port 8080 which is not visible in netstat -tulpen, but still there
+          # with a HTTP listener. We have API server listening on port 8088
+          # instead.
+          make start-e2e-local 2>&1 | sed -r "s/[[:cntrl:]]\[[0-9]{1,3}m//g" > /tmp/e2e-server.log &
+          count=1
+          until curl -f http://127.0.0.1:8088/healthz; do
+            sleep 10;
+            if test $count -ge 60; then
+              echo "Timeout"
+              exit 1
+            fi
+            count=$((count+1))
+          done
+      - name: Checkout latest applicationsets code
+        uses: actions/checkout@v2
+        with:
+          path: applicationsets
+      - name: Run E2E test suite
+        run: |
+          set -x
+          cd "$GITHUB_WORKSPACE/applicationsets"
+          # TODO: re-enable this, or create a validation that ensures this is updated.
+          # make manifests
+          kubectl apply -f manifests/crds/argoproj.io_applicationsets.yaml
+          make build
+          make start-e2e 2>&1 | tee /tmp/appset-e2e-server.log &
+          make test-e2e 
+      - name: Upload e2e-server logs
+        uses: actions/upload-artifact@v2
+        with:
+          name: appset-e2e-server-k8s${{ matrix.k3s-version }}.log
+          path: /tmp/appset-e2e-server.log
+        if: ${{ failure() }}
diff --git a/Makefile b/Makefile
@@ -32,9 +32,21 @@ lint:
 
 
 # Run go fmt against code
+.PHONY: fmt
 fmt:
 	go fmt ./...
 
 # Run go vet against code
+.PHONY: vet
 vet:
 	go vet ./...
+
+# Start the standalone controller for the purpose of running e2e tests
+.PHONY: start-e2e
+start-e2e:
+	NAMESPACE=argocd-e2e "dist/argocd-applicationset" --metrics-addr=:12345 --probe-addr=:12346 --argocd-repo-server=localhost:8081
+
+# Begin the tests, targetting the standalone controller (started by make start-e2e) and the e2e argo-cd (started by make start-e2e)
+.PHONY: test-e2e
+test-e2e:
+	go test -race -count=1 -v -timeout 120s ./test/e2e/applicationsets
diff --git a/go.mod b/go.mod
@@ -6,6 +6,7 @@ require (
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/argoproj/argo-cd v1.7.6
 	github.com/argoproj/gitops-engine v0.1.3-0.20200904164417-c04f859da9b2
+	github.com/argoproj/pkg v0.0.0-20200624215116-23e74cb168fe
 	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/pkg/errors v0.9.1
@@ -22,6 +23,8 @@ require (
 )
 
 replace (
+	github.com/go-logr/logr => github.com/go-logr/logr v0.2.1
+	github.com/go-logr/zapr => github.com/go-logr/zapr v0.2.0
 	k8s.io/api => k8s.io/api v0.18.8
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.18.8
 	k8s.io/apimachinery => k8s.io/apimachinery v0.18.8
@@ -46,6 +49,4 @@ replace (
 	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.18.8
 	k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.18.8
 	k8s.io/sample-controller => k8s.io/sample-controller v0.18.8
-	github.com/go-logr/logr => github.com/go-logr/logr v0.2.1
-	github.com/go-logr/zapr => github.com/go-logr/zapr v0.2.0
 )
diff --git a/main.go b/main.go
@@ -49,7 +49,7 @@ func main() {
 	var debugLog bool
 	var dryRun bool
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
-	flag.StringVar(&metricsAddr, "probe-addr", ":8081", "The address the probe endpoint binds to.")
+	flag.StringVar(&probeBindAddr, "probe-addr", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")