test: e2e test (#143)

Test: - added e2e test for Linux, windows and macOS - includes 14 test cases: imported pem/pfx certs chain, unordered pem/pfx certs chain, azure key vault csr pem/pfx certs chain, self-signed pem/pfx certs, partial pem/pfx certs chain with local cert bundle and corresponding invalid cases. - the E2E automatically run for PRs from `Azure/notation-azure-kv` repo and will be skipped for folked repo Additional files: - dockerfile for build distribution registry on windows - zot configuration for local registry on macOS - generate-certs.sh script for create certificate on azure key vault for future re-generating certificates. --------- Signed-off-by: Junjie Gao <junjiegao@microsoft.com>
Azure · Dec 22, 2023 · e3fce3d · e3fce3d
1 parent 8884168
commit e3fce3d
Show file tree

Hide file tree

Showing 8 changed files with 854 additions and 9 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -32,14 +32,11 @@ jobs:
           DEFAULT_BRANCH: main
           DEFAULT_WORKSPACE: ./Notation.Plugin.AzureKeyVault
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          FILTER_REGEX_EXCLUDE: .*Tests/.*
+          FILTER_REGEX_EXCLUDE: '.*Tests/.*|.*.yml|.*/scripts/generate-certs.sh'
           VALIDATE_MARKDOWN: false
-  build:
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-    name: "Build"
-    runs-on: ${{ matrix.os }}
+  test:
+    name: Unit Testing and Build
+    runs-on: ubuntu-latest
     timeout-minutes: 5
     permissions:
       contents: read
@@ -52,7 +49,147 @@ jobs:
         uses: actions/checkout@v3
       - name: Run unit tests
         run: make test
-      - name: Build testing
-        run: make build
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3
+      - name: Build bianry for E2E testing
+        run: |
+          ./scripts/build.sh v0.0.1 linux-x64
+          ./scripts/build.sh v0.0.1 win-x64
+          ./scripts/build.sh v0.0.1 osx-x64
+      - name: Upload Linux artifact
+        if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push'
+        uses: actions/upload-artifact@v2
+        with:
+          name: linux-amd64-binary
+          path: ./bin/artifacts/notation-azure-kv_0.0.1_linux_amd64.tar.gz
+          retention-days: 1
+      - name: Upload macOS artifact
+        if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push'
+        uses: actions/upload-artifact@v2
+        with:
+          name: darwin-amd64-binary
+          path: ./bin/artifacts/notation-azure-kv_0.0.1_darwin_amd64.tar.gz
+          retention-days: 1
+      - name: Upload Windows artifact
+        if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push'
+        uses: actions/upload-artifact@v2
+        with:
+          name: win-amd64-binary
+          path: ./bin/artifacts/notation-azure-kv_0.0.1_windows_amd64.zip
+          retention-days: 1
+  e2e-linux:
+    name: E2E testing on Linux
+    runs-on: ubuntu-latest
+    needs: test
+    if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push'
+    steps:
+      - name: Check out code into the project directory
+        uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
+        with:
+          name: linux-amd64-binary
+          path: ./bin/artifacts
+      - name: Run download server locally
+        run: |
+          nohup python3 -m http.server --directory ./bin/artifacts/ &
+
+          # prepare the environment variables for E2E
+          artifactName=notation-azure-kv_0.0.1_linux_amd64.tar.gz
+          checksum=$(shasum -a 256 "./bin/artifacts/$artifactName" | awk '{print $1}')
+          echo "pluginChecksum=$checksum" >> "$GITHUB_ENV"
+          echo "pluginDownloadURL=http://localhost:8000/$artifactName" >> "$GITHUB_ENV"
+      - name: Prepare container registry
+        run: |
+          docker run  --name registry --rm -d -p 5000:5000 registry:2
+          docker pull hello-world:latest
+          docker tag hello-world:latest localhost:5000/hello-world:v1
+          docker push localhost:5000/hello-world:v1
+      - name: Azure login
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+      - name: E2E testing
+        uses: ./test/e2e
+        with:
+          pluginDownloadURL: ${{ env.pluginDownloadURL }}
+          pluginChecksum: ${{ env.pluginChecksum }}
+  e2e-windows:
+    name: E2E testing on Windows
+    runs-on: windows-2022
+    needs: test
+    if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push'
+    steps:
+      - name: Check out code into the project directory
+        uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
+        with:
+          name: win-amd64-binary
+          path: ./bin/artifacts
+      - name: Run download server locally
+        run: |
+          # wsl bash
+          bash -c 'nohup python3 -m http.server --directory ./bin/artifacts/ &'
+
+          # Prepare the environment variables for E2E
+          $artifactName = "notation-azure-kv_0.0.1_windows_amd64.zip"
+          $checksum = (Get-FileHash ".\bin\artifacts\$artifactName" -Algorithm SHA256).Hash
+          "pluginChecksum=$checksum" | Out-File -Append -FilePath $Env:GITHUB_ENV
+          "pluginDownloadURL=http://localhost:8000/$artifactName" | Out-File -Append -FilePath $Env:GITHUB_ENV
+        shell: pwsh
+      - name: Prepare container registry
+        run: |
+          docker run  --name registry --rm -d -p 5000:5000 junjiegaomsft/registry:v2.8.2-ltsc2022
+          docker pull hello-world:latest
+          docker tag hello-world:latest localhost:5000/hello-world:v1
+          docker push localhost:5000/hello-world:v1
+        shell: pwsh
+      - name: Azure login
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+      - name: E2E testing
+        uses: ./test/e2e
+        with:
+          pluginDownloadURL: ${{ env.pluginDownloadURL }}
+          pluginChecksum: ${{ env.pluginChecksum }}
+  e2e-macos:
+    name: E2E testing on macOS
+    runs-on: macos-13
+    needs: test
+    if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push'
+    steps:
+      - name: Check out code into the project directory
+        uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
+        with:
+          name: darwin-amd64-binary
+          path: ./bin/artifacts
+      - name: Run download server locally
+        run: |
+          nohup python3 -m http.server --directory ./bin/artifacts/ &
+
+          # prepare the environment variables for E2E
+          artifactName=notation-azure-kv_0.0.1_darwin_amd64.tar.gz
+          checksum=$(shasum -a 256 "./bin/artifacts/$artifactName" | awk '{print $1}')
+          echo "pluginChecksum=$checksum" >> "$GITHUB_ENV"
+          echo "pluginDownloadURL=http://localhost:8000/$artifactName" >> "$GITHUB_ENV"
+      - name: Prepare container registry
+        run: |
+          # start zot registry
+          wget -O zot https://github.com/project-zot/zot/releases/download/v2.0.0-rc7/zot-darwin-amd64-minimal
+          chmod +x zot
+          nohup ./zot serve ./test/e2e/zot/config.json &
+
+          # install oras
+          wget -O oras.tar.gz https://github.com/oras-project/oras/releases/download/v1.1.0/oras_1.1.0_darwin_amd64.tar.gz
+          tar -zxf oras.tar.gz
+          ./oras push localhost:5000/hello-world:v1 --artifact-type  application/octet-stream ./LICENSE
+      - name: Azure login
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+      - name: E2E testing
+        uses: ./test/e2e
+        with:
+          pluginDownloadURL: ${{ env.pluginDownloadURL }}
+          pluginChecksum: ${{ env.pluginChecksum }}