Migrate to kubetest2 for CI

.github/actions/bot/index.js

@@ -198,7 +198,7 @@ class CICommand {
             owner: this.repository_owner,
             repo: this.repository_name,
             workflow_id: 'ci-manual.yaml',
-            ref: 'master',
+            ref: 'main',
             inputs: inputs
         });
         return null;

.github/actions/ci/build/action.yaml

@@ -9,6 +9,9 @@ inputs:
   k8s_version:
     required: true
     type: string
+  os_distro:
+    required: true
+    type: string    
   additional_arguments:
     required: false
     type: string
@@ -25,8 +28,8 @@ runs:
       shell: bash
       run: |
         packer plugins install github.com/hashicorp/amazon
-        AMI_NAME="amazon-eks-node-${{ inputs.k8s_version }}-${{ inputs.build_id }}"
-        make k8s=${{ inputs.k8s_version }} ami_name=${AMI_NAME} ${{ inputs.additional_arguments }}
+        AMI_NAME="amazon-eks-node-${{ inputs.os_distro }}-${{ inputs.k8s_version }}-${{ inputs.build_id }}"
+        make k8s=${{ inputs.k8s_version }} os_distro=${{ inputs.os_distro }} ami_name=${AMI_NAME} ${{ inputs.additional_arguments }}
         echo "ami_id=$(jq -r .builds[0].artifact_id "${AMI_NAME}-manifest.json" | cut -d ':' -f 2)" >> $GITHUB_OUTPUT
     - uses: actions/upload-artifact@v3
       with:

.github/actions/ci/kubetest2/action.yaml

@@ -0,0 +1,67 @@
+name: '[CI] Integration test / kubetest2'
+inputs:
+  build_id:
+    required: true
+    type: string
+  ami_id:
+    required: true
+    type: string
+  os_distro:
+    required: true
+    type: string
+  k8s_version:
+    required: true
+    type: string
+  aws_region:
+    required: true
+    type: string
+runs:
+  using: "composite"
+  steps:
+    - id: kubetest2
+      shell: bash
+      run: |
+        export PATH=${PATH}:$(go env GOPATH)/bin
+        go install sigs.k8s.io/kubetest2/...@latest
+        go install github.com/aws/aws-k8s-tester/kubetest2/...@latest
+
+        case "${{ inputs.os_distro }}" in
+          al2)
+            KUBETEST2_ARGS="--user-data-format=bootstrap.sh"
+            ;;
+          al2023)
+            KUBETEST2_ARGS="--addons=vpc-cni:latest --tune-vpc-cni --user-data-format=nodeadm"
+            ;;
+          *)
+            echo >&2 "unknown os_distro: ${{ inputs.os_distro }}"
+            exit 1
+          ;;
+        esac
+
+        TESTER_NAME=ginkgo
+        # k8s_version below 1.26 use the older ginkgo tester
+        if [ "${{ inputs.k8s_version }}" = "$(echo -e "${{ inputs.k8s_version }}\n1.26" | sort -V | head -n1)" ]; then
+          TESTER_NAME=ginkgo-v1
+        fi
+
+        USER_DATA_FORMAT=nodeadm
+        if [ "${{ inputs.os_distro }}" = "al2" ]; then
+          USER_DATA_FORMAT=bootstrap.sh
+        fi
+
+        kubetest2 eksapi \
+          --up \
+          --down \
+          --kubernetes-version=${{ inputs.k8s_version }} \
+          --generate-ssh-key \
+          --unmanaged-nodes \
+          --ami=${{ inputs.ami_id }} \
+          --user-data-format=$USER_DATA_FORMAT \
+          --region=${{ inputs.aws_region }} \
+          $KUBETEST2_ARGS \
+          --test=$TESTER_NAME \
+          -- \
+          --test-package-marker=latest-${{ inputs.k8s_version }}.txt \
+          --parallel=6 \
+          --focus-regex='\[Conformance\]' \
+          --skip-regex='\[Serial\]|\[Disruptive\]|\[Slow\]|Garbage.collector'

.github/workflows/ci-manual.yaml

@@ -24,7 +24,6 @@ on:
         default: "test"
         options:
           - "build"
-          - "launch"
           - "test"
       build_arguments:
         required: false
@@ -66,7 +65,7 @@ jobs:
             });
   kubernetes-versions:
     runs-on: ubuntu-latest
-    name: ${{ matrix.k8s_version }}
+    name: ${{ matrix.k8s_version }} / ${{ matrix.os_distro }}
     needs:
       - setup
       - notify-start
@@ -78,10 +77,11 @@ jobs:
       fail-fast: false
       matrix:
         k8s_version: ${{ fromJson(needs.setup.outputs.kubernetes_versions) }}
+        os_distro: [al2, al2023]
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: 'master'
+          ref: 'main'
       - uses: aws-actions/configure-aws-credentials@v2
         with:
           aws-region: ${{ secrets.AWS_REGION }}
@@ -94,23 +94,19 @@ jobs:
         with:
           git_sha: ${{ inputs.git_sha }}
           k8s_version: ${{ matrix.k8s_version }}
+          os_distro: ${{ matrix.os_distro }}
           build_id: ${{ needs.setup.outputs.build_id }}
           additional_arguments: ${{ inputs.build_arguments }}
-      - if: ${{ inputs.goal == 'launch' || inputs.goal == 'test' }}
-        name: "${{ needs.setup.outputs.ci_step_name_prefix }} Launch"
-        id: launch
-        uses: ./.github/actions/ci/launch
+      - if: ${{ inputs.goal == 'test' }}
+        name: "${{ needs.setup.outputs.ci_step_name_prefix }} Test"
+        id: test
+        uses: ./.github/actions/ci/kubetest2
         with:
           ami_id: ${{ steps.build.outputs.ami_id }}
           k8s_version: ${{ matrix.k8s_version }}
+          os_distro: ${{ matrix.os_distro }}
           build_id: ${{ needs.setup.outputs.build_id }}
           aws_region: ${{ secrets.AWS_REGION }}
-      - if: ${{ inputs.goal == 'test' }}
-        name: "${{ needs.setup.outputs.ci_step_name_prefix }} Test"
-        id: sonobuoy
-        uses: ./.github/actions/ci/sonobuoy
-        with:
-          cluster_name: ${{ steps.launch.outputs.cluster_name }}
   notify-outcome:
     if: ${{ always() }}
     runs-on: ubuntu-latest
@@ -133,18 +129,17 @@ jobs:
               "cancelled": "🚮"
             };
             const uniqueStepNames = new Set();
-            const stepConclusionsByK8sVersion = new Map();
+            const stepConclusionsByJob = new Map();
             const ciStepNamePrefix = "${{ needs.setup.outputs.ci_step_name_prefix }}";
             for (const job of data.jobs) {
-              if (/\d+\.\d+/.test(job.name)) {
-                const k8sVersion = job.name;
+              if (/^\d+\.\d+/.test(job.name)) {
                 for (const step of job.steps) {
                   if (step.name.startsWith(ciStepNamePrefix)) {
                     const stepName = step.name.substring(ciStepNamePrefix.length).trim();
-                    let stepConclusions = stepConclusionsByK8sVersion.get(k8sVersion);
+                    let stepConclusions = stepConclusionsByJob.get(job.name);
                     if (!stepConclusions) {
                       stepConclusions = new Map();
-                      stepConclusionsByK8sVersion.set(k8sVersion, stepConclusions);
+                      stepConclusionsByJob.set(job.name, stepConclusions);
                     }
                     stepConclusions.set(stepName, step.conclusion);
                     uniqueStepNames.add(stepName);
@@ -153,7 +148,7 @@ jobs:
               }
             }
             const headers = [{
-              data: 'Kubernetes version',
+              data: 'AMI variant',
               header: true
             }];
             for (const stepName of uniqueStepNames.values()) {
@@ -163,10 +158,10 @@ jobs:
               });
             }
             const rows = [];
-            for (const stepConclusionsForK8sVersion of [...stepConclusionsByK8sVersion.entries()].sort()) {
-              const k8sVersion = stepConclusionsForK8sVersion[0];
-              const row = [k8sVersion];
-              for (const step of stepConclusionsForK8sVersion[1].entries()) {
+            for (const stepConclusionsForJob of [...stepConclusionsByJob.entries()].sort()) {
+              const job = stepConclusionsForJob[0];
+              const row = [job];
+              for (const step of stepConclusionsForJob[1].entries()) {
                   row.push(`${step[1]} ${conclusionEmojis[step[1]]}`);
               }
               rows.push(row);