Run e2e tests on AL2023 (with SELinux) and Ubuntu 20.04 as well

.github/workflows/e2e-tests.yaml

@@ -2,16 +2,16 @@ name: E2E Tests
 
 on:
   push:
-    branches: [ "main", "release-**" ]
+    branches: ["main", "release-**"]
   pull_request:
-    branches: [ "main" ]
+    branches: ["main"]
     paths:
-      - 'tests/**'
-      - 'pkg/**'
-      - 'cmd/**'
-      - 'charts/**'
-      - '.github/workflows/**'
-      - 'Dockerfile'
+      - "tests/**"
+      - "pkg/**"
+      - "cmd/**"
+      - "charts/**"
+      - ".github/workflows/**"
+      - "Dockerfile"
 
 # This workflow runs e2e tests and relies on existance of EKS cluster with a `s3-csi-driver-sa` service account
 # already deployed to it, which provides the driver with access to s3.
@@ -24,7 +24,7 @@ on:
 # - uploaded images will be later promoted to public repository by "release" workflow
 concurrency: e2e-cluster
 env:
-  AWS_REGION : "us-east-1"
+  AWS_REGION: "us-east-1"
   COMMIT_ID: ${{ github.event_name == 'push' && github.sha || github.event.pull_request.head.sha }}
   TMP_IMAGE_NAME: "s3-csi-driver-tmp"
   PROMOTED_IMAGE_NAME: "s3-csi-driver"
@@ -46,7 +46,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version-file: 'go.mod'
+          go-version-file: "go.mod"
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Set up QEMU
@@ -73,13 +73,32 @@ jobs:
       matrix:
         cluster-type: ["eksctl", "kops"]
         arch: ["x86", "arm"]
-        family: ["AmazonLinux2", "Bottlerocket"]
+        family:
+          ["AmazonLinux2", "AmazonLinux2023", "Bottlerocket", "Ubuntu2004"]
         exclude:
           - cluster-type: "kops"
             family: "Bottlerocket"
           - cluster-type: "eksctl"
             arch: "arm"
             family: "Bottlerocket"
+          - cluster-type: "eksctl"
+            family: "Ubuntu2004"
+          # We're mainly using AL2023 to test our CSI driver with enforced SELinux,
+          # AL2023 supports switching to enforcing mode via cloud-init, but,
+          # passing user-data (the way we configure cloud-init) is not supported on eksctl for AL2023.
+          # So, we're only running SELinux tests with kops.
+          - cluster-type: "eksctl"
+            family: "AmazonLinux2023"
+        include:
+          # Since we only enable enforcing mode for SELinux in AL2023, it's easier to list it in "include"
+          # field rather than trying to exclude all other variants.
+          - cluster-type: "kops"
+            arch: "arm"
+            family: "AmazonLinux2023"
+            selinux-mode: "enforcing"
+    # TODO: This is temporary, we wanted to run our tests on SELinux enforced hosts even before we support it.
+    #       Our tests will be passing on SELinux enforced hosts as well once we support it and we will remove this configuration.
+    continue-on-error: ${{ matrix.selinux-mode == 'enforcing' }}
     runs-on: ubuntu-latest
     permissions:
       id-token: write
@@ -90,10 +109,10 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version-file: 'go.mod'
+          go-version-file: "go.mod"
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: "3.10"
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@master
         with:
@@ -111,6 +130,7 @@ jobs:
           export CLUSTER_TYPE=${{ matrix.cluster-type }}
           export ARCH=${{ matrix.arch }}
           export AMI_FAMILY=${{ matrix.family }}
+          export SELINUX_MODE=${{ matrix.selinux-mode }}
           tests/e2e-kubernetes/scripts/run.sh
       - name: Update kubeconfig
         run: |
@@ -167,7 +187,7 @@ jobs:
         if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86'
         uses: benchmark-action/github-action-benchmark@v1
         with:
-          tool: 'customBiggerIsBetter'
+          tool: "customBiggerIsBetter"
           output-file-path: tests/e2e-kubernetes/csi-test-artifacts/output.json
           alert-threshold: "200%"
           fail-on-alert: true
@@ -213,20 +233,20 @@ jobs:
       id-token: write
       contents: read
     steps:
-    - name: Set up crane
-      uses: imjasonh/setup-crane@v0.1
-    - name: Configure AWS Credentials
-      uses: aws-actions/configure-aws-credentials@master
-      with:
-        role-to-assume: ${{ secrets.TEST_IAM_ROLE }}
-        aws-region: ${{ env.AWS_REGION }}
-    - name: Login to Amazon ECR
-      id: login-ecr
-      uses: aws-actions/amazon-ecr-login@v2
-    - name: Promote image for release branch
-      env:
-        REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-      run: |
-        export TMP_IMAGE_NAME=${REGISTRY}/${{ env.TMP_IMAGE_NAME }}:${{ env.COMMIT_ID }}
-        export NEW_IMAGE_NAME=${REGISTRY}/${{ env.PROMOTED_IMAGE_NAME }}:${{ env.COMMIT_ID }}
-        crane copy ${TMP_IMAGE_NAME} ${NEW_IMAGE_NAME}
+      - name: Set up crane
+        uses: imjasonh/setup-crane@v0.1
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@master
+        with:
+          role-to-assume: ${{ secrets.TEST_IAM_ROLE }}
+          aws-region: ${{ env.AWS_REGION }}
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+      - name: Promote image for release branch
+        env:
+          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        run: |
+          export TMP_IMAGE_NAME=${REGISTRY}/${{ env.TMP_IMAGE_NAME }}:${{ env.COMMIT_ID }}
+          export NEW_IMAGE_NAME=${REGISTRY}/${{ env.PROMOTED_IMAGE_NAME }}:${{ env.COMMIT_ID }}
+          crane copy ${TMP_IMAGE_NAME} ${NEW_IMAGE_NAME}

tests/e2e-kubernetes/scripts/kops-patch-node-selinux-enforcing.yaml

@@ -0,0 +1,8 @@
+spec:
+  additionalUserData:
+    - name: selinux-settings.txt
+      type: text/cloud-config
+      content: |
+        #cloud-config
+        selinux:
+            mode: enforcing

tests/e2e-kubernetes/scripts/kops.sh

@@ -33,6 +33,7 @@ function kops_create_cluster() {
   KOPS_PATCH_NODE_FILE=${11}
   KOPS_STATE_FILE=${12}
   SSH_KEY=${13}
+  KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE=${14}
 
   if kops_cluster_exists "${CLUSTER_NAME}" "${BIN}" "${KOPS_STATE_FILE}"; then
     kops_delete_cluster "$BIN" "$CLUSTER_NAME" "$KOPS_STATE_FILE"
@@ -58,6 +59,10 @@ function kops_create_cluster() {
   kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE" "Cluster" ""
   kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_NODE_FILE" "InstanceGroup" "Node"
 
+  if [ -n "$KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE" ]; then
+    kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE" "InstanceGroup" "Node"
+  fi
+
   ${BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
   ${BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes
   ${BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}"
@@ -145,4 +150,4 @@ function json_to_yaml() {
     echo "---" >> "$OUT"
     jq ".[$i]" "$IN" | kubectl patch -f - --local -p "{}" --type merge -o yaml >> "$OUT"
   done
-}
+}

tests/e2e-kubernetes/scripts/run.sh

@@ -27,35 +27,69 @@ KUBECTL_BIN=${KUBECTL_INSTALL_PATH}/kubectl
 CLUSTER_TYPE=${CLUSTER_TYPE:-kops}
 ARCH=${ARCH:-x86}
 AMI_FAMILY=${AMI_FAMILY:-AmazonLinux2}
-CLUSTER_NAME=${CLUSTER_NAME:-"s3-csi-cluster-${CLUSTER_TYPE}-${ARCH}.k8s.local"}
-if [[ "${CLUSTER_TYPE}" == "eksctl" && "${ARCH}" == "x86" ]]; then
-  if [[ "${AMI_FAMILY}" == "Bottlerocket" ]]; then
-    CLUSTER_NAME="s3-csi-cluster-bottlerocket"
-  else
-    CLUSTER_NAME="s3-csi-cluster"
-  fi
-elif [[ "${CLUSTER_TYPE}" == "eksctl" && "${ARCH}" == "arm" ]]; then
-  CLUSTER_NAME="s3-csi-cluster-arm"
+SELINUX_MODE=${SELINUX_MODE:-}
+
+# We don't actually create clusters with eksctl as part of our GitHub workflow, we run our tests on pre-existing clusters.
+# Therefore, we need to make sure we're setting correct cluster names here.
+if [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
+    if [[ "${AMI_FAMILY}" == "Bottlerocket" ]]; then
+        CLUSTER_NAME="s3-csi-cluster-bottlerocket"
+    elif [[ "${ARCH}" == "arm" ]]; then
+        CLUSTER_NAME="s3-csi-cluster-arm"
+    else
+        CLUSTER_NAME="s3-csi-cluster"
+    fi
+else
+    # In kops, cluster names must end with ".k8s.local" to use Gossip DNS.
+    # See https://kops.sigs.k8s.io/gossip/#configuring-a-cluster-to-use-gossip
+    # They also need to be valid domain names, that's why we're lowercasing "AMI_FAMILY".
+    CLUSTER_NAME="s3-csi-cluster-${CLUSTER_TYPE}-${AMI_FAMILY,,}-${ARCH}.k8s.local"
 fi
+
 KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.kubeconfig"}
 
+# kops: must include patch version (e.g. 1.19.1)
+# eksctl: mustn't include patch version (e.g. 1.19)
+K8S_VERSION=${K8S_VERSION:-1.28.2}
+K8S_VERSION_MAJOR_MINOR=${K8S_VERSION_MAJOR_MINOR:-1.28}
+K8S_VERSION_KOPS=${K8S_VERSION_KOPS:-${K8S_VERSION}}
+
 KOPS_VERSION=1.28.0
 ZONES=${AWS_AVAILABILITY_ZONES:-$(aws ec2 describe-availability-zones --region ${REGION} | jq -c '.AvailabilityZones[].ZoneName' | grep -v "us-east-1e" | tr '\n' ',' | sed 's/"//g' | sed 's/.$//')} # excluding us-east-1e, see: https://github.com/eksctl-io/eksctl/issues/817
 NODE_COUNT=${NODE_COUNT:-3}
+
+# "AMI_ID" is only used on kops (eksctl directly uses "AMI_FAMILY").
+declare -A AMI_IDS
+AMI_IDS["AmazonLinux2-x86"]="/aws/service/ami-amazon-linux-latest/amzn2-ami-kernel-5.10-hvm-x86_64-gp2"
+AMI_IDS["AmazonLinux2-arm"]="/aws/service/ami-amazon-linux-latest/amzn2-ami-kernel-5.10-hvm-arm64-gp2"
+AMI_IDS["AmazonLinux2023-x86"]="/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64"
+AMI_IDS["AmazonLinux2023-arm"]="/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64"
+AMI_IDS["Bottlerocket-x86"]="/aws/service/bottlerocket/aws-k8s-${K8S_VERSION_MAJOR_MINOR}/x86_64/latest/image_id"
+AMI_IDS["Bottlerocket-arm"]="/aws/service/bottlerocket/aws-k8s-${K8S_VERSION_MAJOR_MINOR}/arm64/latest/image_id"
+AMI_IDS["Ubuntu2004-x86"]="/aws/service/canonical/ubuntu/server/20.04/stable/current/amd64/hvm/ebs-gp2/ami-id"
+AMI_IDS["Ubuntu2004-arm"]="/aws/service/canonical/ubuntu/server/20.04/stable/current/arm64/hvm/ebs-gp2/ami-id"
+
+AMI_ID_DEFAULT=$(aws ssm get-parameters --names "${AMI_IDS["${AMI_FAMILY}-${ARCH}"]}" --region ${REGION} --query 'Parameters[0].Value' --output text)
+AMI_ID=${AMI_ID:-$AMI_ID_DEFAULT}
+if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
+    echo "Using AMI $AMI_ID for $AMI_FAMILY"
+fi
+
 if [[ "${ARCH}" == "x86" ]]; then
   INSTANCE_TYPE_DEFAULT=c5.large
-  AMI_ID_DEFAULT=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64 --region ${REGION} --query 'Parameters[0].Value' --output text)
   KOPS_STATE_FILE_DEFAULT=s3://mountpoint-s3-csi-driver-kops-state-store
 else
   INSTANCE_TYPE_DEFAULT=m7g.medium
-  AMI_ID_DEFAULT=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64 --region ${REGION} --query 'Parameters[0].Value' --output text)
   KOPS_STATE_FILE_DEFAULT=s3://mountpoint-s3-csi-driver-kops-arm-state-store
 fi
 INSTANCE_TYPE=${INSTANCE_TYPE:-$INSTANCE_TYPE_DEFAULT}
-AMI_ID=${AMI_ID:-$AMI_ID_DEFAULT}
 CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.yaml
 KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-${BASE_DIR}/kops-patch.yaml}
 KOPS_PATCH_NODE_FILE=${KOPS_PATCH_NODE_FILE:-${BASE_DIR}/kops-patch-node.yaml}
+KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE=${KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE:-${BASE_DIR}/kops-patch-node-selinux-enforcing.yaml}
+if [[ "${SELINUX_MODE}" != "enforcing" ]]; then
+    KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE=""
+fi
 KOPS_STATE_FILE=${KOPS_STATE_FILE:-$KOPS_STATE_FILE_DEFAULT}
 SSH_KEY=${SSH_KEY:-""}
 HELM_RELEASE_NAME=mountpoint-s3-csi-driver
@@ -64,10 +98,6 @@ EKSCTL_VERSION=${EKSCTL_VERSION:-0.161.0}
 EKSCTL_PATCH_FILE=${EKSCTL_PATCH_FILE:-${BASE_DIR}/eksctl-patch.yaml}
 CI_ROLE_ARN=${CI_ROLE_ARN:-""}
 
-# kops: must include patch version (e.g. 1.19.1)
-# eksctl: mustn't include patch version (e.g. 1.19)
-K8S_VERSION_KOPS=${K8S_VERSION_KOPS:-${K8S_VERSION:-1.28.2}}
-
 mkdir -p ${TEST_DIR}
 mkdir -p ${BIN_DIR}
 export PATH="$PATH:${BIN_DIR}"
@@ -114,7 +144,8 @@ function create_cluster() {
       "$KOPS_PATCH_FILE" \
       "$KOPS_PATCH_NODE_FILE" \
       "$KOPS_STATE_FILE" \
-      "$SSH_KEY"
+      "$SSH_KEY" \
+      "$KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE"
   elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
     eksctl_create_cluster \
       "$CLUSTER_NAME" \