diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 70eedd1..d87ed89 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -2,16 +2,16 @@ name: E2E Tests on: push: - branches: [ "main", "release-**" ] + branches: ["main", "release-**"] pull_request: - branches: [ "main" ] + branches: ["main"] paths: - - 'tests/**' - - 'pkg/**' - - 'cmd/**' - - 'charts/**' - - '.github/workflows/**' - - 'Dockerfile' + - "tests/**" + - "pkg/**" + - "cmd/**" + - "charts/**" + - ".github/workflows/**" + - "Dockerfile" # This workflow runs e2e tests and relies on existance of EKS cluster with a `s3-csi-driver-sa` service account # already deployed to it, which provides the driver with access to s3. @@ -24,7 +24,7 @@ on: # - uploaded images will be later promoted to public repository by "release" workflow concurrency: e2e-cluster env: - AWS_REGION : "us-east-1" + AWS_REGION: "us-east-1" COMMIT_ID: ${{ github.event_name == 'push' && github.sha || github.event.pull_request.head.sha }} TMP_IMAGE_NAME: "s3-csi-driver-tmp" PROMOTED_IMAGE_NAME: "s3-csi-driver" @@ -46,7 +46,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version-file: 'go.mod' + go-version-file: "go.mod" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Set up QEMU @@ -73,13 +73,32 @@ jobs: matrix: cluster-type: ["eksctl", "kops"] arch: ["x86", "arm"] - family: ["AmazonLinux2", "Bottlerocket"] + family: + ["AmazonLinux2", "AmazonLinux2023", "Bottlerocket", "Ubuntu2004"] exclude: - cluster-type: "kops" family: "Bottlerocket" - cluster-type: "eksctl" arch: "arm" family: "Bottlerocket" + - cluster-type: "eksctl" + family: "Ubuntu2004" + # We're mainly using AL2023 to test our CSI driver with enforced SELinux, + # AL2023 supports switching to enforcing mode via cloud-init, but, + # passing user-data (the way we configure cloud-init) is not supported on eksctl for AL2023. + # So, we're only running SELinux tests with kops. + - cluster-type: "eksctl" + family: "AmazonLinux2023" + include: + # Since we only enable enforcing mode for SELinux in AL2023, it's easier to list it in "include" + # field rather than trying to exclude all other variants. + - cluster-type: "kops" + arch: "arm" + family: "AmazonLinux2023" + selinux-mode: "enforcing" + # TODO: This is temporary, we wanted to run our tests on SELinux enforced hosts even before we support it. + # Our tests will be passing on SELinux enforced hosts as well once we support it and we will remove this configuration. + continue-on-error: ${{ matrix.selinux-mode == 'enforcing' }} runs-on: ubuntu-latest permissions: id-token: write @@ -90,10 +109,10 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version-file: 'go.mod' + go-version-file: "go.mod" - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: "3.10" - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@master with: @@ -111,6 +130,7 @@ jobs: export CLUSTER_TYPE=${{ matrix.cluster-type }} export ARCH=${{ matrix.arch }} export AMI_FAMILY=${{ matrix.family }} + export SELINUX_MODE=${{ matrix.selinux-mode }} tests/e2e-kubernetes/scripts/run.sh - name: Update kubeconfig run: | @@ -167,7 +187,7 @@ jobs: if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86' uses: benchmark-action/github-action-benchmark@v1 with: - tool: 'customBiggerIsBetter' + tool: "customBiggerIsBetter" output-file-path: tests/e2e-kubernetes/csi-test-artifacts/output.json alert-threshold: "200%" fail-on-alert: true @@ -213,20 +233,20 @@ jobs: id-token: write contents: read steps: - - name: Set up crane - uses: imjasonh/setup-crane@v0.1 - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@master - with: - role-to-assume: ${{ secrets.TEST_IAM_ROLE }} - aws-region: ${{ env.AWS_REGION }} - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - name: Promote image for release branch - env: - REGISTRY: ${{ steps.login-ecr.outputs.registry }} - run: | - export TMP_IMAGE_NAME=${REGISTRY}/${{ env.TMP_IMAGE_NAME }}:${{ env.COMMIT_ID }} - export NEW_IMAGE_NAME=${REGISTRY}/${{ env.PROMOTED_IMAGE_NAME }}:${{ env.COMMIT_ID }} - crane copy ${TMP_IMAGE_NAME} ${NEW_IMAGE_NAME} + - name: Set up crane + uses: imjasonh/setup-crane@v0.1 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@master + with: + role-to-assume: ${{ secrets.TEST_IAM_ROLE }} + aws-region: ${{ env.AWS_REGION }} + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + - name: Promote image for release branch + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: | + export TMP_IMAGE_NAME=${REGISTRY}/${{ env.TMP_IMAGE_NAME }}:${{ env.COMMIT_ID }} + export NEW_IMAGE_NAME=${REGISTRY}/${{ env.PROMOTED_IMAGE_NAME }}:${{ env.COMMIT_ID }} + crane copy ${TMP_IMAGE_NAME} ${NEW_IMAGE_NAME} diff --git a/tests/e2e-kubernetes/scripts/kops-patch-node-selinux-enforcing.yaml b/tests/e2e-kubernetes/scripts/kops-patch-node-selinux-enforcing.yaml new file mode 100644 index 0000000..0be2ec4 --- /dev/null +++ b/tests/e2e-kubernetes/scripts/kops-patch-node-selinux-enforcing.yaml @@ -0,0 +1,8 @@ +spec: + additionalUserData: + - name: selinux-settings.txt + type: text/cloud-config + content: | + #cloud-config + selinux: + mode: enforcing diff --git a/tests/e2e-kubernetes/scripts/kops.sh b/tests/e2e-kubernetes/scripts/kops.sh index 74bcc19..58cdfc4 100755 --- a/tests/e2e-kubernetes/scripts/kops.sh +++ b/tests/e2e-kubernetes/scripts/kops.sh @@ -33,6 +33,7 @@ function kops_create_cluster() { KOPS_PATCH_NODE_FILE=${11} KOPS_STATE_FILE=${12} SSH_KEY=${13} + KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE=${14} if kops_cluster_exists "${CLUSTER_NAME}" "${BIN}" "${KOPS_STATE_FILE}"; then kops_delete_cluster "$BIN" "$CLUSTER_NAME" "$KOPS_STATE_FILE" @@ -58,6 +59,10 @@ function kops_create_cluster() { kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE" "Cluster" "" kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_NODE_FILE" "InstanceGroup" "Node" + if [ -n "$KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE" ]; then + kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE" "InstanceGroup" "Node" + fi + ${BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}" ${BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes ${BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}" @@ -145,4 +150,4 @@ function json_to_yaml() { echo "---" >> "$OUT" jq ".[$i]" "$IN" | kubectl patch -f - --local -p "{}" --type merge -o yaml >> "$OUT" done -} \ No newline at end of file +} diff --git a/tests/e2e-kubernetes/scripts/run.sh b/tests/e2e-kubernetes/scripts/run.sh index e5f3262..ff3fcf0 100755 --- a/tests/e2e-kubernetes/scripts/run.sh +++ b/tests/e2e-kubernetes/scripts/run.sh @@ -27,35 +27,69 @@ KUBECTL_BIN=${KUBECTL_INSTALL_PATH}/kubectl CLUSTER_TYPE=${CLUSTER_TYPE:-kops} ARCH=${ARCH:-x86} AMI_FAMILY=${AMI_FAMILY:-AmazonLinux2} -CLUSTER_NAME=${CLUSTER_NAME:-"s3-csi-cluster-${CLUSTER_TYPE}-${ARCH}.k8s.local"} -if [[ "${CLUSTER_TYPE}" == "eksctl" && "${ARCH}" == "x86" ]]; then - if [[ "${AMI_FAMILY}" == "Bottlerocket" ]]; then - CLUSTER_NAME="s3-csi-cluster-bottlerocket" - else - CLUSTER_NAME="s3-csi-cluster" - fi -elif [[ "${CLUSTER_TYPE}" == "eksctl" && "${ARCH}" == "arm" ]]; then - CLUSTER_NAME="s3-csi-cluster-arm" +SELINUX_MODE=${SELINUX_MODE:-} + +# We don't actually create clusters with eksctl as part of our GitHub workflow, we run our tests on pre-existing clusters. +# Therefore, we need to make sure we're setting correct cluster names here. +if [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then + if [[ "${AMI_FAMILY}" == "Bottlerocket" ]]; then + CLUSTER_NAME="s3-csi-cluster-bottlerocket" + elif [[ "${ARCH}" == "arm" ]]; then + CLUSTER_NAME="s3-csi-cluster-arm" + else + CLUSTER_NAME="s3-csi-cluster" + fi +else + # In kops, cluster names must end with ".k8s.local" to use Gossip DNS. + # See https://kops.sigs.k8s.io/gossip/#configuring-a-cluster-to-use-gossip + # They also need to be valid domain names, that's why we're lowercasing "AMI_FAMILY". + CLUSTER_NAME="s3-csi-cluster-${CLUSTER_TYPE}-${AMI_FAMILY,,}-${ARCH}.k8s.local" fi + KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.kubeconfig"} +# kops: must include patch version (e.g. 1.19.1) +# eksctl: mustn't include patch version (e.g. 1.19) +K8S_VERSION=${K8S_VERSION:-1.28.2} +K8S_VERSION_MAJOR_MINOR=${K8S_VERSION_MAJOR_MINOR:-1.28} +K8S_VERSION_KOPS=${K8S_VERSION_KOPS:-${K8S_VERSION}} + KOPS_VERSION=1.28.0 ZONES=${AWS_AVAILABILITY_ZONES:-$(aws ec2 describe-availability-zones --region ${REGION} | jq -c '.AvailabilityZones[].ZoneName' | grep -v "us-east-1e" | tr '\n' ',' | sed 's/"//g' | sed 's/.$//')} # excluding us-east-1e, see: https://github.com/eksctl-io/eksctl/issues/817 NODE_COUNT=${NODE_COUNT:-3} + +# "AMI_ID" is only used on kops (eksctl directly uses "AMI_FAMILY"). +declare -A AMI_IDS +AMI_IDS["AmazonLinux2-x86"]="/aws/service/ami-amazon-linux-latest/amzn2-ami-kernel-5.10-hvm-x86_64-gp2" +AMI_IDS["AmazonLinux2-arm"]="/aws/service/ami-amazon-linux-latest/amzn2-ami-kernel-5.10-hvm-arm64-gp2" +AMI_IDS["AmazonLinux2023-x86"]="/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64" +AMI_IDS["AmazonLinux2023-arm"]="/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64" +AMI_IDS["Bottlerocket-x86"]="/aws/service/bottlerocket/aws-k8s-${K8S_VERSION_MAJOR_MINOR}/x86_64/latest/image_id" +AMI_IDS["Bottlerocket-arm"]="/aws/service/bottlerocket/aws-k8s-${K8S_VERSION_MAJOR_MINOR}/arm64/latest/image_id" +AMI_IDS["Ubuntu2004-x86"]="/aws/service/canonical/ubuntu/server/20.04/stable/current/amd64/hvm/ebs-gp2/ami-id" +AMI_IDS["Ubuntu2004-arm"]="/aws/service/canonical/ubuntu/server/20.04/stable/current/arm64/hvm/ebs-gp2/ami-id" + +AMI_ID_DEFAULT=$(aws ssm get-parameters --names "${AMI_IDS["${AMI_FAMILY}-${ARCH}"]}" --region ${REGION} --query 'Parameters[0].Value' --output text) +AMI_ID=${AMI_ID:-$AMI_ID_DEFAULT} +if [[ "${CLUSTER_TYPE}" == "kops" ]]; then + echo "Using AMI $AMI_ID for $AMI_FAMILY" +fi + if [[ "${ARCH}" == "x86" ]]; then INSTANCE_TYPE_DEFAULT=c5.large - AMI_ID_DEFAULT=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64 --region ${REGION} --query 'Parameters[0].Value' --output text) KOPS_STATE_FILE_DEFAULT=s3://mountpoint-s3-csi-driver-kops-state-store else INSTANCE_TYPE_DEFAULT=m7g.medium - AMI_ID_DEFAULT=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64 --region ${REGION} --query 'Parameters[0].Value' --output text) KOPS_STATE_FILE_DEFAULT=s3://mountpoint-s3-csi-driver-kops-arm-state-store fi INSTANCE_TYPE=${INSTANCE_TYPE:-$INSTANCE_TYPE_DEFAULT} -AMI_ID=${AMI_ID:-$AMI_ID_DEFAULT} CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.yaml KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-${BASE_DIR}/kops-patch.yaml} KOPS_PATCH_NODE_FILE=${KOPS_PATCH_NODE_FILE:-${BASE_DIR}/kops-patch-node.yaml} +KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE=${KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE:-${BASE_DIR}/kops-patch-node-selinux-enforcing.yaml} +if [[ "${SELINUX_MODE}" != "enforcing" ]]; then + KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE="" +fi KOPS_STATE_FILE=${KOPS_STATE_FILE:-$KOPS_STATE_FILE_DEFAULT} SSH_KEY=${SSH_KEY:-""} HELM_RELEASE_NAME=mountpoint-s3-csi-driver @@ -64,10 +98,6 @@ EKSCTL_VERSION=${EKSCTL_VERSION:-0.161.0} EKSCTL_PATCH_FILE=${EKSCTL_PATCH_FILE:-${BASE_DIR}/eksctl-patch.yaml} CI_ROLE_ARN=${CI_ROLE_ARN:-""} -# kops: must include patch version (e.g. 1.19.1) -# eksctl: mustn't include patch version (e.g. 1.19) -K8S_VERSION_KOPS=${K8S_VERSION_KOPS:-${K8S_VERSION:-1.28.2}} - mkdir -p ${TEST_DIR} mkdir -p ${BIN_DIR} export PATH="$PATH:${BIN_DIR}" @@ -114,7 +144,8 @@ function create_cluster() { "$KOPS_PATCH_FILE" \ "$KOPS_PATCH_NODE_FILE" \ "$KOPS_STATE_FILE" \ - "$SSH_KEY" + "$SSH_KEY" \ + "$KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE" elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then eksctl_create_cluster \ "$CLUSTER_NAME" \