From e35a5679d184b5d9bd8aecf3609cd2c0a94d6de1 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Thu, 22 Feb 2024 19:12:41 -0500 Subject: [PATCH] feat: support arm64 --- .github/workflows/cd.yml | 4 ++++ .github/workflows/ci.yml | 22 +++++++++++++++++++--- Dockerfile | 29 ++++++++++++++++++++++++----- dependencies/python/build-venvs.sh | 26 ++++++++++++++++++++++---- lib/functions/validation.sh | 18 ++++++++++++++++++ lib/linter.sh | 1 + scripts/linterVersions.sh | 12 +++++++++--- 7 files changed, 97 insertions(+), 15 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index c9f11251df5..dd3b7c58743 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -73,6 +73,9 @@ jobs: sudo rm -rf /opt/ghc || true sudo rm -rf /usr/local/.ghcup || true + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -127,6 +130,7 @@ jobs: cache-to: type=registry,ref=${{ env.CONTAINER_IMAGE_ID }}-buildcache,mode=max load: false push: true + platforms: linux/amd64,linux/arm64/v8 secrets: | GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} tags: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e8b9f915d7..acc9439df0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: name: Build and Test runs-on: ubuntu-latest permissions: - contents: read + contents: read concurrency: # Ref: https://docs.github.com/en/actions/learn-github-actions/contexts#github-context # github.head_ref: head_ref or source branch of the pull request @@ -27,9 +27,17 @@ jobs: images: - prefix: slim- target: slim + platform: linux/amd64 - prefix: "" target: standard - timeout-minutes: 60 + platform: linux/amd64 + - prefix: slim- + target: slim + platform: linux/arm64/v8 + - prefix: "" + target: standard + platform: linux/arm64/v8 + timeout-minutes: 180 env: CONTAINER_IMAGE_ID: "ghcr.io/super-linter/super-linter:${{ matrix.images.prefix }}latest" CONTAINER_IMAGE_TARGET: "${{ matrix.images.target }}" @@ -84,6 +92,9 @@ jobs: sudo rm -rf /opt/ghc || true sudo rm -rf /usr/local/.ghcup || true + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -99,6 +110,7 @@ jobs: cache-from: type=registry,ref=${{ env.CONTAINER_IMAGE_ID }}-buildcache outputs: type=docker,dest=/tmp/${{ env.CONTAINER_IMAGE_OUTPUT_IMAGE_NAME }}.tar push: false + platforms: ${{ matrix.images.platform }} secrets: | GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} tags: | @@ -114,6 +126,7 @@ jobs: - name: Test Local Action (debug log) uses: ./ + if: matrix.images.platform == 'linux/amd64' env: LOG_LEVEL: DEBUG CREATE_LOG_FILE: true @@ -125,11 +138,13 @@ jobs: TYPESCRIPT_STANDARD_TSCONFIG_FILE: ".github/linters/tsconfig.json" - name: Get the contents of the log file + if: matrix.images.platform == 'linux/amd64' run: | sudo cat super-linter.log sudo rm -v super-linter.log - name: Test Local Action (default log) + if: matrix.images.platform == 'linux/amd64' uses: ./ env: VALIDATE_ALL_CODEBASE: false @@ -144,6 +159,7 @@ jobs: run: make validate-container-image-labels - name: Upload ${{ env.CONTAINER_IMAGE_OUTPUT_IMAGE_NAME }} container image + if: matrix.images.platform == 'linux/amd64' uses: actions/upload-artifact@v4.3.2 with: name: ${{ env.CONTAINER_IMAGE_OUTPUT_IMAGE_NAME }} @@ -220,7 +236,7 @@ jobs: if: github.event_name == 'pull_request' && github.repository == github.event.pull_request.head.repo.full_name && github.repository == 'super-linter/super-linter' runs-on: ubuntu-latest permissions: - contents: read + contents: read steps: - uses: actions/checkout@v4 with: diff --git a/Dockerfile b/Dockerfile index 9ca92d7e7fa..548f089e60b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ FROM rhysd/actionlint:1.6.27 as actionlint FROM scalameta/scalafmt:v3.8.1 as scalafmt FROM zricethezav/gitleaks:v8.18.2 as gitleaks FROM yoheimuta/protolint:0.49.6 as protolint -FROM ghcr.io/clj-kondo/clj-kondo:2024.03.13-alpine as clj-kondo +FROM ghcr.io/clj-kondo/clj-kondo:2024.03.13 as clj-kondo FROM dart:3.3.4-sdk as dart FROM mcr.microsoft.com/dotnet/sdk:8.0.204-alpine3.19 as dotnet-sdk FROM mcr.microsoft.com/powershell:7.4-alpine-3.17 as powershell @@ -63,11 +63,17 @@ RUN apk add --no-cache \ SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-c"] +ARG TARGETARCH + COPY dependencies/python/ /stage WORKDIR /stage RUN ./build-venvs.sh && rm -rfv /stage -FROM python:3.12.3-alpine3.19 as npm-builder +# On arm64, `npm install` gives ECONNRESET errors: +# ERR! network Client network socket disconnected before secure TLS connection was established +# +# Since we only need node_modules, we can build it on an amd64 image instead +FROM --platform=$BUILDPLATFORM python:3.12.3-alpine3.19 as npm-builder RUN apk add --no-cache \ bash \ @@ -109,10 +115,10 @@ SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-c"] COPY scripts/install-lintr.sh scripts/install-r-package-or-fail.R / RUN /install-lintr.sh && rm -rf /install-lintr.sh /install-r-package-or-fail.R -FROM powershell as powershell-installer - +FROM --platform=linux/amd64 powershell as powershell-installer # Copy the value of the PowerShell install directory to a file so we can reuse it # when copying PowerShell stuff in the main image +RUN echo "${PS_VERSION}" > /tmp/PS_VERSION RUN echo "${PS_INSTALL_FOLDER}" > /tmp/PS_INSTALL_FOLDER FROM python:3.12.3-alpine3.19 as base_image @@ -348,7 +354,7 @@ COPY --from=kubeconfrm /kubeconform /usr/bin/ ##################### # Install clj-kondo # ##################### -COPY --from=clj-kondo /bin/clj-kondo /usr/bin/ +COPY --from=clj-kondo /usr/local/bin/clj-kondo /usr/bin/ #################### # Install dart-sdk # @@ -407,6 +413,8 @@ ENTRYPOINT ["/action/lib/linter.sh"] FROM base_image as slim +ARG TARGETARCH + # Run to build version file and validate image ENV IMAGE="slim" COPY scripts/linterVersions.sh / @@ -471,16 +479,26 @@ RUN dotnet help ######################################### # Install Powershell + PSScriptAnalyzer # ######################################### +COPY --from=powershell-installer /tmp/PS_VERSION /tmp/PS_VERSION COPY --from=powershell-installer /tmp/PS_INSTALL_FOLDER /tmp/PS_INSTALL_FOLDER COPY --from=powershell /opt/microsoft/powershell /opt/microsoft/powershell +ARG TARGETARCH # Disable Powershell telemetry ENV POWERSHELL_TELEMETRY_OPTOUT=1 ARG PSSA_VERSION='1.22.0' RUN PS_INSTALL_FOLDER="$(cat /tmp/PS_INSTALL_FOLDER)" \ && echo "PS_INSTALL_FOLDER: ${PS_INSTALL_FOLDER}" \ + && if [[ "${TARGETARCH}" == "amd64" ]]; then \ && ln -s "${PS_INSTALL_FOLDER}/pwsh" /usr/bin/pwsh \ && chmod a+x,o-w "${PS_INSTALL_FOLDER}/pwsh" \ && pwsh -c "Install-Module -Name PSScriptAnalyzer -RequiredVersion ${PSSA_VERSION} -Scope AllUsers -Force" \ + ; else \ + # Download arm64 version of PowerShell + PS_PACKAGE_URL="https://github.com/PowerShell/PowerShell/releases/download/v${PS_VERSION}/powershell-${PS_VERSION}-linux-arm64.tar.gz" \ + && wget -q -O - "${PS_PACKAGE_URL}" | tar -xz -C "${PS_INSTALL_FOLDER}" \ + && ln -s "${PS_INSTALL_FOLDER}/pwsh" /usr/bin/pwsh \ + && chmod a+x,o-w "${PS_INSTALL_FOLDER}/pwsh" \ + ; fi \ && rm -rf /tmp/PS_INSTALL_FOLDER ############################################################# @@ -491,6 +509,7 @@ RUN --mount=type=secret,id=GITHUB_TOKEN /install-arm-ttk.sh && rm -rf /install-a # Run to build version file and validate image again because we installed more linters ENV IMAGE="standard" +ARG TARGETARCH COPY scripts/linterVersions.sh / RUN /linterVersions.sh \ && rm -rfv /linterVersions.sh diff --git a/dependencies/python/build-venvs.sh b/dependencies/python/build-venvs.sh index 4b1fcfa7367..d75d329480e 100755 --- a/dependencies/python/build-venvs.sh +++ b/dependencies/python/build-venvs.sh @@ -8,11 +8,21 @@ ##################### set -euo pipefail -apk add --no-cache --virtual .python-build-deps \ - gcc \ - linux-headers \ - musl-dev \ +packages=( + gcc + linux-headers + musl-dev python3-dev +) + +if [[ "${TARGETARCH}" != "amd64" ]]; then + # libffi-dev is required for building wheel for cffi, + # until https://github.com/python-cffi/cffi/issues/69 lands in a release + packages+=(libffi-dev) +fi + +apk add --no-cache --virtual .python-build-deps \ + "${packages[@]}" ############################ # Create staging directory # @@ -24,6 +34,14 @@ mkdir -p /venvs ######################################## pip install virtualenv +if [[ "${TARGETARCH}" != "amd64" ]]; then + # Install Rust compiler (required by checkov on arm64) # + # remove this once https://github.com/bridgecrewio/checkov/pull/6045 is merged + apk add --no-cache curl + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + export PATH=$PATH:$HOME/.cargo/bin +fi + ####################################################### # Iterate through requirments.txt to install binaries # ####################################################### diff --git a/lib/functions/validation.sh b/lib/functions/validation.sh index 0c847ba1772..f423185088b 100755 --- a/lib/functions/validation.sh +++ b/lib/functions/validation.sh @@ -25,6 +25,24 @@ function ValidateBooleanConfigurationVariables() { ValidateBooleanVariable "YAML_ERROR_ON_WARNING" "${YAML_ERROR_ON_WARNING}" } +function ValidatePlatformArchitecture() { + local PLATFORM_ARCHITECTURE + PLATFORM_ARCHITECTURE=$(uname -m) + debug "Platform architecture: ${PLATFORM_ARCHITECTURE}" + + if [[ "${PLATFORM_ARCHITECTURE}" == "x86_64" ]]; then + return 0 + fi + + if [[ "${PLATFORM_ARCHITECTURE}" == "aarch64" || "${PLATFORM_ARCHITECTURE}" == "arm64" ]]; then + eval "export VALIDATE_SCALAFMT='false'" + eval "export VALIDATE_POWERSHELL='false'" + return 0 + fi + + fatal "Unsupported platform architecture: ${PLATFORM_ARCHITECTURE}" +} + function ValidateGitHubWorkspace() { local GITHUB_WORKSPACE GITHUB_WORKSPACE="${1}" diff --git a/lib/linter.sh b/lib/linter.sh index 76241a297cf..7a5ff7d747d 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -108,6 +108,7 @@ declare -l YAML_ERROR_ON_WARNING YAML_ERROR_ON_WARNING="${YAML_ERROR_ON_WARNING:-false}" ValidateBooleanConfigurationVariables +ValidatePlatformArchitecture ########### # GLOBALS # diff --git a/scripts/linterVersions.sh b/scripts/linterVersions.sh index 2f0e4239ec5..1e7c864f904 100755 --- a/scripts/linterVersions.sh +++ b/scripts/linterVersions.sh @@ -60,7 +60,6 @@ LINTER_NAMES_ARRAY['R']="R" LINTER_NAMES_ARRAY['RAKU']="raku" LINTER_NAMES_ARRAY['RENOVATE']="renovate-config-validator" LINTER_NAMES_ARRAY['RUBY']="rubocop" -LINTER_NAMES_ARRAY['SCALAFMT']="scalafmt" LINTER_NAMES_ARRAY['SHELL_SHFMT']="shfmt" LINTER_NAMES_ARRAY['SNAKEMAKE_LINT']="snakemake" LINTER_NAMES_ARRAY['SNAKEMAKE_SNAKEFMT']="snakefmt" @@ -79,15 +78,22 @@ LINTER_NAMES_ARRAY['TYPESCRIPT_STANDARD']="ts-standard" LINTER_NAMES_ARRAY['XML']="xmllint" LINTER_NAMES_ARRAY['YAML']="yamllint" +if [[ "${TARGETARCH}" == "amd64" ]]; then + LINTER_NAMES_ARRAY['SCALAFMT']="scalafmt" +fi + if [[ "${IMAGE}" == "standard" ]]; then LINTER_NAMES_ARRAY['ARM']="arm-ttk" LINTER_NAMES_ARRAY['CSHARP']="dotnet" - LINTER_NAMES_ARRAY['ENV']="dotenv-linter" - LINTER_NAMES_ARRAY['POWERSHELL']="pwsh" LINTER_NAMES_ARRAY['RUST_2015']="rustfmt" LINTER_NAMES_ARRAY['RUST_2018']="rustfmt" LINTER_NAMES_ARRAY['RUST_2021']="rustfmt" LINTER_NAMES_ARRAY['RUST_CLIPPY']="clippy" + + if [[ "${TARGETARCH}" == "amd64" ]]; then + LINTER_NAMES_ARRAY['ENV']="dotenv-linter" + LINTER_NAMES_ARRAY['POWERSHELL']="pwsh" + fi fi echo "Building linter version file: ${VERSION_FILE}"