Fix Docker and Clang workflow for sharing container, build artifacts

.github/workflows/clang.yml

@@ -11,9 +11,37 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  clang-format:
+  clang-docker:
     if: ${{ github.event.workflow_run.conclusion == 'success' }}
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        version: [ "5.0", "6.0", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17"]
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v4
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Check and download container artifact
+        if: ${{ github.ref_name != 'main' }}
+        id: container-artifact
+        uses: croemheld/download-artifact@v4.1.2-allow-not-found
+        with:
+          name: clang-${{ matrix.version }}
+          github-token: ${{ secrets.ARTIFACT_TOKEN }}
+          run-id: ${{ github.event.workflow_run.id }}
+          allow-not-found: true
+      - name: Load container artifact if available
+        if: ${{ steps.container-artifact.outputs.download-path != '' }}
+        run: |
+          docker load --input clang-${{ matrix.version }}.tar.gz
+  clang-format:
+    needs: clang-docker
+    runs-on: ubuntu-latest
     container: croemheld/icarus-ci:clang-15
     steps:
       - name: Checkout the repository
@@ -61,6 +89,8 @@ jobs:
         uses: actions/checkout@v4
       - name: Download ICARUS build artifact
         uses: actions/download-artifact@v4.1.2
+        with:
+          name: build
       - name: Run clang-tidy on ICARUS build
         run: |
           ./tools/run-clang-tidy.py -p build

.github/workflows/docker.yml

@@ -1,6 +1,12 @@
 name: Docker
 
-on: ["push"]
+on:
+  push:
+    paths-ignore:
+      - '**.md'
+  pull_request:
+    paths-ignore:
+      - '**.md'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
@@ -40,8 +46,24 @@ jobs:
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Free up GitHub runner space
+        run: |
+          sudo rm -rf "/usr/share/dotnet"
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
       - name: Build container image
         run: ./docker/build.sh --jobs 2 --llvm-version ${{ matrix.version }}
+      - name: Save compressed container image
+        if: ${{ github.ref_name != 'main' }}
+        run: |
+          docker save croemheld/icarus-ci:clang-${{ matrix.version }} | gzip > clang-${{ matrix.version }}.tar.gz
+      - name: Upload container image as artifact
+        if: ${{ github.ref_name != 'main' }}
+        uses: actions/upload-artifact@v4.3.1
+        with:
+          name: clang-${{ matrix.version }}
+          path: clang-${{ matrix.version }}.tar.gz
+          retention-days: 1
       - name: Push container image to Docker Hub
         if: ${{ github.ref_name == 'main' }}
         run: docker push croemheld/icarus-ci:clang-${{ matrix.version }}

docker/Dockerfile

@@ -72,7 +72,7 @@ ARG CMAKE_TARGETS
 
 RUN apt-get update && \
     apt-get install --no-install-recommends -y \
-        ca-certificates git wget build-essential ninja-build python3 zlib1g-dev && \
+        ca-certificates git wget build-essential ninja-build python3 python3-distutils zlib1g-dev && \
     apt-get clean && rm -rf /var/lib/apt/lists/* && \
     ./install-cmake.sh "${CMAKE_VERSION}" && \
     cd llvm-project && ./install-llvm.sh ${CMAKE_TARGETS}
@@ -96,7 +96,9 @@ ARG CMAKE_VERSION
 RUN apt-get update && \
     apt-get install --no-install-recommends -y \
         libc6-dev libstdc++-7-dev wget ca-certificates binutils ninja-build zlib1g-dev && \
-    apt-get install -y python3-pip python3-setuptools python3-venv libxml2-dev libxslt-dev python-dev && \
-    pip3 install gcovr && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get update && \
+    apt-get install --no-install-recommends -y \
+        python3-pip python3-setuptools python3-venv python3-lxml libxml2-dev libxslt-dev && \
     apt-get clean && rm -rf /var/lib/apt/lists/* && \
+    pip3 install gcovr wllvm && \
     ./install-cmake.sh "${CMAKE_VERSION}"

docker/README.md

@@ -31,10 +31,23 @@ There are two main reasons for providing Docker images with a custom LLVM instal
 
 ## Docker image size comparison
 
-Below is a graph which displays the Docker image sizes for a full LLVM installation (red) and a custom LLVM installation
-based on the required libraries for the **icarus** project.
+Below is a graph which displays the Docker image sizes for a full LLVM installation (red) building only the `clang` and
+some of the `clang-tools-extra` targets, and two custom LLVM builds based on the required libraries for the **icarus**
+project. The first one (green) depicts the size of the uncompressed Docker image with only the required LLVM libraries
+installed, while the second (yellow) also includes the [compiler-rt Runtime Library](https://compiler-rt.llvm.org/) as well as additional system
+and pip3 packages required for the code coverage tool `gcovr`.
 
 ![Docker image size comparison](docker-image-comparison.png)
 
-The custom LLVM installation required for the **icarus** project only require about 3 to 5 times less memory compared to
-the full LLVM installation.
+The custom LLVM builds required for the **icarus** project only require between 20% and 50% of space compared to
+the full LLVM installation. In addition, we now also have everything present for additional analysis of **icarus**,
+such as a code coverage tool as well as a 
+
+## Full LLVM installation
+
+Since we also want to have code coverage reports using the [compiler-rt Runtime Library](https://compiler-rt.llvm.org/),
+we need a full LLVM installation. This is because the runtime library is built using `llvm-config`, which collects the
+previously built libraries. If a single library is not installed on the system, the build process fails.
+
+It might be possible to make `llvm-config` think that all installed libraries are in fact thecomplete collection of LLVM
+libraries that exist.

docker/build-docker.sh

@@ -23,11 +23,13 @@ CMAKE_TARGETS=""
 PARALLEL_JOBS=""
 PRINT_DRY_RUN=""
 
+INSTALL_WHOLE=0
+
 function show_usage() {
   cat << EOF
 Usage: ./docker/build-docker.sh [-h | --help] [-s | --source <source>] [-r | --repository <repository>]
                                 [-t | --tag <tag>] [-c | --commit <commit or branch>] [-i | --install-target <target>]
-                                [-j | --jobs <jobs>] [-d | --dry-run]
+                                [-j | --jobs <jobs>] [-d | --dry-run] [-a | --install-all]
 EOF
 }
 
@@ -73,6 +75,9 @@ while [[ "${#}" -gt "0" ]]; do
     -d|--dry-run)
       PRINT_DRY_RUN="1"
       shift;;
+    -a|--install-all)
+      INSTALL_WHOLE=1
+      shift;;
     *)
       printf '%s\n' "Unknown argument '${1}'"
       exit 1;;
@@ -114,7 +119,12 @@ if [ "$(docker images -q "${DOCKER_REP}:base" 2> /dev/null)" == "" ]; then
     "${ICARUS_SRC}"
 fi
 
+if [ "${INSTALL_WHOLE}" -eq "1" ]; then
+  CMAKE_TARGETS="all"
+fi
+
 docker_build docker build \
+  --force-rm \
   -t "${DOCKER_REP}:${DOCKER_TAG}" \
   -f "${ICARUS_SRC}/docker/Dockerfile" \
   --build-arg "DOCKER_BASEIMG=${DOCKER_REP}:base" \

docker/build.sh

@@ -57,6 +57,9 @@ while [[ "${#}" -gt "0" ]]; do
       shift
       LLVM_VERSIONS+=("${1}")
       shift;;
+    --install-all)
+      INSTALL_TARGETS+=("-a")
+      shift;;
     *)
       printf '%s\n' "Unknown argument '${1}'"
       exit 1;;

docker/install-llvm.sh

@@ -14,52 +14,68 @@ CMAKE_ADDRESS="$(pwd)/llvm/CMakeLists.txt"
 LLVM_MAJORVER="$(grep 'LLVM_VERSION_MAJOR ' "${CMAKE_ADDRESS}" | awk -F'[()]' '{print $2}' | cut -d ' ' -f2)"
 CMAKE_TARGETS="${*}"
 
-#######################################
-# Version > 8 does not include stddef #
-# or any builtin headers, add target! #
-#######################################
+if [ "${CMAKE_TARGETS}" = "all" ]; then
+  NINJA_INSTALL="install"
+else
+  #######################################
+  # Version > 8 does not include stddef #
+  # or any builtin headers, add target! #
+  #######################################
 
-NINJA_TARGETS="clang clang-tidy clang-format clang-headers llvm-headers llvm-config cmake-exports"
+  NINJA_TARGETS="clang clang-tidy clang-format clang-headers llvm-headers llvm-config cmake-exports"
 
-if [ "${#CMAKE_TARGETS[@]}" -gt "0" ]; then
-  NINJA_TARGETS="${CMAKE_TARGETS} ${NINJA_TARGETS}"
-fi
+  if [ "${#CMAKE_TARGETS[@]}" -gt "0" ]; then
+    NINJA_TARGETS="${CMAKE_TARGETS} ${NINJA_TARGETS}"
+  fi
+
+  if [ "${LLVM_MAJORVER}" -gt "8" ]; then
+    NINJA_TARGETS="${NINJA_TARGETS} clang-resource-headers clang-cmake-exports"
+  fi
 
-if [ "${LLVM_MAJORVER}" -gt "8" ]; then
-  NINJA_TARGETS="${NINJA_TARGETS} clang-resource-headers clang-cmake-exports"
+  #######################################
+  # Only install minimum LLVM libraries #
+  #######################################
+
+  LIBRARY_NAMES=$(cmake -DLLVM_PACKAGE_VERSION="${LLVM_MAJORVER}" -P ../LLVMLibraries.cmake 2>&1 | xargs -d ';')
+  NINJA_INSTALL="$(printf '%s %s' "${NINJA_TARGETS}" "${LIBRARY_NAMES}" | sed 's/[^ ]* */install-&/g')"
 fi
 
 #######################################
-# Only install minimum LLVM libraries #
+# Build projects with CMake arguments #
 #######################################
 
-LIBRARY_NAMES=$(cmake -DLLVM_PACKAGE_VERSION="${LLVM_MAJORVER}" -P ../LLVMLibraries.cmake 2>&1 | xargs -d ';')
-NINJA_INSTALL="$(printf '%s %s' "${NINJA_TARGETS}" "${LIBRARY_NAMES}" | sed 's/[^ ]* */install-&/g')"
+cd build && echo "Install targets: ${NINJA_INSTALL}" && ninja ${NINJA_INSTALL}
+
+if [ "${CMAKE_TARGETS}" != "all" ]; then
+  #######################################
+  # Comment include of LLVMExports file #
+  #######################################
+
+  # We comment the include of the LLVMExports.cmake file in order to generate a docker image with the
+  # bare minimum of libraries that are needed for this project. If we include that file, we are going
+  # to get a CMake error saying that the LLVM libraries were not installed completely.
+  patch_cmake_include "LLVM_CMAKE_DIR" "LLVMExports.cmake" "/usr/local/lib/cmake/llvm/LLVMConfig.cmake"
+  if [ "${LLVM_MAJORVER}" -gt "8" ]; then
+    patch_cmake_include "CLANG_CMAKE_DIR" "ClangTargets.cmake" "/usr/local/lib/cmake/clang/ClangConfig.cmake"
+  fi
+fi
 
 #######################################
-# Build projects with CMake arguments #
+# Remove build directory to get space #
 #######################################
 
-cd build && echo "Install targets: ${NINJA_INSTALL}" && ninja ${NINJA_INSTALL}
+cd .. && rm -rf build
 
 #######################################
 # Compile compiler-rt with LLVM build #
 #######################################
 
-cd .. && mkdir -p build-compiler-rt && cd build-compiler-rt
+mkdir -p build-compiler-rt && cd build-compiler-rt
 cmake -G "Ninja" \
+  -DCMAKE_C_COMPILER=clang \
+  -DCMAKE_CXX_COMPILER=clang++ \
   -DLLVM_CONFIG_PATH=/usr/local/bin/llvm-config \
   ../compiler-rt
 ninja install
 
-#######################################
-# Comment include of LLVMExports file #
-#######################################
-
-# We comment the include of the LLVMExports.cmake file in order to generate a docker image with the
-# bare minimum of libraries that are needed for this project. If we include that file, we are going
-# to get a CMake error saying that the LLVM libraries were not installed completely.
-patch_cmake_include "LLVM_CMAKE_DIR" "LLVMExports.cmake" "/usr/local/lib/cmake/llvm/LLVMConfig.cmake"
-if [ "${LLVM_MAJORVER}" -gt "8" ]; then
-  patch_cmake_include "CLANG_CMAKE_DIR" "ClangTargets.cmake" "/usr/local/lib/cmake/clang/ClangConfig.cmake"
-fi
+cd .. && rm -rf build-compiler-rt