From d2c222713bebb218b1bd130d89068d9ceda4404c Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Tue, 22 Apr 2025 15:55:21 +0200 Subject: [PATCH 01/10] download node_modules of aardvark --- .../buildUbuntuDevel.docker/Dockerfile.arm64 | 71 +++++++++--------- .../buildUbuntuDevel.docker/Dockerfile.x86-64 | 74 ++++++++++--------- .../buildUbuntuDevel.docker/tools/install.sh | 5 +- 3 files changed, 80 insertions(+), 70 deletions(-) diff --git a/containers/buildUbuntuDevel.docker/Dockerfile.arm64 b/containers/buildUbuntuDevel.docker/Dockerfile.arm64 index 896eb753..c126ea07 100644 --- a/containers/buildUbuntuDevel.docker/Dockerfile.arm64 +++ b/containers/buildUbuntuDevel.docker/Dockerfile.arm64 @@ -3,7 +3,7 @@ ARG COMPILER_VERSION=13 ARG CLANG_VERSION=19 # First rebuild glibc with a special option and provide some .deb files: -FROM ubuntu:24.04 as glibcdebs +FROM ubuntu:24.04 AS glibcdebs COPY ./static /static RUN /static/prepare.sh @@ -12,27 +12,36 @@ RUN /static/rebuild_glibc.sh # Now prepare a standard Ubuntu image and install the provided .deb files # for glibc as a preparation: -FROM ubuntu:24.04 as prepare +FROM ubuntu:24.04 AS prepare -RUN apt-get update && apt-get upgrade -y && apt-get install -y libgd3 linux-libc-dev libcrypt-dev libnsl-dev rpcsvc-proto libtirpc-dev curl gnupg gnupg2 +RUN apt-get update --fix-missing && \ + apt-get upgrade -y && \ + apt-get install -y \ + software-properties-common net-tools lsb-release \ + build-essential cmake make bison flex python3 ccache git libjemalloc-dev \ + vim exuberant-ctags gdb fish psmisc sudo debhelper debconf jq libdb-dev \ + gcovr prometheus bc tcpdump liburing-dev cppcheck libopenblas-dev gfortran \ + cmake ninja-build git liblapack-dev python3-pip libgd3 linux-libc-dev libcrypt-dev \ + libnsl-dev rpcsvc-proto libtirpc-dev curl gnupg gnupg2 \ + && rm -rf /var/cache/apt/archives /var/lib/apt/lists COPY --from=glibcdebs /root/debs /debs RUN dpkg -i /debs/*.deb # Now build V8: -#FROM ubuntu:24.04 as v81 +#FROM ubuntu:24.04 AS v81 # #COPY ./tools/build_v8_arm64.sh /tools/build_v8_arm64.sh #RUN /tools/build_v8_arm64.sh # -#FROM v81 as v8 +#FROM v81 AS v8 # #COPY ./tools/install_v8_arm64.sh /tools/install_v8_arm64.sh #RUN /tools/install_v8_arm64.sh # Build OpenMP static library -FROM ubuntu:24.04 as openmp +FROM ubuntu:24.04 AS openmp ARG COMPILER_VERSION ARG CLANG_VERSION @@ -40,7 +49,8 @@ ARG CLANG_FULL_VERSION RUN apt-get update --fix-missing && \ apt-get upgrade -y && \ - apt-get install -y software-properties-common net-tools wget lsb-release + apt-get install -y software-properties-common net-tools curl lsb-release dpkg-dev git \ + && rm -rf /var/cache/apt/archives /var/lib/apt/lists WORKDIR / @@ -48,16 +58,14 @@ COPY static/llvm.sources ./ RUN cat llvm.sources >> /etc/apt/sources.list.d/ubuntu.sources -RUN wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc - -RUN apt-get update && apt-get install dpkg-dev -y +RUN curl -o- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc -RUN apt-get source llvm-${CLANG_VERSION}-dev -y && \ - apt-get build-dep llvm-${CLANG_VERSION}-dev -y - -RUN apt-get update --fix-missing && apt-get install -y \ - clang-${CLANG_VERSION} clang++-${CLANG_VERSION} \ - cmake ninja-build git +RUN apt-get update --fix-missing && \ + apt-get source llvm-${CLANG_VERSION}-dev -y && \ + apt-get build-dep -y llvm-${CLANG_VERSION}-dev && \ + apt-get update --fix-missing && \ + apt-get install -y clang-${CLANG_VERSION} clang++-${CLANG_VERSION} && \ + rm -rf /var/cache/apt/archives /var/lib/apt/lists COPY patches/diff_llvm.patch . @@ -94,27 +102,21 @@ ARG SCCACHE_VERSION="0.9.1" ENV COMPILER_VERSION=${COMPILER_VERSION} ENV CLANG_VERSION=${CLANG_VERSION} -RUN apt-get update --fix-missing && \ - apt-get install -y software-properties-common net-tools wget && \ - apt-get update && \ - apt-get upgrade -y - COPY static/llvm.sources ./ RUN cat llvm.sources >> /etc/apt/sources.list.d/ubuntu.sources -RUN wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc +RUN curl -o- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update --fix-missing && \ - apt-get install -y build-essential gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION} \ - cmake make bison flex python3 ccache git libjemalloc-dev vim exuberant-ctags gdb fish \ - psmisc sudo debhelper debconf jq wget libdb-dev gcovr prometheus bc tcpdump \ - liburing-dev cppcheck clang-${CLANG_VERSION} lldb-${CLANG_VERSION} lld-${CLANG_VERSION} \ - libc++-${CLANG_VERSION}-dev libc++abi-${CLANG_VERSION}-dev libclang-common-${CLANG_VERSION}-dev \ - libclang-rt-${CLANG_VERSION}-dev libomp-${CLANG_VERSION}-dev liblapack-dev libopenblas-dev gfortran \ - && apt-get clean + apt-get install -y gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION} \ + clang-${CLANG_VERSION} lldb-${CLANG_VERSION} lld-${CLANG_VERSION} \ + libc++-${CLANG_VERSION}-dev libc++abi-${CLANG_VERSION}-dev libclang-common-${CLANG_VERSION}-dev \ + libclang-rt-${CLANG_VERSION}-dev libomp-${CLANG_VERSION}-dev && \ + rm -rf /var/cache/apt/archives /var/lib/apt/lists + # sccache for cloud compiler cache: @@ -131,9 +133,12 @@ RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | b nvm use ${NODE_VERSION} && \ node --version && \ npm install -g yarn && \ - ln -s /root/.nvm/versions/node/v${NODE_VERSION}/bin/* /usr/local/bin/ - - + ln -s /root/.nvm/versions/node/v${NODE_VERSION}/bin/* /usr/local/bin/ +RUN cd / ; git clone -n --depth=1 --filter=tree:0 https://github.com/arangodb/arangodb && \ + cd /arangodb && git sparse-checkout set --no-cone js/apps/system/_admin/aardvark/APP/react && \ + git checkout && \ + cd /arangodb/js/apps/system/_admin/aardvark/APP/react && yarn install && mv node_modules / && \ + cd /; rm -rf /arangodb # Fetch V8: #COPY --from=v8 /opt/v8/lib /opt/v8/lib @@ -162,6 +167,6 @@ RUN LC_ALL="C" update-ccache-symlinks RUN git config --global --add safe.directory '*' -RUN apt-get install -y python3-pip && pip3 install py7zr psutil cppcheck-junit --break-system-packages +RUN pip3 install py7zr psutil cppcheck-junit lcov_cobertura --break-system-packages CMD [ "/usr/bin/fish" ] diff --git a/containers/buildUbuntuDevel.docker/Dockerfile.x86-64 b/containers/buildUbuntuDevel.docker/Dockerfile.x86-64 index fac65196..6ab78a47 100644 --- a/containers/buildUbuntuDevel.docker/Dockerfile.x86-64 +++ b/containers/buildUbuntuDevel.docker/Dockerfile.x86-64 @@ -3,7 +3,7 @@ ARG COMPILER_VERSION=13 ARG CLANG_VERSION=19 # First rebuild glibc with a special option and provide some .deb files: -FROM ubuntu:24.04 as glibcdebs +FROM ubuntu:24.04 AS glibcdebs COPY ./static /static RUN /static/prepare.sh @@ -12,34 +12,44 @@ RUN /static/rebuild_glibc.sh # Now prepare a standard Ubuntu image and install the provided .deb files # for glibc as a preparation: -FROM ubuntu:24.04 as prepare +FROM ubuntu:24.04 AS prepare -RUN apt-get update && apt-get upgrade -y && apt-get install -y libgd3 linux-libc-dev libcrypt-dev libnsl-dev rpcsvc-proto libtirpc-dev curl gnupg gnupg2 +RUN apt-get update --fix-missing && \ + apt-get upgrade -y && \ + apt-get install -y \ + software-properties-common net-tools lsb-release \ + build-essential cmake make bison flex python3 ccache git libjemalloc-dev \ + vim exuberant-ctags gdb fish psmisc sudo debhelper debconf jq libdb-dev \ + gcovr prometheus bc tcpdump liburing-dev cppcheck libopenblas-dev gfortran \ + cmake ninja-build git liblapack-dev python3-pip libgd3 linux-libc-dev libcrypt-dev \ + libnsl-dev rpcsvc-proto libtirpc-dev curl gnupg gnupg2 \ + && rm -rf /var/cache/apt/archives /var/lib/apt/lists COPY --from=glibcdebs /root/debs /debs RUN dpkg -i /debs/*.deb # Now build V8: -#FROM ubuntu:24.04 as v81 +#FROM ubuntu:24.04 AS v81 # #COPY ./tools/build_v8.sh /tools/build_v8.sh #RUN /tools/build_v8.sh # -#FROM v81 as v8 +#FROM v81 AS v8 # #COPY ./tools/install_v8.sh /tools/install_v8.sh #RUN /tools/install_v8.sh # Build OpenMP static library -FROM ubuntu:24.04 as openmp +FROM ubuntu:24.04 AS openmp ARG COMPILER_VERSION ARG CLANG_VERSION RUN apt-get update --fix-missing && \ apt-get upgrade -y && \ - apt-get install -y software-properties-common net-tools wget lsb-release + apt-get install -y software-properties-common net-tools curl lsb-release dpkg-dev git \ + && rm -rf /var/cache/apt/archives /var/lib/apt/lists WORKDIR / @@ -47,16 +57,14 @@ COPY static/llvm.sources ./ RUN cat llvm.sources >> /etc/apt/sources.list.d/ubuntu.sources -RUN wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc - -RUN apt-get update && apt-get install dpkg-dev -y +RUN curl -o- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc -RUN apt-get source llvm-${CLANG_VERSION}-dev -y && \ - apt-get build-dep llvm-${CLANG_VERSION}-dev -y - -RUN apt-get update --fix-missing && apt-get install -y \ - clang-${CLANG_VERSION} clang++-${CLANG_VERSION} \ - cmake ninja-build git +RUN apt-get update --fix-missing && \ + apt-get source llvm-${CLANG_VERSION}-dev -y && \ + apt-get build-dep -y llvm-${CLANG_VERSION}-dev && \ + apt-get update --fix-missing && \ + apt-get install -y clang-${CLANG_VERSION} clang++-${CLANG_VERSION} && \ + rm -rf /var/cache/apt/archives /var/lib/apt/lists COPY patches/diff_llvm.patch . @@ -72,9 +80,8 @@ WORKDIR build RUN cmake -G Ninja -DLLVM_ENABLE_PROJECTS="clang;openmp" -DCMAKE_C_COMPILER=clang-${CLANG_VERSION} \ -DCMAKE_CXX_COMPILER=clang++-${CLANG_VERSION} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DLIBOMP_ENABLE_SHARED=OFF \ -DENABLE_OMPT_TOOLS=OFF -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_INSTALL_PREFIX=/usr \ - -DENABLE_LIBOMPTARGET=OFF -DLIBOMP_INSTALL_ALIASES=OFF ../llvm && ninja - -RUN mkdir -p /opt/omp && cp ./lib/libomp.so.5 /opt/omp/libomp.a + -DENABLE_LIBOMPTARGET=OFF -DLIBOMP_INSTALL_ALIASES=OFF ../llvm && ninja && \ + mkdir -p /opt/omp && cp ./lib/libomp.so.5 /opt/omp/libomp.a # Finally, build the actual build image based on the above preparation: @@ -93,27 +100,20 @@ ARG SCCACHE_VERSION="0.9.1" ENV COMPILER_VERSION=${COMPILER_VERSION} ENV CLANG_VERSION=${CLANG_VERSION} -RUN apt-get update --fix-missing && \ - apt-get install -y software-properties-common net-tools wget && \ - apt-get update && \ - apt-get upgrade -y - COPY static/llvm.sources ./ RUN cat llvm.sources >> /etc/apt/sources.list.d/ubuntu.sources -RUN wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc +RUN curl -o- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update --fix-missing && \ - apt-get install -y build-essential gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION} \ - cmake make bison flex python3 ccache git libjemalloc-dev vim exuberant-ctags gdb fish \ - psmisc sudo debhelper debconf jq wget libdb-dev gcovr prometheus bc tcpdump \ - liburing-dev cppcheck clang-${CLANG_VERSION} lldb-${CLANG_VERSION} lld-${CLANG_VERSION} \ - libc++-${CLANG_VERSION}-dev libc++abi-${CLANG_VERSION}-dev libclang-common-${CLANG_VERSION}-dev \ - libclang-rt-${CLANG_VERSION}-dev libomp-${CLANG_VERSION}-dev liblapack-dev libopenblas-dev gfortran \ - && apt-get clean + apt-get install -y gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION} \ + clang-${CLANG_VERSION} lldb-${CLANG_VERSION} lld-${CLANG_VERSION} \ + libc++-${CLANG_VERSION}-dev libc++abi-${CLANG_VERSION}-dev libclang-common-${CLANG_VERSION}-dev \ + libclang-rt-${CLANG_VERSION}-dev libomp-${CLANG_VERSION}-dev findutils && \ + rm -rf /var/cache/apt/archives /var/lib/apt/lists # sccache for cloud compiler cache: @@ -131,8 +131,12 @@ RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | b nvm use ${NODE_VERSION} && \ node --version && \ npm install -g yarn && \ - ln -s /root/.nvm/versions/node/v${NODE_VERSION}/bin/* /usr/local/bin/ - + ln -s /root/.nvm/versions/node/v${NODE_VERSION}/bin/* /usr/local/bin/ +RUN cd / ; git clone -n --depth=1 --filter=tree:0 https://github.com/arangodb/arangodb && \ + cd /arangodb && git sparse-checkout set --no-cone js/apps/system/_admin/aardvark/APP/react && \ + git checkout && \ + cd /arangodb/js/apps/system/_admin/aardvark/APP/react && yarn install && mv node_modules / && \ + cd /; rm -rf /arangodb # Fetch V8: #COPY --from=v8 /opt/v8/lib /opt/v8/lib @@ -161,6 +165,6 @@ RUN LC_ALL="C" update-ccache-symlinks RUN git config --global --add safe.directory '*' -RUN apt-get install -y python3-pip && pip3 install py7zr psutil cppcheck-junit lcov_cobertura --break-system-packages +RUN pip3 install py7zr psutil cppcheck-junit lcov_cobertura --break-system-packages CMD [ "/usr/bin/fish" ] diff --git a/containers/buildUbuntuDevel.docker/tools/install.sh b/containers/buildUbuntuDevel.docker/tools/install.sh index ccff3c8f..2c9dc307 100755 --- a/containers/buildUbuntuDevel.docker/tools/install.sh +++ b/containers/buildUbuntuDevel.docker/tools/install.sh @@ -28,8 +28,9 @@ export OPENSSLPATH=`echo $OPENSSLVERSION | sed 's/\.[0-9]*$//g'` [ "$ARCH" = "x86_64" -a ${OPENSSLPATH:0:1} = "3" ] && X86_64_SUFFIX=64 cd /tmp -wget https://www.openssl.org/source/openssl-$OPENSSLVERSION.tar.gz -tar xzf openssl-$OPENSSLVERSION.tar.gz +TARBALL=openssl-$OPENSSLVERSION.tar.gz +curl -L --output $TARBALL https://www.openssl.org/source/$TARBALL +tar xzf $TARBALL cd openssl-$OPENSSLVERSION ./config --prefix=/opt no-async no-dso make -j$(nproc) || exit 1 From ce371262e2ce707e4c02cc2181ad840dc7823a4a Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Tue, 22 Apr 2025 15:57:25 +0200 Subject: [PATCH 02/10] forward --- helper.linux.fish | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper.linux.fish b/helper.linux.fish index 5a1a45e4..0c7523f3 100755 --- a/helper.linux.fish +++ b/helper.linux.fish @@ -22,7 +22,7 @@ else end set -gx UBUNTUBUILDIMAGE_312_NAME arangodb/ubuntubuildarangodb-devel -set -gx UBUNTUBUILDIMAGE_312_TAG 13 +set -gx UBUNTUBUILDIMAGE_312_TAG 14 set -gx UBUNTUBUILDIMAGE_312 $UBUNTUBUILDIMAGE_312_NAME:$UBUNTUBUILDIMAGE_312_TAG-$UBUNTUBUILDIMAGE_TAG_ARCH set -gx UBUNTUBUILDIMAGE_311_NAME arangodb/ubuntubuildarangodb-311 From f74ceef0b1f693342d68131729f1b859ae829781 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Wed, 23 Apr 2025 10:38:00 +0200 Subject: [PATCH 03/10] download node modules in its own container step --- .../buildUbuntuDevel.docker/Dockerfile.arm64 | 46 ++++++++++--------- .../buildUbuntuDevel.docker/Dockerfile.x86-64 | 45 ++++++++++-------- 2 files changed, 50 insertions(+), 41 deletions(-) diff --git a/containers/buildUbuntuDevel.docker/Dockerfile.arm64 b/containers/buildUbuntuDevel.docker/Dockerfile.arm64 index c126ea07..b5f3a2c7 100644 --- a/containers/buildUbuntuDevel.docker/Dockerfile.arm64 +++ b/containers/buildUbuntuDevel.docker/Dockerfile.arm64 @@ -81,10 +81,31 @@ WORKDIR build RUN cmake -G Ninja -DLLVM_ENABLE_PROJECTS="clang;openmp" -DCMAKE_C_COMPILER=clang-${CLANG_VERSION} \ -DCMAKE_CXX_COMPILER=clang++-${CLANG_VERSION} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DLIBOMP_ENABLE_SHARED=OFF \ -DENABLE_OMPT_TOOLS=OFF -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_INSTALL_PREFIX=/usr \ - -DENABLE_LIBOMPTARGET=OFF -DLIBOMP_INSTALL_ALIASES=OFF ../llvm && ninja + -DENABLE_LIBOMPTARGET=OFF -DLIBOMP_INSTALL_ALIASES=OFF ../llvm && ninja && \ + mkdir -p /opt/omp && cp ./lib/libomp.so.5 /opt/omp/libomp.a -RUN mkdir -p /opt/omp && cp ./lib/libomp.so.5 /opt/omp/libomp.a +FROM prepare AS nodejs +# yarn for UI build: +ARG NODE_VERSION="20.14.0" +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash && \ + export NVM_DIR="$HOME/.nvm" &&\ + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" \ + [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" && \ + nvm install ${NODE_VERSION} && \ + nvm use ${NODE_VERSION} && \ + node --version && \ + npm install -g yarn && \ + ln -s /root/.nvm/versions/node/v${NODE_VERSION}/bin/* /usr/local/bin/ + + +FROM nodejs AS node_modules + +RUN cd / ; git clone -n --depth=1 --filter=tree:0 https://github.com/arangodb/arangodb && \ + cd /arangodb && git sparse-checkout set --no-cone js/apps/system/_admin/aardvark/APP/react && \ + git checkout && \ + cd /arangodb/js/apps/system/_admin/aardvark/APP/react && yarn install && mv node_modules / && \ + cd / && rm -rf /arangodb && tar -cvJf /root/node_modules.tar.xz node_modules && rm -rf /node_modules # Finally, build the actual build image based on the above preparation: @@ -117,29 +138,10 @@ RUN apt-get update --fix-missing && \ libclang-rt-${CLANG_VERSION}-dev libomp-${CLANG_VERSION}-dev && \ rm -rf /var/cache/apt/archives /var/lib/apt/lists - # sccache for cloud compiler cache: RUN curl -L https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-${ARCH}-unknown-linux-musl.tar.gz | tar xvz -C /tmp && mv /tmp/sccache-v${SCCACHE_VERSION}-${ARCH}-unknown-linux-musl/sccache /usr/bin/sccache && chmod +x /usr/bin/sccache && rm -rf /tmp/sccache-v${SCCACHE_VERSION}-${ARCH}-unknown-linux-musl - -# yarn for UI build: -ARG NODE_VERSION="20.14.0" -RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash && \ - export NVM_DIR="$HOME/.nvm" &&\ - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" \ - [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" && \ - nvm install ${NODE_VERSION} && \ - nvm use ${NODE_VERSION} && \ - node --version && \ - npm install -g yarn && \ - ln -s /root/.nvm/versions/node/v${NODE_VERSION}/bin/* /usr/local/bin/ -RUN cd / ; git clone -n --depth=1 --filter=tree:0 https://github.com/arangodb/arangodb && \ - cd /arangodb && git sparse-checkout set --no-cone js/apps/system/_admin/aardvark/APP/react && \ - git checkout && \ - cd /arangodb/js/apps/system/_admin/aardvark/APP/react && yarn install && mv node_modules / && \ - cd /; rm -rf /arangodb - # Fetch V8: #COPY --from=v8 /opt/v8/lib /opt/v8/lib #COPY --from=v8 /opt/v8/include /opt/v8/include @@ -167,6 +169,8 @@ RUN LC_ALL="C" update-ccache-symlinks RUN git config --global --add safe.directory '*' +COPY --from=node_modules /root/node_modules.tar.xz /node_modules.tar.xz + RUN pip3 install py7zr psutil cppcheck-junit lcov_cobertura --break-system-packages CMD [ "/usr/bin/fish" ] diff --git a/containers/buildUbuntuDevel.docker/Dockerfile.x86-64 b/containers/buildUbuntuDevel.docker/Dockerfile.x86-64 index 6ab78a47..0e313a3b 100644 --- a/containers/buildUbuntuDevel.docker/Dockerfile.x86-64 +++ b/containers/buildUbuntuDevel.docker/Dockerfile.x86-64 @@ -84,9 +84,31 @@ RUN cmake -G Ninja -DLLVM_ENABLE_PROJECTS="clang;openmp" -DCMAKE_C_COMPILER=clan mkdir -p /opt/omp && cp ./lib/libomp.so.5 /opt/omp/libomp.a +FROM prepare AS nodejs +# yarn for UI build: +ARG NODE_VERSION="20.14.0" +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash && \ + export NVM_DIR="$HOME/.nvm" &&\ + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" \ + [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" && \ + nvm install ${NODE_VERSION} && \ + nvm use ${NODE_VERSION} && \ + node --version && \ + npm install -g yarn && \ + ln -s /root/.nvm/versions/node/v${NODE_VERSION}/bin/* /usr/local/bin/ + + +FROM nodejs AS node_modules + +RUN cd / ; git clone -n --depth=1 --filter=tree:0 https://github.com/arangodb/arangodb && \ + cd /arangodb && git sparse-checkout set --no-cone js/apps/system/_admin/aardvark/APP/react && \ + git checkout && \ + cd /arangodb/js/apps/system/_admin/aardvark/APP/react && yarn install && mv node_modules / && \ + cd / && rm -rf /arangodb && tar -cvJf /root/node_modules.tar.xz node_modules && rm -rf /node_modules + # Finally, build the actual build image based on the above preparation: -FROM prepare +FROM nodejs MAINTAINER Max Neunhoeffer @@ -115,29 +137,10 @@ RUN apt-get update --fix-missing && \ libclang-rt-${CLANG_VERSION}-dev libomp-${CLANG_VERSION}-dev findutils && \ rm -rf /var/cache/apt/archives /var/lib/apt/lists - # sccache for cloud compiler cache: RUN curl -L https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-${ARCH}-unknown-linux-musl.tar.gz | tar xvz -C /tmp && mv /tmp/sccache-v${SCCACHE_VERSION}-${ARCH}-unknown-linux-musl/sccache /usr/bin/sccache && chmod +x /usr/bin/sccache && rm -rf /tmp/sccache-v${SCCACHE_VERSION}-${ARCH}-unknown-linux-musl - -# yarn for UI build: -ARG NODE_VERSION="20.14.0" -RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash && \ - export NVM_DIR="$HOME/.nvm" &&\ - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" \ - [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" && \ - nvm install ${NODE_VERSION} && \ - nvm use ${NODE_VERSION} && \ - node --version && \ - npm install -g yarn && \ - ln -s /root/.nvm/versions/node/v${NODE_VERSION}/bin/* /usr/local/bin/ -RUN cd / ; git clone -n --depth=1 --filter=tree:0 https://github.com/arangodb/arangodb && \ - cd /arangodb && git sparse-checkout set --no-cone js/apps/system/_admin/aardvark/APP/react && \ - git checkout && \ - cd /arangodb/js/apps/system/_admin/aardvark/APP/react && yarn install && mv node_modules / && \ - cd /; rm -rf /arangodb - # Fetch V8: #COPY --from=v8 /opt/v8/lib /opt/v8/lib #COPY --from=v8 /opt/v8/include /opt/v8/include @@ -165,6 +168,8 @@ RUN LC_ALL="C" update-ccache-symlinks RUN git config --global --add safe.directory '*' +COPY --from=node_modules /root/node_modules.tar.xz /node_modules.tar.xz + RUN pip3 install py7zr psutil cppcheck-junit lcov_cobertura --break-system-packages CMD [ "/usr/bin/fish" ] From dbfff8a2773114089c095d25f44f34e6c6f6926a Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Wed, 23 Apr 2025 10:51:46 +0200 Subject: [PATCH 04/10] next version --- helper.linux.fish | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper.linux.fish b/helper.linux.fish index 52d55081..a6d5b65f 100755 --- a/helper.linux.fish +++ b/helper.linux.fish @@ -22,7 +22,7 @@ else end set -gx UBUNTUBUILDIMAGE_312_NAME arangodb/ubuntubuildarangodb-devel -set -gx UBUNTUBUILDIMAGE_312_TAG 14 +set -gx UBUNTUBUILDIMAGE_312_TAG 15 set -gx UBUNTUBUILDIMAGE_312 $UBUNTUBUILDIMAGE_312_NAME:$UBUNTUBUILDIMAGE_312_TAG-$UBUNTUBUILDIMAGE_TAG_ARCH set -gx UBUNTUBUILDIMAGE_311_NAME arangodb/ubuntubuildarangodb-311 From 43a450a9a18ea11ee68a868d30b217b7c2bc378e Mon Sep 17 00:00:00 2001 From: Vadim Kondratev Date: Wed, 23 Apr 2025 14:06:02 +0200 Subject: [PATCH 05/10] Avoid duplication --- scripts/buildArangoDB312.fish | 148 +--------------------------------- 1 file changed, 1 insertion(+), 147 deletions(-) mode change 100755 => 120000 scripts/buildArangoDB312.fish diff --git a/scripts/buildArangoDB312.fish b/scripts/buildArangoDB312.fish deleted file mode 100755 index a5fe96e8..00000000 --- a/scripts/buildArangoDB312.fish +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env fish -# This is for static gcc13.2.0 and clang19.1.7 builds on Ubuntu -source ./scripts/lib/build.fish - -if test "$PARALLELISM" = "" - set -xg PARALLELISM 64 -end - -if test "$SAN" = "On" - set PARALLELISM 10 -end - -echo "Using parallelism $PARALLELISM" - -if test "$COMPILER_VERSION" = "" - set -xg COMPILER_VERSION clang19.1.7 -end -echo "Using compiler version $COMPILER_VERSION" - -if test "$COMPILER_VERSION" = "clang19.1.7" - set -xg CC_NAME clang - set -xg CXX_NAME clang++ -else if test "$COMPILER_VERSION" = "13.2.0" - set -xg CC_NAME gcc - set -xg CXX_NAME g++ -else - set -xg CC_NAME gcc-$COMPILER_VERSION - set -xg CXX_NAME g++-$COMPILER_VERSION -end - -if test "$OPENSSL_VERSION" = "" - set -xg OPENSSL_VERSION 3.4 -end -echo "Using openssl version $OPENSSL_VERSION" - -if test "$ARCH" = "x86_64" -a (string sub -s 1 -l 1 "$OPENSSLPATH") = "3" - set -xg X86_64_SUFFIX "64" -end - -if test "$ARCH" = "x86_64" - set -xg LAPACK_LIB_PATH "/usr/lib/x86_64-linux-gnu/lapack/liblapack.a" - set -xg BLAS_LIB_PATH "/usr/lib/x86_64-linux-gnu/blas/libblas.a" -else - set -xg LAPACK_LIB_PATH "/usr/lib/aarch64-linux-gnu/lapack/liblapack.a" - set -xg BLAS_LIB_PATH "/usr/lib/aarch64-linux-gnu/blas/libblas.a" -end - -set -l pie "" - -if test "$STATIC_EXECUTABLES" = "" - set -xg STATIC_EXECUTABLES On -end - -set -g FULLARGS $argv \ - -DCMAKE_BUILD_TYPE=$BUILDMODE \ - -DCMAKE_INSTALL_PREFIX=/ \ - -DSTATIC_EXECUTABLES=$STATIC_EXECUTABLES \ - -DUSE_ENTERPRISE=$ENTERPRISEEDITION \ - -DUSE_MAINTAINER_MODE=$MAINTAINER \ - -DCMAKE_LIBRARY_PATH="/opt/lib$X86_64_SUFFIX;/opt/lib" \ - -DOPENSSL_ROOT_DIR=/opt \ - -DUSE_STRICT_OPENSSL_VERSION=$USE_STRICT_OPENSSL \ - -DBUILD_REPO_INFO=$BUILD_REPO_INFO \ - -DARANGODB_BUILD_DATE="$ARANGODB_BUILD_DATE" \ - -DLAPACK_LIBRARIES="$LAPACK_LIB_PATH" \ - -DBLAS_LIBRARIES="$BLAS_LIB_PATH" - -if test "$MAINTAINER" = "On" - set -g FULLARGS $FULLARGS \ - -DCMAKE_EXE_LINKER_FLAGS="-Wl,--build-id=sha1 $pie -fno-stack-protector -fuse-ld=lld -fopenmp=libomp -L/opt/omp -llapack -lgfortran" \ - -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld" -else - set -g FULLARGS $FULLARGS \ - -DCMAKE_EXE_LINKER_FLAGS="-Wl,--build-id=sha1 $pie $inline -fno-stack-protector -fuse-ld=lld -fopenmp=libomp -L/opt/omp -llapack -lgfortran" \ - -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld" \ - -DUSE_CATCH_TESTS=Off \ - -DUSE_GOOGLE_TESTS=Off -end - -if test "$BUILD_SEPP" = "On" - set -g FULLARGS $FULLARGS -DBUILD_SEPP=ON -end - -if test "$SAN" = "On" - # Suppress leaks detection only during building - set -gx SAN_OPTIONS "detect_leaks=0" - set -l SANITIZERS "-fsanitize=address -fsanitize=undefined -fsanitize=float-divide-by-zero -fsanitize=leak -fsanitize-address-use-after-return=never" - if test "$SAN_MODE" = "TSan" - set SANITIZERS "-fsanitize=thread" - end - set -g FULLARGS $FULLARGS \ - -DUSE_JEMALLOC=Off \ - -DCMAKE_C_FLAGS="-pthread $SANITIZERS -fno-sanitize=alignment" \ - -DCMAKE_CXX_FLAGS="-pthread $SANITIZERS -fno-sanitize=vptr -fno-sanitize=alignment" \ - -DBASE_LIBS="-pthread" -else if test "$COVERAGE" = "On" - echo "Building with LCOV Coverage" - set -g FULLARGS $FULLARGS \ - -DUSE_JEMALLOC=$JEMALLOC_OSKAR \ - -DCMAKE_C_FLAGS="$pie -fno-stack-protector -fprofile-instr-generate -fcoverage-mapping -mllvm -runtime-counter-relocation --coverage" \ - -DCMAKE_CXX_FLAGS="$pie -fno-stack-protector -fprofile-instr-generate -fcoverage-mapping -mllvm -runtime-counter-relocation --coverage" \ - -DCMAKE_LD_FLAGS="$pie -fno-stack-protector -fprofile-instr-generate -fcoverage-mapping -mllvm -runtime-counter-relocation --coverage" \ - -DUSE_COVERAGE=ON \ - -DV8_LDFLAGS=--coverage -else - set -g FULLARGS $FULLARGS \ - -DUSE_JEMALLOC=$JEMALLOC_OSKAR - - if test "$MAINTAINER" = "On" - set -g FULLARGS $FULLARGS \ - -DCMAKE_C_FLAGS="$pie -fno-stack-protector" \ - -DCMAKE_CXX_FLAGS="$pie -fno-stack-protector" - else - set -g FULLARGS $FULLARGS \ - -DCMAKE_C_FLAGS="$pie $inline -fno-stack-protector" \ - -DCMAKE_CXX_FLAGS="$pie $inline -fno-stack-protector" - end -end - -if test "$MINIMAL_DEBUG_INFO" = "On" - set -g FULLARGS $FULLARGS \ - -DUSE_MINIMAL_DEBUGINFO=On -end - -setupCcacheBinPath ubuntu -and setupCcache ubuntu -and cleanBuildDirectory -and cd $INNERWORKDIR/ArangoDB/build -and TT_init -and cmakeCcache -and selectArchitecture -and selectMaintainer -and runCmake -and TT_cmake -and if test "$SKIP_MAKE" = "On" - echo "Finished cmake at "(date)", skipping build" -else - echo "Finished cmake at "(date)", now starting build" - and set -xg DESTDIR (pwd)/install - and runMake install - and generateJsSha1Sum ArangoDB/build/install/usr/share/arangodb3/js - and TT_make - and installTargets - and echo "Finished at "(date) - and shutdownCcache - and TT_strip -end diff --git a/scripts/buildArangoDB312.fish b/scripts/buildArangoDB312.fish new file mode 120000 index 00000000..1bed2e68 --- /dev/null +++ b/scripts/buildArangoDB312.fish @@ -0,0 +1 @@ +buildArangoDBDevel.fish \ No newline at end of file From 8416b0b993e036c3eed3e6e2f13c264eb8ac0fd1 Mon Sep 17 00:00:00 2001 From: Vadim Kondratev Date: Wed, 23 Apr 2025 15:29:28 +0300 Subject: [PATCH 06/10] Update buildArangoDBDevel.fish --- scripts/buildArangoDBDevel.fish | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/buildArangoDBDevel.fish b/scripts/buildArangoDBDevel.fish index c625df7b..10cfacc9 100755 --- a/scripts/buildArangoDBDevel.fish +++ b/scripts/buildArangoDBDevel.fish @@ -63,7 +63,8 @@ set -g FULLARGS $argv \ -DBUILD_REPO_INFO=$BUILD_REPO_INFO \ -DARANGODB_BUILD_DATE="$ARANGODB_BUILD_DATE" \ -DLAPACK_LIBRARIES="$LAPACK_LIB_PATH" \ - -DBLAS_LIBRARIES="$BLAS_LIB_PATH" + -DBLAS_LIBRARIES="$BLAS_LIB_PATH" \ + -DNODE_MODULES_BUNDLE="$NODE_MODULES_BUNDLE" if test "$MAINTAINER" = "On" set -g FULLARGS $FULLARGS \ From 456fbb93a93e28b56a3447b24260b1faf8062db9 Mon Sep 17 00:00:00 2001 From: Vadim Kondratev Date: Thu, 24 Apr 2025 10:18:30 +0200 Subject: [PATCH 07/10] Make NODE_MODULES_BUNDLE configurable --- helper.linux.fish | 2 ++ scripts/buildArangoDBDevel.fish | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/helper.linux.fish b/helper.linux.fish index a6d5b65f..6a7f0784 100755 --- a/helper.linux.fish +++ b/helper.linux.fish @@ -1747,6 +1747,7 @@ function runInContainer -e MAINTAINER="$MAINTAINER" \ -e MINIMAL_DEBUG_INFO="$MINIMAL_DEBUG_INFO" \ -e NODE_NAME="$NODE_NAME" \ + -e NODE_MODULES_BUNDLE="$NODE_MUDULES_BUNDLE" \ -e NOSTRIP="$NOSTRIP" \ -e NO_RM_BUILD="$NO_RM_BUILD" \ -e ONLYGREY="$ONLYGREY" \ @@ -1879,6 +1880,7 @@ function interactiveContainer -e MINIMAL_DEBUG_INFO="$MINIMAL_DEBUG_INFO" \ -e NODE_NAME="$NODE_NAME" \ -e NOSTRIP="$NOSTRIP" \ + -e NODE_MODULES_BUNDLE="$NODE_MUDULES_BUNDLE" \ -e NO_RM_BUILD="$NO_RM_BUILD" \ -e ONLYGREY="$ONLYGREY" \ -e OPENSSL_VERSION="$OPENSSL_VERSION" \ diff --git a/scripts/buildArangoDBDevel.fish b/scripts/buildArangoDBDevel.fish index 10cfacc9..b0200228 100755 --- a/scripts/buildArangoDBDevel.fish +++ b/scripts/buildArangoDBDevel.fish @@ -63,8 +63,7 @@ set -g FULLARGS $argv \ -DBUILD_REPO_INFO=$BUILD_REPO_INFO \ -DARANGODB_BUILD_DATE="$ARANGODB_BUILD_DATE" \ -DLAPACK_LIBRARIES="$LAPACK_LIB_PATH" \ - -DBLAS_LIBRARIES="$BLAS_LIB_PATH" \ - -DNODE_MODULES_BUNDLE="$NODE_MODULES_BUNDLE" + -DBLAS_LIBRARIES="$BLAS_LIB_PATH" if test "$MAINTAINER" = "On" set -g FULLARGS $FULLARGS \ @@ -123,6 +122,12 @@ if test "$MINIMAL_DEBUG_INFO" = "On" -DUSE_MINIMAL_DEBUGINFO=On end +if test -n "$NODE_MODULES_BUNDLE" -a test -e "/node_modules.tar.xz" + set -g FULLARGS $FULLARGS -DNODE_MODULES_BUNDLE="/node_modules.tar.xz" +else if test -e "$NODE_MODULES_BUNDLE" + set -g FULLARGS $FULLARGS -DNODE_MODULES_BUNDLE="$NODE_MODULES_BUNDLE" +end + setupCcacheBinPath ubuntu and setupCcache ubuntu and cleanBuildDirectory From 12fbe7a2b981fbfaaa9b0cd2cb4cd1efbc77f511 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Thu, 24 Apr 2025 13:45:41 +0200 Subject: [PATCH 08/10] fix derive --- containers/buildUbuntuDevel.docker/Dockerfile.arm64 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containers/buildUbuntuDevel.docker/Dockerfile.arm64 b/containers/buildUbuntuDevel.docker/Dockerfile.arm64 index 9d942cd2..a68529d4 100644 --- a/containers/buildUbuntuDevel.docker/Dockerfile.arm64 +++ b/containers/buildUbuntuDevel.docker/Dockerfile.arm64 @@ -109,7 +109,7 @@ RUN cd / ; git clone -n --depth=1 --filter=tree:0 https://github.com/arangodb/ar # Finally, build the actual build image based on the above preparation: -FROM prepare +FROM nodejs MAINTAINER Max Neunhoeffer From ca17e6543715a1ea4a9d996f184b5b018051c19e Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Thu, 24 Apr 2025 14:28:08 +0200 Subject: [PATCH 09/10] skip this subdir, its gone for no occurant reason --- scripts/buildArangoDBDevel.fish | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/buildArangoDBDevel.fish b/scripts/buildArangoDBDevel.fish index b0200228..892f4458 100755 --- a/scripts/buildArangoDBDevel.fish +++ b/scripts/buildArangoDBDevel.fish @@ -39,10 +39,10 @@ end if test "$ARCH" = "x86_64" set -xg LAPACK_LIB_PATH "/usr/lib/x86_64-linux-gnu/lapack/liblapack.a" - set -xg BLAS_LIB_PATH "/usr/lib/x86_64-linux-gnu/blas/libblas.a" + set -xg BLAS_LIB_PATH "/usr/lib/x86_64-linux-gnu/libblas.a" else set -xg LAPACK_LIB_PATH "/usr/lib/aarch64-linux-gnu/lapack/liblapack.a" - set -xg BLAS_LIB_PATH "/usr/lib/aarch64-linux-gnu/blas/libblas.a" + set -xg BLAS_LIB_PATH "/usr/lib/aarch64-linux-gnu/libblas.a" end set -l pie "" From 038b239ddb19a630cd246c872c0fb9e6e5e581fb Mon Sep 17 00:00:00 2001 From: Vadim Kondratev Date: Thu, 24 Apr 2025 15:32:58 +0200 Subject: [PATCH 10/10] Fix fish shell syntax --- scripts/buildArangoDBDevel.fish | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/buildArangoDBDevel.fish b/scripts/buildArangoDBDevel.fish index 892f4458..f469a54f 100755 --- a/scripts/buildArangoDBDevel.fish +++ b/scripts/buildArangoDBDevel.fish @@ -122,7 +122,7 @@ if test "$MINIMAL_DEBUG_INFO" = "On" -DUSE_MINIMAL_DEBUGINFO=On end -if test -n "$NODE_MODULES_BUNDLE" -a test -e "/node_modules.tar.xz" +if test (string trim "$NODE_MODULES_BUNDLE") = "" -a -e "/node_modules.tar.xz" set -g FULLARGS $FULLARGS -DNODE_MODULES_BUNDLE="/node_modules.tar.xz" else if test -e "$NODE_MODULES_BUNDLE" set -g FULLARGS $FULLARGS -DNODE_MODULES_BUNDLE="$NODE_MODULES_BUNDLE"