From f4ab629dddcf11e156455760a8368f3e4551e622 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Mon, 2 Jan 2023 01:31:49 +0300 Subject: [PATCH 1/3] chore(cross): Upgrade aarch64-unknown-linux-gnu to Ubuntu 20.04 --- rust/cubestore/Cross.toml | 6 ++-- rust/cubestore/cross/README.md | 10 ++---- .../aarch64-unknown-linux-gnu.Dockerfile | 7 ++-- ...86_64-unknown-linux-gnu-stretch.Dockerfile | 33 ------------------- 4 files changed, 9 insertions(+), 47 deletions(-) delete mode 100644 rust/cubestore/cross/x86_64-unknown-linux-gnu-stretch.Dockerfile diff --git a/rust/cubestore/Cross.toml b/rust/cubestore/Cross.toml index 2f4e80ce3b2bd..b8eefe197fc63 100644 --- a/rust/cubestore/Cross.toml +++ b/rust/cubestore/Cross.toml @@ -11,13 +11,13 @@ image = "cubejs/rust-cross:x86_64-pc-windows-gnu-10012022" xargo = false [target.x86_64-unknown-linux-gnu] -image = "cubejs/rust-cross:x86_64-unknown-linux-gnu" +image = "cubejs/rust-cross:x86_64-unknown-linux-gnu-02012023" xargo = false [target.x86_64-unknown-linux-musl] -image = "cubejs/rust-cross:x86_64-unknown-linux-musl" +image = "cubejs/rust-cross:x86_64-unknown-linux-musl-02012023" xargo = false [target.aarch64-unknown-linux-gnu] -image = "cubejs/rust-cross:aarch64-unknown-linux-gnu" +image = "cubejs/rust-cross:aarch64-unknown-linux-gnu-02012023" xargo = false diff --git a/rust/cubestore/cross/README.md b/rust/cubestore/cross/README.md index e4c3d143a192d..e70788533641d 100644 --- a/rust/cubestore/cross/README.md +++ b/rust/cubestore/cross/README.md @@ -18,19 +18,16 @@ Keep in mind: ```sh # dmY -export CROSS_VERSION=10012022 +export CROSS_VERSION=02012023 # docker build -t cubejs/rust-cross:x86_64-apple-darwin-$CROSS_VERSION -f x86_64-apple-darwin.Dockerfile . # docker buildx build --platform linux/amd64 -t cubejs/rust-cross:x86_64-pc-windows-gnu-$CROSS_VERSION -f x86_64-pc-windows-gnu.Dockerfile . # docker buildx build --platform linux/amd64 -t cubejs/rust-cross:x86_64-pc-windows-msvc-$CROSS_VERSION -f x86_64-pc-windows-msvc.Dockerfile . -docker buildx build --platform linux/amd64 -t cubejs/rust-cross:x86_64-unknown-linux-gnu-$CROSS_VERSION -f x86_64-unknown-linux-gnu-stretch.Dockerfile . +docker buildx build --platform linux/amd64 -t cubejs/rust-cross:x86_64-unknown-linux-gnu-$CROSS_VERSION -f x86_64-unknown-linux-gnu.Dockerfile . docker buildx build --platform linux/amd64 -t cubejs/rust-cross:x86_64-unknown-linux-musl-$CROSS_VERSION -f x86_64-unknown-linux-musl.Dockerfile . docker buildx build --platform linux/amd64 -t cubejs/rust-cross:aarch64-unknown-linux-gnu-$CROSS_VERSION -f aarch64-unknown-linux-gnu.Dockerfile . -# Experimental -docker buildx build --platform linux/amd64 -t cubejs/rust-cross:x86_64-unknown-linux-gnu-buster-$CROSS_VERSION -f x86_64-unknown-linux-gnu-buster.Dockerfile . - #docker push cubejs/rust-cross:x86_64-apple-darwin #docker push cubejs/rust-cross:x86_64-pc-windows-gnu-$CROSS_VERSION #docker push cubejs/rust-cross:x86_64-pc-windows-msvc-$CROSS_VERSION @@ -38,9 +35,6 @@ docker push cubejs/rust-cross:x86_64-unknown-linux-gnu-$CROSS_VERSION docker push cubejs/rust-cross:x86_64-unknown-linux-musl-$CROSS_VERSION docker push cubejs/rust-cross:aarch64-unknown-linux-gnu-$CROSS_VERSION -# Experimental -docker push cubejs/rust-cross:x86_64-unknown-linux-gnu-buster-$CROSS_VERSION - # Verify versions docker run --rm -it cubejs/rust-cross:x86_64-unknown-linux-gnu-$CROSS_VERSION cc --version docker run --rm -it cubejs/rust-cross:x86_64-unknown-linux-gnu-buster-$CROSS_VERSION cc --version diff --git a/rust/cubestore/cross/aarch64-unknown-linux-gnu.Dockerfile b/rust/cubestore/cross/aarch64-unknown-linux-gnu.Dockerfile index 9ac231966f857..658aef5d7bbcc 100644 --- a/rust/cubestore/cross/aarch64-unknown-linux-gnu.Dockerfile +++ b/rust/cubestore/cross/aarch64-unknown-linux-gnu.Dockerfile @@ -1,12 +1,13 @@ -# Based on top of ubuntu 16.04 +# Based on top of ubuntu 20.04 +# TODO: Migrate to https://github.com/cross-rs/cross/pkgs/container/aarch64-unknown-linux-gnu, when it will be released! # https://github.com/rust-embedded/cross/blob/master/docker/Dockerfile.aarch64-unknown-linux-gnu -FROM rustembedded/cross:aarch64-unknown-linux-gnu +FROM cubejs/cross-aarch64-unknown-linux-gnu:31122022 RUN apt-get update \ && apt-get -y upgrade \ && DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common pkg-config wget apt-transport-https ca-certificates \ && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \ - && add-apt-repository "deb https://apt.llvm.org/xenial/ llvm-toolchain-xenial-12 main" \ + && add-apt-repository "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-12 main" \ && apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-multilib g++-multilib \ && DEBIAN_FRONTEND=noninteractive apt-get install -y llvm-12 clang-12 libclang-12-dev clang-12 make libssl-dev libsasl2-dev \ diff --git a/rust/cubestore/cross/x86_64-unknown-linux-gnu-stretch.Dockerfile b/rust/cubestore/cross/x86_64-unknown-linux-gnu-stretch.Dockerfile deleted file mode 100644 index 8d95d9ded61a2..0000000000000 --- a/rust/cubestore/cross/x86_64-unknown-linux-gnu-stretch.Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM debian:stretch-slim - -RUN apt-get update \ - && apt-get -y upgrade \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common pkg-config wget gnupg git apt-transport-https ca-certificates \ - && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \ - && add-apt-repository "deb https://apt.llvm.org/stretch/ llvm-toolchain-stretch-12 main" \ - && apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y llvm-12 clang-12 libclang-12-dev clang-12 make cmake \ - && rm -rf /var/lib/apt/lists/*; - -RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 100 -RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 -RUN update-alternatives --install /usr/bin/cc cc /usr/bin/clang-12 100 -RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-12 100 - -# https://www.openssl.org/source/old/1.1.1/ -ARG OPENSSL_VERSION=1.1.1l - -RUN wget https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz -O - | tar -xz &&\ - cd openssl-${OPENSSL_VERSION} && \ - ./Configure no-shared no-async --prefix=/openssl --openssldir=/openssl/ssl linux-x86_64-clang && \ - make depend && \ - make -j $(nproc) && \ - make install_sw && \ - make install_ssldirs && \ - cd .. && rm -rf ${OPENSSL_VERSION} - -ENV PKG_CONFIG_ALLOW_CROSS=1 -ENV OPENSSL_STATIC=true -ENV OPENSSL_DIR=/openssl - -ENV PATH="/cargo/bin:$PATH" From 4162d40f25aa66d30cc75bf5d1dcf03190f02b3f Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Mon, 2 Jan 2023 21:00:40 +0300 Subject: [PATCH 2/3] chore: install libz --- rust/cubestore/Cargo.lock | 6 +-- .../aarch64-unknown-linux-gnu.Dockerfile | 42 ++++++++++++------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/rust/cubestore/Cargo.lock b/rust/cubestore/Cargo.lock index c6186c340a768..23d05929a014e 100644 --- a/rust/cubestore/Cargo.lock +++ b/rust/cubestore/Cargo.lock @@ -2333,9 +2333,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.125" +version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "libloading" @@ -5246,7 +5246,7 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 1.0.0", + "cfg-if 0.1.10", "rand 0.8.4", "static_assertions", ] diff --git a/rust/cubestore/cross/aarch64-unknown-linux-gnu.Dockerfile b/rust/cubestore/cross/aarch64-unknown-linux-gnu.Dockerfile index 658aef5d7bbcc..fc64fad98c15c 100644 --- a/rust/cubestore/cross/aarch64-unknown-linux-gnu.Dockerfile +++ b/rust/cubestore/cross/aarch64-unknown-linux-gnu.Dockerfile @@ -8,11 +8,12 @@ RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common pkg-config wget apt-transport-https ca-certificates \ && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \ && add-apt-repository "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-12 main" \ - && apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-multilib g++-multilib \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y llvm-12 clang-12 libclang-12-dev clang-12 make libssl-dev libsasl2-dev \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y llvm-12 clang-12 libclang-12-dev clang-12 \ + make cmake libsasl2-dev \ libc6 libc6-dev libc6-arm64-cross libc6-dev-arm64-cross \ gcc-aarch64-linux-gnu g++-aarch64-linux-gnu \ + && apt-get autoremove -y \ && rm -rf /var/lib/apt/lists/*; RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 100 @@ -20,15 +21,28 @@ RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 RUN update-alternatives --install /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-12 100 # https://www.openssl.org/source/old/1.1.1/ -ARG OPENSSL_VERSION=1.1.1l - -ENV MACHINE=armv8 -ENV ARCH=arm -ENV CC=aarch64-linux-gnu-gcc +ENV OPENSSL_VERSION=1.1.1l +ENV LIZB_VERSION=1.2.13 + +ENV ARCH=arm \ + MACHINE=armv8 \ + AS=aarch64-linux-gnu-as \ + AR=aarch64-linux-gnu-ar \ + CC=aarch64-linux-gnu-gcc \ + CXX=aarch64-linux-gnu-g++ \ + CPP=aarch64-linux-gnu-cpp \ + LD=aarch64-linux-gnu-ld + +RUN wget https://zlib.net/zlib-${LIZB_VERSION}.tar.gz -O - | tar -xz && \ + cd zlib-${LIZB_VERSION} && \ + ./configure --prefix=/usr/aarch64-linux-gnu && \ + make -j $(nproc) && \ + make install && \ + cd .. && rm -rf zlib-${LIZB_VERSION}; RUN wget https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz -O - | tar -xz &&\ cd openssl-${OPENSSL_VERSION} && \ - ./Configure --prefix=/openssl --openssldir=/openssl/lib linux-aarch64 && \ + ./Configure --prefix=/usr/aarch64-linux-gnu --openssldir=/usr/aarch64-linux-gnu/lib linux-aarch64 && \ make depend && \ make -j $(nproc) && \ make install_sw && \ @@ -39,13 +53,11 @@ ENV PKG_CONFIG_ALLOW_CROSS=true ENV PKG_CONFIG_ALL_STATIC=true ENV RUSTFLAGS="-C target-feature=-crt-static" -ENV OPENSSL_DIR=/openssl \ +ENV OPENSSL_DIR=/usr/aarch64-linux-gnu \ OPENSSL_STATIC=yes \ - OPENSSL_INCLUDE_DIR=/openssl/include \ - OPENSSL_LIB_DIR=/openssl/lib - -ENV OPENSSL_ROOT_DIR=/openssl -ENV OPENSSL_LIBRARIES=/openssl/lib -ENV LIBZ_SYS_STATIC=1 + OPENSSL_INCLUDE_DIR=/usr/aarch64-linux-gnu/include \ + OPENSSL_LIB_DIR=/usr/aarch64-linux-gnu/lib \ + OPENSSL_LIBRARIES=/usr/aarch64-linux-gnu/lib \ + LIBZ_SYS_STATIC=1 ENV PATH="/cargo/bin:$PATH" From 1af7f75657e8a4663835d543200d5beeb0019755 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Mon, 2 Jan 2023 21:03:50 +0300 Subject: [PATCH 3/3] chore: test cross images on PR --- .github/workflows/push-cross-images.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/push-cross-images.yml b/.github/workflows/push-cross-images.yml index db13c611e46ec..22f9af996b267 100644 --- a/.github/workflows/push-cross-images.yml +++ b/.github/workflows/push-cross-images.yml @@ -1,12 +1,10 @@ -name: Build +name: Cross Images on: push: paths: - '.github/workflows/push-cross-images.yml' - 'rust/cubestore/cross/**' - branches: - - 'master' jobs: docker-dev: @@ -41,5 +39,5 @@ jobs: context: ./ file: ./rust/cubestore/cross/${{ matrix.target }}.Dockerfile platforms: linux/amd64 - push: true + push: ${{ github.ref == 'refs/heads/master' }} tags: cubejs/rust-cross:${{ matrix.target }}