From d4fb60889c12ae6880b40e17d0d7d4cd52d2542a Mon Sep 17 00:00:00 2001 From: Bulat Gayazov Date: Thu, 9 Oct 2025 15:15:44 +0000 Subject: [PATCH] Added dependencies caching and updated readme --- .devcontainer/Dockerfile | 19 ++++--- .github/actions/prepare_vm/action.yaml | 57 +++++++++++++-------- .github/workflows/examples.yaml | 7 ++- .github/workflows/tests.yaml | 8 +-- .github/workflows/warmup_cache.yaml | 7 +++ CMakePresets.json | 2 +- README.md | 68 ++++++++++++++++++++------ 7 files changed, 118 insertions(+), 50 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 8c9d84d085f..6b2665269f4 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get -y update && apt-get -y install software-properties-common && add-ap RUN apt-get -y update && apt-get -y install \ git gdb ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \ rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev libgtest-dev libgmock-dev \ - libbz2-dev libdouble-conversion-dev libstdc++-13-dev gcc-13 g++-13 liblz4-dev libssl-dev \ + libbz2-dev liblz4-dev libdouble-conversion-dev libssl-dev libstdc++-13-dev gcc-13 g++-13 && apt-get clean && rm -rf /var/lib/apt/lists/* # Install CMake @@ -35,32 +35,39 @@ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 10000 && \ # Install abseil-cpp ENV ABSEIL_CPP_VERSION=20230802.0 +ENV ABSEIL_CPP_INSTALL_DIR=~/ydb_deps/abseil-cpp RUN wget -O abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSEIL_CPP_VERSION}.tar.gz && \ tar -xvzf abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz && cd abseil-cpp-${ABSEIL_CPP_VERSION} && \ mkdir build && cd build && \ cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DABSL_PROPAGATE_CXX_STD=ON .. && \ - cmake --build . --config Release --target install + cmake --build . --config Release && \ + cmake --install . --config Release --prefix ${ABSEIL_CPP_INSTALL_DIR} # Install protobuf ENV PROTOBUF_VERSION=3.21.12 +ENV PROTOBUF_INSTALL_DIR=~/ydb_deps/protobuf RUN wget -O protobuf-${PROTOBUF_VERSION}.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v${PROTOBUF_VERSION}.tar.gz && \ tar -xvzf protobuf-${PROTOBUF_VERSION}.tar.gz && cd protobuf-${PROTOBUF_VERSION} && \ mkdir build && cd build && \ cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. && \ - cmake --build . --config Release --target install + cmake --build . --config Release && \ + cmake --install . --config Release --prefix ${PROTOBUF_INSTALL_DIR} # Install grpc -ENV GRPC_VERSION=1.41.0 +ENV GRPC_VERSION=1.54.3 +ENV GRPC_INSTALL_DIR=~/ydb_deps/grpc RUN wget -O grpc-${GRPC_VERSION}.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v${GRPC_VERSION}.tar.gz && \ tar -xvzf grpc-${GRPC_VERSION}.tar.gz && cd grpc-${GRPC_VERSION} && \ mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ + cmake -G Ninja -DCMAKE_PREFIX_PATH="${ABSEIL_CPP_INSTALL_DIR};${PROTOBUF_INSTALL_DIR}" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \ -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \ -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \ -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF .. && \ - cmake --build . --config Release --target install + cmake --build . --config Release && \ + cmake --install . --config Release --prefix ${GRPC_INSTALL_DIR} # Install base64 ENV BASE64_VERSION=0.5.2 diff --git a/.github/actions/prepare_vm/action.yaml b/.github/actions/prepare_vm/action.yaml index 4ddc0b3c9de..d95ab630e67 100644 --- a/.github/actions/prepare_vm/action.yaml +++ b/.github/actions/prepare_vm/action.yaml @@ -11,9 +11,19 @@ runs: run: | sudo add-apt-repository ppa:ubuntu-toolchain-r/test sudo apt-get -y update - sudo apt-get -y install git ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \ + sudo apt-get -y install git gdb ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \ rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev libgtest-dev libgmock-dev \ - libbz2-dev liblz4-dev libdouble-conversion-dev libstdc++-13-dev gcc-13 g++-13 + libbz2-dev liblz4-dev libdouble-conversion-dev libssl-dev libstdc++-13-dev gcc-13 g++-13 + + # Install ccache + (V=4.8.1; curl -L https://github.com/ccache/ccache/releases/download/v${V}/ccache-${V}-linux-x86_64.tar.xz | \ + sudo tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${V}-linux-x86_64/ccache) + + ccache -o base_dir=`realpath ..` + ccache -o cache_dir=~/.ccache + ccache -z + export ENABLE_CCACHE="-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + wget https://apt.llvm.org/llvm.sh chmod u+x llvm.sh sudo ./llvm.sh 16 @@ -28,8 +38,9 @@ runs: tar -xvzf abseil-cpp-20230802.0.tar.gz cd abseil-cpp-20230802.0 mkdir build && cd build - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DABSL_PROPAGATE_CXX_STD=ON .. - sudo cmake --build . --config Release --target install + cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release -DABSL_PROPAGATE_CXX_STD=ON .. + cmake --build . --config Release + cmake --install . --config Release --prefix ~/ydb_deps/absl cd ../../ # Install protobuf @@ -37,50 +48,54 @@ runs: tar -xvzf protobuf-3.21.12.tar.gz cd protobuf-3.21.12 mkdir build && cd build - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. - sudo cmake --build . --config Release --target install + cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON .. + cmake --build . --config Release + cmake --install . --config Release --prefix ~/ydb_deps/protobuf cd ../../ # Install gRPC - wget -O grpc-1.41.0.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.41.0.tar.gz - tar -xvzf grpc-1.41.0.tar.gz && cd grpc-1.41.0 + wget -O grpc-1.54.3.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.54.3.tar.gz + tar -xvzf grpc-1.54.3.tar.gz && cd grpc-1.54.3 mkdir build && cd build - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ + cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_PREFIX_PATH="~/ydb_deps/absl;~/ydb_deps/protobuf" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \ -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \ -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \ -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF .. - sudo cmake --build . --config Release --target install + cmake --build . --config Release + cmake --install . --config Release --prefix ~/ydb_deps/grpc cd ../../ # Install base64 wget -O base64-0.5.2.tar.gz https://github.com/aklomp/base64/archive/refs/tags/v0.5.2.tar.gz tar -xvzf base64-0.5.2.tar.gz && cd base64-0.5.2 mkdir build && cd build - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. - sudo cmake --build . --config Release --target install + cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release .. + cmake --build . --config Release + cmake --install . --config Release --prefix ~/ydb_deps/base64 cd ../../ # Install brotli wget -O brotli-1.1.0.tar.gz https://github.com/google/brotli/archive/refs/tags/v1.1.0.tar.gz tar -xvzf brotli-1.1.0.tar.gz && cd brotli-1.1.0 mkdir build && cd build - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. - sudo cmake --build . --config Release --target install + cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release .. + cmake --build . --config Release + cmake --install . --config Release --prefix ~/ydb_deps/brotli cd ../../ # Install jwt-cpp wget -O jwt-cpp-0.7.0.tar.gz https://github.com/Thalhammer/jwt-cpp/archive/refs/tags/v0.7.0.tar.gz tar -xvzf jwt-cpp-0.7.0.tar.gz && cd jwt-cpp-0.7.0 mkdir build && cd build - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. - sudo cmake --build . --config Release --target install + cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release .. + cmake --build . --config Release + cmake --install . --config Release --prefix ~/ydb_deps/jwt-cpp cd ../../ - # Install ccache - (V=4.8.1; curl -L https://github.com/ccache/ccache/releases/download/v${V}/ccache-${V}-linux-x86_64.tar.xz | \ - sudo tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${V}-linux-x86_64/ccache) - sudo rm -rf llvm.sh abseil-cpp-20230802.0.tar.gz protobuf-3.21.12.tar.gz grpc-1.41.0.tar.gz \ + # Clean up + ccache -s + sudo rm -rf llvm.sh abseil-cpp-20230802.0.tar.gz protobuf-3.21.12.tar.gz grpc-1.54.3.tar.gz \ base64-0.5.2.tar.gz brotli-1.1.0.tar.gz jwt-cpp-0.7.0.tar.gz abseil-cpp-20230802.0 \ - protobuf-3.21.12 grpc-1.41.0 base64-0.5.2 brotli-1.1.0 jwt-cpp-0.7.0 + protobuf-3.21.12 grpc-1.54.3 base64-0.5.2 brotli-1.1.0 jwt-cpp-0.7.0 diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 5d0f38651b4..eed16ccbebb 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -46,8 +46,6 @@ jobs: if: github.event.pull_request.head.sha == '' with: submodules: true - - name: Install dependencies - uses: ./.github/actions/prepare_vm - name: Prepare ccache timestamp id: ccache_cache_timestamp shell: cmake -P {0} @@ -61,6 +59,8 @@ jobs: key: ubuntu-22.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }} restore-keys: | ubuntu-22.04-ccache-${{ matrix.compiler }}- + - name: Install dependencies + uses: ./.github/actions/prepare_vm - name: Build uses: ./.github/actions/build with: @@ -70,6 +70,9 @@ jobs: run: | cd build examples/basic_example/basic_example -e localhost:2136 -d /local + if [ ${{ matrix.ydb-version }} == "trunk" ]; then + examples/basic_example/basic_example -e localhost:2135 -d /local -c /tmp/ydb_certs/ca.pem + fi examples/bulk_upsert_simple/bulk_upsert_simple -e localhost:2136 -d /local -p /local/bulk examples/pagination/pagination -e localhost:2136 -d /local -p /local/pagination examples/ttl/ttl -e localhost:2136 -d /local -p /local/ttl diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 1e205985d24..4eb73216f68 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -35,8 +35,6 @@ jobs: if: github.event.pull_request.head.sha == '' with: submodules: true - - name: Install dependencies - uses: ./.github/actions/prepare_vm - name: Prepare ccache timestamp id: ccache_cache_timestamp shell: cmake -P {0} @@ -50,6 +48,8 @@ jobs: key: ubuntu-22.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }} restore-keys: | ubuntu-22.04-ccache-${{ matrix.compiler }}- + - name: Install dependencies + uses: ./.github/actions/prepare_vm - name: Build uses: ./.github/actions/build with: @@ -95,8 +95,6 @@ jobs: if: github.event.pull_request.head.sha == '' with: submodules: true - - name: Install dependencies - uses: ./.github/actions/prepare_vm - name: Prepare ccache timestamp id: ccache_cache_timestamp shell: cmake -P {0} @@ -110,6 +108,8 @@ jobs: key: ubuntu-22.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }} restore-keys: | ubuntu-22.04-ccache-${{ matrix.compiler }}- + - name: Install dependencies + uses: ./.github/actions/prepare_vm - name: Build uses: ./.github/actions/build with: diff --git a/.github/workflows/warmup_cache.yaml b/.github/workflows/warmup_cache.yaml index 4f8a0627866..5c2673c5143 100644 --- a/.github/workflows/warmup_cache.yaml +++ b/.github/workflows/warmup_cache.yaml @@ -17,6 +17,13 @@ jobs: uses: actions/checkout@v4 with: submodules: true + - name: Restore cache files + uses: actions/cache/restore@v4 + with: + path: ~/.ccache + key: ubuntu-22.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }} + restore-keys: | + ubuntu-22.04-ccache-${{ matrix.compiler }}- - name: Install dependencies uses: ./.github/actions/prepare_vm - name: Prepare ccache timestamp diff --git a/CMakePresets.json b/CMakePresets.json index 4432dcb354b..d33c11b6811 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -11,7 +11,7 @@ "displayName": "Generic Config", "cacheVariables": { "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", - "CMAKE_PREFIX_PATH": "~/install" + "CMAKE_PREFIX_PATH": "~/ydb_deps/absl;~/ydb_deps/protobuf;~/ydb_deps/grpc;~/ydb_deps/base64;~/ydb_deps/brotli;~/ydb_deps/jwt-cpp" }, "generator": "Ninja", "binaryDir": "${sourceDir}/build" diff --git a/README.md b/README.md index ffe4afbc21d..dea76f1e629 100644 --- a/README.md +++ b/README.md @@ -44,38 +44,74 @@ ```bash sudo apt-get -y update -sudo apt-get -y install git cmake ninja-build libidn11-dev ragel yasm protobuf-compiler \ - protobuf-compiler-grpc libprotobuf-dev libgrpc++-dev libgrpc-dev libgrpc++1 libgrpc10 \ - rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev liblz4-dev \ - libgtest-dev libgmock-dev libbz2-dev libdouble-conversion-dev libssl-dev +sudo apt-get -y install git gdb ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \ + rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev libgtest-dev libgmock-dev \ + libbz2-dev liblz4-dev libdouble-conversion-dev libssl-dev libstdc++-13-dev gcc-13 g++-13 wget https://apt.llvm.org/llvm.sh chmod u+x llvm.sh sudo ./llvm.sh 16 -wget https://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.15.tar.gz -tar -xvzf libiconv-1.15.tar.gz && cd libiconv-1.15 -./configure --prefix=/usr/local -make -sudo make install - +# Install abseil-cpp +wget -O abseil-cpp-20230802.0.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.0.tar.gz +tar -xvzf abseil-cpp-20230802.0.tar.gz +cd abseil-cpp-20230802.0 +mkdir build && cd build +cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DABSL_PROPAGATE_CXX_STD=ON .. +cmake --build . --config Release +cmake --install . --config Release --prefix ~/ydb_deps/absl +cd ../../ + +# Install protobuf +wget -O protobuf-3.21.12.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz +tar -xvzf protobuf-3.21.12.tar.gz +cd protobuf-3.21.12 +mkdir build && cd build +cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON .. +cmake --build . --config Release +cmake --install . --config Release --prefix ~/ydb_deps/protobuf +cd ../../ + +# Install gRPC +wget -O grpc-1.54.3.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.54.3.tar.gz +tar -xvzf grpc-1.54.3.tar.gz && cd grpc-1.54.3 +mkdir build && cd build +cmake -G Ninja -DCMAKE_PREFIX_PATH="~/ydb_deps/absl;~/ydb_deps/protobuf" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ + -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \ + -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \ + -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \ + -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF .. +cmake --build . --config Release +cmake --install . --config Release --prefix ~/ydb_deps/grpc +cd ../../ + +# Install base64 wget -O base64-0.5.2.tar.gz https://github.com/aklomp/base64/archive/refs/tags/v0.5.2.tar.gz tar -xvzf base64-0.5.2.tar.gz && cd base64-0.5.2 mkdir build && cd build -cmake -DCMAKE_BUILD_TYPE=Release .. -sudo cmake --build . --config Release --target install +cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. +cmake --build . --config Release +cmake --install . --config Release --prefix ~/ydb_deps/base64 +cd ../../ +# Install brotli wget -O brotli-1.1.0.tar.gz https://github.com/google/brotli/archive/refs/tags/v1.1.0.tar.gz tar -xvzf brotli-1.1.0.tar.gz && cd brotli-1.1.0 mkdir build && cd build -cmake -DCMAKE_BUILD_TYPE=Release .. -sudo cmake --build . --config Release --target install +cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. +cmake --build . --config Release +cmake --install . --config Release --prefix ~/ydb_deps/brotli +cd ../../ +# Install jwt-cpp wget -O jwt-cpp-0.7.0.tar.gz https://github.com/Thalhammer/jwt-cpp/archive/refs/tags/v0.7.0.tar.gz tar -xvzf jwt-cpp-0.7.0.tar.gz && cd jwt-cpp-0.7.0 mkdir build && cd build -cmake -DCMAKE_BUILD_TYPE=Release .. -sudo cmake --build . --config Release --target install +cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. +cmake --build . --config Release +cmake --install . --config Release --prefix ~/ydb_deps/jwt-cpp +cd ../../ ``` ### Create the work directory