diff --git a/dev/vcpkg/CONTRIBUTING.md b/dev/vcpkg/CONTRIBUTING.md index ed78f53ca9bb..72af7ea2d4dc 100644 --- a/dev/vcpkg/CONTRIBUTING.md +++ b/dev/vcpkg/CONTRIBUTING.md @@ -13,7 +13,7 @@ Please init vcpkg env first: Vcpkg already maintains a lot of libraries. You can find them by vcpkg cli. -(NOTE: Please always use cli because [packages on vcpkg.io](https://vcpkg.io/en/packages.html) is outdate). +(NOTE: Please always use cli because [packages on vcpkg.io](https://vcpkg.io/en/packages.html) is outdated). ``` $ ./.vcpkg/vcpkg search folly @@ -65,37 +65,11 @@ See also [Versioning](https://learn.microsoft.com/en-us/vcpkg/users/versioning). Otherwise, you must create a new port in `./ports/$package` to override the vcpkg's original version. -**If a new ports has been merged in vcpkg main branch**. -You can find git tree-ish and checkout it. -For example, arrow 12.0.0 has been merged but not include in last release (2023.04.15). - -``` patch -# https://patch-diff.githubusercontent.com/raw/microsoft/vcpkg/pull/31321.patch - -diff --git a/versions/a-/arrow.json b/versions/a-/arrow.json -index 07c7ef67cb27c..f7bbe94b4f914 100644 ---- a/versions/a-/arrow.json -+++ b/versions/a-/arrow.json -@@ -1,5 +1,10 @@ - { - "versions": [ -+ { -+ "git-tree": "881bfaaab349dae46929b36e5b84e7036a009ad3", -+ "version": "12.0.0", -+ "port-version": 0 -+ }, - { - "git-tree": "21fea47a1e9c7bf68e6c088ad5a6b7b6e33c2fcb", - "version": "11.0.0", -``` - -Git tree-ish is `21fea47a1e9c7bf68e6c088ad5a6b7b6e33c2fcb`. Then fetch and checkout it. - -``` sh -cd .vcpkg -git fetch origin master -git archive 21fea47a1e9c7bf68e6c088ad5a6b7b6e33c2fcb | tar -x -C ../ports/arrow -``` +**If a newer version of a library is supported in a later version of vcpkg**. +Developers can configure `vcpkg-configuration.json` to allow importing a new port, overriding the +corresponding port specified in the `builtin-baseline`. This approach also applies to libraries +that have been removed from vcpkg at the `builtin-baseline`; by setting a historical vcpkg version, +you can import such ports. **If you want to modify port based on vcpkg version**. Copy port directory from `./.vcpkg/ports/$package` to `./ports/$package`. diff --git a/dev/vcpkg/README.md b/dev/vcpkg/README.md index 986a8ec689eb..19615e7d63d8 100644 --- a/dev/vcpkg/README.md +++ b/dev/vcpkg/README.md @@ -1,8 +1,18 @@ # Build Gluten + Velox in Vcpkg Environment -## Build in Docker +## Overview + +Currently, the `builtin-baseline` set in `vcpkg.json` is the commit hash for the `2025.09.17` tag of vcpkg. +The versions of all dependency libraries are determined by their respective ports at this vcpkg version, +except for those overridden in `vcpkg.json`, `vcpkg-configuration.json`, and overlay ports. + +## Build in docker + +For main branch code, you can follow the commands below. +- Pull the docker image: `docker pull apache/gluten:vcpkg-centos-7` +- Build native code: `bash dev/ci-velox-buildstatic-centos-7.sh` +- Build JVM code: `mvn clean install -Pbackends-velox -Pspark-3.5 -DskipTests` -Please install make and docker on your system, then `make`. The gluten packages will be placed in `$GLUTEN_REPO/package/target/gluten-velox-bundle-*.jar`. ## Setup build environment manually @@ -15,7 +25,7 @@ Please install build depends on your system to compile all libraries: sudo $GLUTEN_REPO/dev/vcpkg/setup-build-depends.sh ``` -GCC-11 is the minimum required compiler. It needs to be enabled beforehand. Take Centos-7/8 as example: +GCC 11 is the minimum required compiler. It needs to be enabled beforehand. Take Centos 7/8 as example: ``` sh # CentOS 8 @@ -32,7 +42,7 @@ For unsupported linux distro, you can install the following packages from packag * wget * curl * git >= 2.7.4 -* gcc >= 9 +* gcc >= 11 * pkg-config * autotools * flex >= 2.6.0 @@ -48,4 +58,4 @@ You can configure [binary cache](https://learn.microsoft.com/en-us/vcpkg/users/b ``` sh $GLUTEN_REPO/dev/buildbundle-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --build_benchmarks=ON --enable_s3=ON --enable_hdfs=ON -``` \ No newline at end of file +``` diff --git a/dev/vcpkg/init.sh b/dev/vcpkg/init.sh index c5e649952d92..1247a1449396 100755 --- a/dev/vcpkg/init.sh +++ b/dev/vcpkg/init.sh @@ -68,13 +68,12 @@ SCRIPT_ROOT="$(realpath "$(dirname "$0")")" cd "$SCRIPT_ROOT" if [ ! -d "$VCPKG_ROOT" ] || [ -z "$(ls "$VCPKG_ROOT")" ]; then - git clone https://github.com/microsoft/vcpkg.git --branch 2023.10.19 "$VCPKG_ROOT" + # The builtin-baseline (commit hash) specified in vcpkg.json should exist in this branch. + # Therefore, upgrading the builtin-baseline may require updating the branch. + git clone https://github.com/microsoft/vcpkg.git --branch 2025.09.17 "$VCPKG_ROOT" fi [ -f "$VCPKG" ] || "$VCPKG_ROOT/bootstrap-vcpkg.sh" -disableMetrics -sed -i "s/3.27.1/3.28.3/g" $VCPKG_ROOT/scripts/vcpkgTools.xml -sed -i "s/192374a68e2971f04974a194645726196d9b8ee7abd650d1e6f65f7aa2ccc9b186c3edb473bb4958c764532edcdd42f4182ee1fcb86b17d78b0bcd6305ce3df1/bd311ca835ef0914952f21d70d1753564d58de2ede02e80ede96e78cd2f40b4189e006007643ebb37792e13edd97eb4a33810bc8aca1eab6dd428eaffe1d2e38/g" $VCPKG_ROOT/scripts/vcpkgTools.xml - EXTRA_FEATURES="" if [ "$BUILD_TESTS" = "ON" ]; then EXTRA_FEATURES+="--x-feature=duckdb " diff --git a/dev/vcpkg/ports/aws-sdk-cpp/portfile.cmake b/dev/vcpkg/ports/aws-sdk-cpp/portfile.cmake index 71e7014868c9..59ae9ceeb981 100644 --- a/dev/vcpkg/ports/aws-sdk-cpp/portfile.cmake +++ b/dev/vcpkg/ports/aws-sdk-cpp/portfile.cmake @@ -45,7 +45,7 @@ vcpkg_cmake_configure( "-DBUILD_SHARED_LIBS=OFF" "-DAWS_SDK_WARNINGS_ARE_ERRORS=OFF" "-DCMAKE_INSTALL_RPATH=${rpath}" - "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" # use extra cmake files + "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common/modules" # use extra cmake files ) vcpkg_cmake_install() diff --git a/dev/vcpkg/vcpkg-configuration.json b/dev/vcpkg/vcpkg-configuration.json index dcbbb75b757b..0ffc49fda947 100644 --- a/dev/vcpkg/vcpkg-configuration.json +++ b/dev/vcpkg/vcpkg-configuration.json @@ -14,6 +14,12 @@ "repository": "https://github.com/Microsoft/vcpkg", "baseline": "f82160a747b1b529daad7ce2f75a0d6c6a0d285d", "packages": [ "google-cloud-cpp"] + }, + { + "kind": "git", + "repository": "https://github.com/Microsoft/vcpkg", + "baseline": "a7b6122f6b6504d16d96117336a0562693579933", + "packages": [ "upb"] } ] diff --git a/dev/vcpkg/vcpkg.json b/dev/vcpkg/vcpkg.json index cfadb20fb52e..29d3c31cfda8 100644 --- a/dev/vcpkg/vcpkg.json +++ b/dev/vcpkg/vcpkg.json @@ -1,6 +1,6 @@ { "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json", - "builtin-baseline": "a7b6122f6b6504d16d96117336a0562693579933", + "builtin-baseline": "4334d8b4c8916018600212ab4dd4bbdc343065d1", "default-features": ["velox"], "features": { "velox": { @@ -108,6 +108,13 @@ "overrides": [ { "name": "fmt", "version": "10.1.1" }, { "name": "liblzma", "version": "5.6.2" }, - { "name": "xsimd", "version": "10.0.0" } + { "name": "xsimd", "version": "10.0.0" }, + { "name": "snappy", "version": "1.1.8"}, + { "name": "protobuf", "version": "3.21.8"}, + { "name": "abseil", "version": "20230802.1"}, + { "name": "re2", "version": "2023-07-01"}, + { "name": "glog", "version": "0.6.0"}, + { "name": "gflags", "version": "2.2.2"}, + { "name": "azure-identity-cpp", "version": "1.11.0"} ] }