diff --git a/.buildkite/pipelines/build_linux.json.py b/.buildkite/pipelines/build_linux.json.py index 3178391fb..c9443a17d 100755 --- a/.buildkite/pipelines/build_linux.json.py +++ b/.buildkite/pipelines/build_linux.json.py @@ -36,7 +36,7 @@ "cpu": "6", "ephemeralStorage": "20G", "memory": "64G", - "image": os.getenv("DOCKER_IMAGE", "docker.elastic.co/ml-dev/ml-linux-build:32") + "image": os.getenv("DOCKER_IMAGE", "docker.elastic.co/ml-dev/ml-linux-build:32a") }, "aarch64": { "provider": "aws", @@ -101,7 +101,7 @@ def main(args): "cpu": "6", "ephemeralStorage": "20G", "memory": "64G", - "image": "docker.elastic.co/ml-dev/ml-linux-aarch64-cross-build:15" + "image": "docker.elastic.co/ml-dev/ml-linux-aarch64-cross-build:15a" }, "commands": [ ".buildkite/scripts/steps/build_and_test.sh" diff --git a/3rd_party/licenses/pytorch-INFO.csv b/3rd_party/licenses/pytorch-INFO.csv index 6750bd122..04bd473a0 100644 --- a/3rd_party/licenses/pytorch-INFO.csv +++ b/3rd_party/licenses/pytorch-INFO.csv @@ -1,2 +1,2 @@ name,version,revision,url,license,copyright,sourceURL -PyTorch,2.5.1,a8d6afb511a69687bbb2b7e88a3cf67917e1697e,https://pytorch.org,BSD-3-Clause,, +PyTorch,2.7.1,e2d141dbde55c2a4370fac5165b0561b6af4798b,https://pytorch.org,BSD-3-Clause,, diff --git a/build-setup/linux.md b/build-setup/linux.md index adcc5d1a0..22c241a6e 100644 --- a/build-setup/linux.md +++ b/build-setup/linux.md @@ -329,7 +329,7 @@ Then copy the shared libraries to the system directory: (cd /opt/intel/oneapi/mkl/2024.0 && tar cf - lib) | (cd /usr/local/gcc103 && sudo tar xvf -) ``` -### PyTorch 2.5.1 +### PyTorch 2.7.1 (This step requires a reasonable amount of memory. It failed on a machine with 8GB of RAM. It succeeded on a 16GB machine. You can specify the number of parallel jobs using environment variable MAX_JOBS. Lower number of jobs will reduce memory usage.) @@ -348,7 +348,7 @@ sudo /usr/local/gcc103/bin/python3.10 -m pip install install numpy pyyaml setupt Then obtain the PyTorch code: ``` -git clone --depth=1 --branch=v2.5.1 git@github.com:pytorch/pytorch.git +git clone --depth=1 --branch=v2.7.1 git@github.com:pytorch/pytorch.git cd pytorch git submodule sync git submodule update --init --recursive @@ -383,7 +383,7 @@ export USE_MKLDNN=ON export USE_QNNPACK=OFF export USE_PYTORCH_QNNPACK=OFF [ $(uname -m) = x86_64 ] && export USE_XNNPACK=OFF -export PYTORCH_BUILD_VERSION=2.5.1 +export PYTORCH_BUILD_VERSION=2.7.1 export PYTORCH_BUILD_NUMBER=1 /usr/local/gcc103/bin/python3.10 setup.py install ``` diff --git a/build-setup/macos.md b/build-setup/macos.md index 6f6b2afae..b2441b9ae 100644 --- a/build-setup/macos.md +++ b/build-setup/macos.md @@ -127,7 +127,7 @@ Download the graphical installer for Python 3.10.9 from diff --git a/dev-tools/docker/linux_aarch64_cross_image/Dockerfile b/dev-tools/docker/linux_aarch64_cross_image/Dockerfile index 20559d1bb..4a8944f25 100644 --- a/dev-tools/docker/linux_aarch64_cross_image/Dockerfile +++ b/dev-tools/docker/linux_aarch64_cross_image/Dockerfile @@ -27,7 +27,7 @@ RUN \ RUN \ mkdir -p /usr/local/sysroot-aarch64-linux-gnu/usr && \ cd /usr/local/sysroot-aarch64-linux-gnu/usr && \ - wget --quiet -O - https://s3-eu-west-2.amazonaws.com/ml-cpp-artifacts/dependencies/usr-aarch64-linux-gnu-15.tar.bz2 | tar jxf - && \ + wget --quiet -O - https://s3-eu-west-2.amazonaws.com/ml-cpp-artifacts/dependencies/usr-aarch64-linux-gnu-15a.tar.bz2 | tar jxf - && \ cd .. && \ ln -s usr/lib lib && \ ln -s usr/lib64 lib64 diff --git a/dev-tools/docker/linux_aarch64_native_builder/Dockerfile b/dev-tools/docker/linux_aarch64_native_builder/Dockerfile index ec326a475..97a960a30 100644 --- a/dev-tools/docker/linux_aarch64_native_builder/Dockerfile +++ b/dev-tools/docker/linux_aarch64_native_builder/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:15 +FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:15a MAINTAINER David Roberts diff --git a/dev-tools/docker/linux_aarch64_native_image/Dockerfile b/dev-tools/docker/linux_aarch64_native_image/Dockerfile index 7f533f224..6dcb50696 100644 --- a/dev-tools/docker/linux_aarch64_native_image/Dockerfile +++ b/dev-tools/docker/linux_aarch64_native_image/Dockerfile @@ -35,7 +35,7 @@ ARG build_dir=/usr/src # Build gcc 10.3 RUN \ cd ${build_dir} && \ - wget --quiet -O - http://ftpmirror.gnu.org/gcc/gcc-10.3.0/gcc-10.3.0.tar.gz | tar zxf - && \ + wget --quiet -O - https://mirrorservice.org/sites/sourceware.org/pub/gcc/releases/gcc-10.3.0/gcc-10.3.0.tar.gz | tar zxf - && \ cd gcc-10.3.0 && \ contrib/download_prerequisites && \ sed -i -e 's/$(SHLIB_LDFLAGS)/-Wl,-z,relro -Wl,-z,now $(SHLIB_LDFLAGS)/' libgcc/config/t-slibgcc && \ @@ -141,7 +141,7 @@ RUN \ # If the PyTorch branch is changed also update PYTORCH_BUILD_VERSION RUN \ cd ${build_dir} && \ - git -c advice.detachedHead=false clone --depth=1 --branch=v2.5.1 https://github.com/pytorch/pytorch.git && \ + git -c advice.detachedHead=false clone --depth=1 --branch=v2.7.1 https://github.com/pytorch/pytorch.git && \ cd pytorch && \ git submodule sync && \ git submodule update --init --recursive && \ @@ -155,7 +155,7 @@ RUN \ export USE_MKLDNN=ON && \ export USE_QNNPACK=OFF && \ export USE_PYTORCH_QNNPACK=OFF && \ - export PYTORCH_BUILD_VERSION=2.5.1 && \ + export PYTORCH_BUILD_VERSION=2.7.1 && \ export PYTORCH_BUILD_NUMBER=1 && \ /usr/local/bin/python3.10 setup.py install && \ mkdir /usr/local/gcc103/include/pytorch && \ diff --git a/dev-tools/docker/linux_aarch64_native_tester/Dockerfile b/dev-tools/docker/linux_aarch64_native_tester/Dockerfile index 3cccfbfc3..eb3d55804 100644 --- a/dev-tools/docker/linux_aarch64_native_tester/Dockerfile +++ b/dev-tools/docker/linux_aarch64_native_tester/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:15 +FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:15a MAINTAINER David Roberts diff --git a/dev-tools/docker/linux_builder/Dockerfile b/dev-tools/docker/linux_builder/Dockerfile index f9f2ecdc0..d484c80b5 100644 --- a/dev-tools/docker/linux_builder/Dockerfile +++ b/dev-tools/docker/linux_builder/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-build:32 +FROM docker.elastic.co/ml-dev/ml-linux-build:32a MAINTAINER David Roberts diff --git a/dev-tools/docker/linux_dependency_builder_image/Dockerfile b/dev-tools/docker/linux_dependency_builder_image/Dockerfile index 33defeecc..9d2819c2f 100644 --- a/dev-tools/docker/linux_dependency_builder_image/Dockerfile +++ b/dev-tools/docker/linux_dependency_builder_image/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-build:32 AS builder +FROM docker.elastic.co/ml-dev/ml-linux-build:32a AS builder # This is basically automating the setup instructions in build-setup/linux.md diff --git a/dev-tools/docker/linux_image/Dockerfile b/dev-tools/docker/linux_image/Dockerfile index 65ba95877..4f876b4ad 100644 --- a/dev-tools/docker/linux_image/Dockerfile +++ b/dev-tools/docker/linux_image/Dockerfile @@ -35,7 +35,7 @@ ARG build_dir=/usr/src # Build gcc 10.3 RUN \ cd ${build_dir} && \ - wget --quiet -O - http://ftpmirror.gnu.org/gcc/gcc-10.3.0/gcc-10.3.0.tar.gz | tar zxf - && \ + wget --quiet -O - https://mirrorservice.org/sites/sourceware.org/pub/gcc/releases/gcc-10.3.0/gcc-10.3.0.tar.gz | tar zxf - && \ cd gcc-10.3.0 && \ contrib/download_prerequisites && \ sed -i -e 's/$(SHLIB_LDFLAGS)/-Wl,-z,relro -Wl,-z,now $(SHLIB_LDFLAGS)/' libgcc/config/t-slibgcc && \ @@ -154,7 +154,7 @@ gpgkey=https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS. # If the PyTorch branch is changed also update PYTORCH_BUILD_VERSION RUN \ cd ${build_dir} && \ - git -c advice.detachedHead=false clone --depth=1 --branch=v2.5.1 https://github.com/pytorch/pytorch.git && \ + git -c advice.detachedHead=false clone --depth=1 --branch=v2.7.1 https://github.com/pytorch/pytorch.git && \ cd pytorch && \ git submodule sync && \ git submodule update --init --recursive && \ @@ -169,7 +169,7 @@ RUN \ export USE_QNNPACK=OFF && \ export USE_PYTORCH_QNNPACK=OFF && \ export USE_XNNPACK=OFF && \ - export PYTORCH_BUILD_VERSION=2.5.1 && \ + export PYTORCH_BUILD_VERSION=2.7.1 && \ export PYTORCH_BUILD_NUMBER=1 && \ export MAX_JOBS=10 && \ /usr/local/bin/python3.10 setup.py install && \ diff --git a/dev-tools/docker/linux_tester/Dockerfile b/dev-tools/docker/linux_tester/Dockerfile index 8c7f6c6eb..d4e7845e6 100644 --- a/dev-tools/docker/linux_tester/Dockerfile +++ b/dev-tools/docker/linux_tester/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-build:32 +FROM docker.elastic.co/ml-dev/ml-linux-build:32a MAINTAINER David Roberts diff --git a/dev-tools/download_macos_deps.sh b/dev-tools/download_macos_deps.sh index 0b7294fda..3ac2cabfa 100755 --- a/dev-tools/download_macos_deps.sh +++ b/dev-tools/download_macos_deps.sh @@ -28,7 +28,7 @@ case `uname -m` in ARCHIVE=local-x86_64-apple-macosx12.0-1.tar.bz2 ;; arm64) - ARCHIVE=local-arm64-apple-macosx11.1-12.tar.bz2 + ARCHIVE=local-arm64-apple-macosx12.2-1.tar.bz2 ;; *) @@ -38,7 +38,7 @@ case `uname -m` in esac -URL="https://s3-eu-west-2.amazonaws.com/ml-cpp-artifacts/dependencies/$ARCHIVE" +URL="https://ml-cpp-artifacts.s3.eu-west-2.amazonaws.com/dependencies/$ARCHIVE" echo "Downloading dependencies from $URL" cd "$TMPDIR" && curl -s -S --retry 5 -O "$URL" diff --git a/dev-tools/download_windows_deps.ps1 b/dev-tools/download_windows_deps.ps1 index dbda28994..c0747a428 100755 --- a/dev-tools/download_windows_deps.ps1 +++ b/dev-tools/download_windows_deps.ps1 @@ -9,11 +9,13 @@ # limitation. # $ErrorActionPreference="Stop" +# TODO: Fix the windows build and use the latest archive +# $Archive="usr-x86_64-windows-2016-16.zip" $Archive="usr-x86_64-windows-2016-15.zip" $Destination="C:\" -# If PyTorch is not version 2.5.1 then we need the latest download +# If PyTorch is not version 2.7.1 then we need the latest download if (!(Test-Path "$Destination\usr\local\include\pytorch\torch\csrc\api\include\torch\version.h") -Or - !(Select-String -Path "$Destination\usr\local\include\pytorch\torch\csrc\api\include\torch\version.h" -Pattern "2.5.1" -Quiet)) { + !(Select-String -Path "$Destination\usr\local\include\pytorch\torch\csrc\api\include\torch\version.h" -Pattern "2.7.1" -Quiet)) { Remove-Item "$Destination\usr" -Recurse -Force -ErrorAction Ignore $ZipSource="https://storage.googleapis.com/elastic-ml-public/dependencies/$Archive" $ZipDestination="$Env:TEMP\$Archive" diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index a77ed44de..7abf9bbed 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -28,6 +28,12 @@ //=== Regressions +== {es} version 8.19.8 + +=== Enhancements + +* Update the PyTorch library to version 2.7.1. (See {ml-pull}2882[#2882].) + == {es} version 8.19.0 === Enhancements