From 924dd7d491a34bd9ac1ce80ab88f53f03cb8d465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrej=20Chud=C3=BD?= Date: Thu, 1 May 2025 14:27:52 +0200 Subject: [PATCH 1/9] Add support for python 3.12 --- .circleci/config.yml | 1 + python/python/Dockerfile.python3.12 | 105 ++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 python/python/Dockerfile.python3.12 diff --git a/.circleci/config.yml b/.circleci/config.yml index 47b101f..18d67ff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -272,6 +272,7 @@ python-versions: &python-versions - "3.9" - "3.10" - "3.11" + - "3.12" r-versions: &r-versions - "4.0.4" diff --git a/python/python/Dockerfile.python3.12 b/python/python/Dockerfile.python3.12 new file mode 100644 index 0000000..7066f3c --- /dev/null +++ b/python/python/Dockerfile.python3.12 @@ -0,0 +1,105 @@ +## WARNING: This image is plane copy of: +## https://github.com/docker-library/python/blob/7c8595e8e2b1c8bca0b6d9d146675b94c2a37ec7/3.11/bullseye/Dockerfile +## This is temporary solution and we come with proper solution in the future. +## Ticket: https://linear.app/deepnote/issue/PLA-3219/cleanup-build-pipeline-for-deepnote-python-images + + +FROM buildpack-deps:bullseye + +# ensure local python is preferred over distribution python +ENV PATH /usr/local/bin:$PATH + +# cannot remove LANG even though https://bugs.python.org/issue19846 is fixed +# last attempted removal of LANG broke many users: +# https://github.com/docker-library/python/pull/570 +ENV LANG C.UTF-8 + +# runtime dependencies +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + libbluetooth-dev \ + tk-dev \ + uuid-dev \ + ; \ + rm -rf /var/lib/apt/lists/* + +ENV GPG_KEY 7169605F62C751356D054A26A821E680E5FA6305 +ENV PYTHON_VERSION 3.12.7 + +RUN set -eux; \ + \ + wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz"; \ + wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc"; \ + GNUPGHOME="$(mktemp -d)"; export GNUPGHOME; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY"; \ + gpg --batch --verify python.tar.xz.asc python.tar.xz; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" python.tar.xz.asc; \ + mkdir -p /usr/src/python; \ + tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz; \ + rm python.tar.xz; \ + \ + cd /usr/src/python; \ + gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ + ./configure \ + --build="$gnuArch" \ + --enable-loadable-sqlite-extensions \ + --enable-optimizations \ + --enable-option-checking=fatal \ + --enable-shared \ + --with-lto \ + --with-system-expat \ + --with-ensurepip \ + ; \ + nproc="$(nproc)"; \ + EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)"; \ + LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"; \ + make -j "$nproc" \ + "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ + "LDFLAGS=${LDFLAGS:-}" \ + "PROFILE_TASK=${PROFILE_TASK:-}" \ + ; \ + # https://github.com/docker-library/python/issues/784 + # prevent accidental usage of a system installed libpython of the same version + rm python; \ + make -j "$nproc" \ + "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ + "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" \ + "PROFILE_TASK=${PROFILE_TASK:-}" \ + python \ + ; \ + make install; \ + \ + # enable GDB to load debugging data: https://github.com/docker-library/python/pull/701 + bin="$(readlink -ve /usr/local/bin/python3)"; \ + dir="$(dirname "$bin")"; \ + mkdir -p "/usr/share/gdb/auto-load/$dir"; \ + cp -vL Tools/gdb/libpython.py "/usr/share/gdb/auto-load/$bin-gdb.py"; \ + \ + cd /; \ + rm -rf /usr/src/python; \ + \ + find /usr/local -depth \ + \( \ + \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ + -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \ + \) -exec rm -rf '{}' + \ + ; \ + \ + ldconfig; \ + \ + export PYTHONDONTWRITEBYTECODE=1; \ + python3 --version; \ + pip3 --version + +# make some useful symlinks that are expected to exist ("/usr/local/bin/python" and friends) +RUN set -eux; \ + for src in idle3 pip3 pydoc3 python3 python3-config; do \ + dst="$(echo "$src" | tr -d 3)"; \ + [ -s "/usr/local/bin/$src" ]; \ + [ ! -e "/usr/local/bin/$dst" ]; \ + ln -svT "$src" "/usr/local/bin/$dst"; \ + done + +CMD ["python3"] \ No newline at end of file From aca8f790ce1604d03319f9b232fe2cac1686aa1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrej=20Chud=C3=BD?= Date: Fri, 2 May 2025 09:54:33 +0200 Subject: [PATCH 2/9] Remove this commit befor merege: Point to constrain from PR --- python/datascience/Dockerfile.datascience | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/datascience/Dockerfile.datascience b/python/datascience/Dockerfile.datascience index ac08427..98f08de 100644 --- a/python/datascience/Dockerfile.datascience +++ b/python/datascience/Dockerfile.datascience @@ -18,4 +18,4 @@ RUN if [ "$(printf '%s\n' "$PYTHON_VERSION" "3.11" | sort -V | head -n1)" = "3.1 RUN python -m venv --system-site-packages ~/venv RUN . ~/venv/bin/activate \ - && pip install --no-cache-dir -r requirements.txt -c https://tk.deepnote.com/constraints${PYTHON_VERSION}.txt \ No newline at end of file + && pip install --no-cache-dir -r requirements.txt -c https://deepnote-staging-runtime-artifactory.s3.us-east-1.amazonaws.com/deepnote-toolkit/b41caa1/constraints${PYTHON_VERSION}.txt \ No newline at end of file From 614c515ee00ee1822ca8cdbea7fde813cd22423a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrej=20Chud=C3=BD?= Date: Fri, 2 May 2025 10:23:27 +0200 Subject: [PATCH 3/9] Update torch --- python/datascience/requirements-3.11+.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/datascience/requirements-3.11+.txt b/python/datascience/requirements-3.11+.txt index 228eb20..6afdebb 100644 --- a/python/datascience/requirements-3.11+.txt +++ b/python/datascience/requirements-3.11+.txt @@ -14,7 +14,7 @@ tabulate==0.9.* sympy==1.12 squarify==0.4.* tensorflow>=2.10,<3 -torch==2.1.* -torchvision==0.16.* +torch==2.* +torchvision==0.* snowflake-snowpark-python==1.26.* geopandas==1.0.* \ No newline at end of file From 0d36e963fe0c05839f4ce376495ae107bab4f5bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrej=20Chud=C3=BD?= Date: Fri, 2 May 2025 11:05:34 +0200 Subject: [PATCH 4/9] Split python 3.11 and 3.12 requirments --- python/datascience/Dockerfile.datascience | 12 ++++++---- ...ements-3.11+.txt => requirements-3.11.txt} | 0 python/datascience/requirements-3.12.txt | 24 +++++++++++++++++++ 3 files changed, 31 insertions(+), 5 deletions(-) rename python/datascience/{requirements-3.11+.txt => requirements-3.11.txt} (100%) create mode 100644 python/datascience/requirements-3.12.txt diff --git a/python/datascience/Dockerfile.datascience b/python/datascience/Dockerfile.datascience index 98f08de..49c95c4 100644 --- a/python/datascience/Dockerfile.datascience +++ b/python/datascience/Dockerfile.datascience @@ -3,19 +3,21 @@ ARG CIRCLE_PULL_REQUEST FROM deepnote/python:${PYTHON_VERSION}${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}} # Add the requirements files -ADD requirements-3.11+.txt /requirements-3.11+.txt +ADD requirements-3.11.txt /requirements-3.11.txt +ADD requirements-3.12.txt /requirements-3.12.txt ADD requirements-below-3.11.txt /requirements-below-3.11.txt #Determine the Python version and set the version-specifications file ARG PYTHON_VERSION -RUN if [ "$(printf '%s\n' "$PYTHON_VERSION" "3.11" | sort -V | head -n1)" = "3.11" ]; then \ - mv "requirements-3.11+.txt" "requirements.txt" \ +RUN if [ "$(printf '%s\n' "$PYTHON_VERSION" "3.12" | sort -V | head -n1)" = "3.12" ]; then \ + mv /requirements-3.12.txt /requirements.txt \ + ; elif [ "$(printf '%s\n' "$PYTHON_VERSION" "3.11" | sort -V | head -n1)" = "3.11" ]; then \ + mv /requirements-3.11.txt /requirements.txt \ ; else \ - mv "requirements-below-3.11.txt" "requirements.txt" \ + mv /requirements-below-3.11.txt /requirements.txt \ ; fi - RUN python -m venv --system-site-packages ~/venv RUN . ~/venv/bin/activate \ && pip install --no-cache-dir -r requirements.txt -c https://deepnote-staging-runtime-artifactory.s3.us-east-1.amazonaws.com/deepnote-toolkit/b41caa1/constraints${PYTHON_VERSION}.txt \ No newline at end of file diff --git a/python/datascience/requirements-3.11+.txt b/python/datascience/requirements-3.11.txt similarity index 100% rename from python/datascience/requirements-3.11+.txt rename to python/datascience/requirements-3.11.txt diff --git a/python/datascience/requirements-3.12.txt b/python/datascience/requirements-3.12.txt new file mode 100644 index 0000000..1cee169 --- /dev/null +++ b/python/datascience/requirements-3.12.txt @@ -0,0 +1,24 @@ +# Requirements for Python 3.12 +# Package versions use '<=' to specify maximum compatible versions +# This prevents unexpected breaking changes while allowing installation +# of the latest compatible version within the specified constraints +scipy<=1.15.2 +matplotlib<=3.10.0 +scikit-learn<=1.6.0 +agate<=1.13.0 +keras<=3.9.0 +nltk<=3.9.0 +spacy<=3.8.0 +seaborn<=0.13.1 +scrapy<=2.11.0 +jsonify==0.5 +datascience>=0.17,<1 +textblob<=0.19.0 +tabulate<=0.9.0 +sympy<=1.14.0 +squarify<=0.4.3 +tensorflow>=2.15,<3 +torch<=2.7.0 +torchvision<=0.17.0 +snowflake-snowpark-python<=1.31.0 +geopandas<=1.0.0 \ No newline at end of file From ce3901a8a79963558574a738cf6db9fa89464b33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrej=20Chud=C3=BD?= Date: Fri, 2 May 2025 12:08:35 +0200 Subject: [PATCH 5/9] Add runtime dependnecies --- python/base/Dockerfile.base | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/base/Dockerfile.base b/python/base/Dockerfile.base index 3153f9c..22f481b 100644 --- a/python/base/Dockerfile.base +++ b/python/base/Dockerfile.base @@ -21,6 +21,12 @@ RUN apt-get update \ ssh \ gnupg2 \ ca-certificates \ + # Required for Python packages that need to connect to MS SQL Server (like pymssql) + freetds-dev \ + # Enables Python to connect to various databases through ODBC drivers + unixodbc-dev \ + # Required for secure network communications (HTTPS, secure database connections) + libssl-dev \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* ENV SHELL=/bin/bash \ From 2361beb0eadc5965183f7f73e39d975870c49dbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrej=20Chud=C3=BD?= Date: Mon, 5 May 2025 13:44:29 +0200 Subject: [PATCH 6/9] Fix: Set deepnote base for python 3.12 --- python/python/Dockerfile.python3.12 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/python/Dockerfile.python3.12 b/python/python/Dockerfile.python3.12 index 7066f3c..a13fa8a 100644 --- a/python/python/Dockerfile.python3.12 +++ b/python/python/Dockerfile.python3.12 @@ -3,8 +3,8 @@ ## This is temporary solution and we come with proper solution in the future. ## Ticket: https://linear.app/deepnote/issue/PLA-3219/cleanup-build-pipeline-for-deepnote-python-images - -FROM buildpack-deps:bullseye +ARG CIRCLE_PULL_REQUEST +FROM deepnote/python:base${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}} # ensure local python is preferred over distribution python ENV PATH /usr/local/bin:$PATH From 138138de6ab5af8f4890166217a5d47bfb4fc173 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrej=20Chud=C3=BD?= Date: Mon, 5 May 2025 17:13:43 +0200 Subject: [PATCH 7/9] Fix: Revert changein in requ 3.11 --- python/datascience/requirements-3.11.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/datascience/requirements-3.11.txt b/python/datascience/requirements-3.11.txt index 6afdebb..228eb20 100644 --- a/python/datascience/requirements-3.11.txt +++ b/python/datascience/requirements-3.11.txt @@ -14,7 +14,7 @@ tabulate==0.9.* sympy==1.12 squarify==0.4.* tensorflow>=2.10,<3 -torch==2.* -torchvision==0.* +torch==2.1.* +torchvision==0.16.* snowflake-snowpark-python==1.26.* geopandas==1.0.* \ No newline at end of file From 1247d38f0afe8c45aaaaec80fa222578b91a0c39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrej=20Chud=C3=BD?= Date: Mon, 5 May 2025 18:01:31 +0200 Subject: [PATCH 8/9] Update constraints --- python/datascience/Dockerfile.datascience | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/datascience/Dockerfile.datascience b/python/datascience/Dockerfile.datascience index 49c95c4..a3d83b7 100644 --- a/python/datascience/Dockerfile.datascience +++ b/python/datascience/Dockerfile.datascience @@ -20,4 +20,4 @@ RUN if [ "$(printf '%s\n' "$PYTHON_VERSION" "3.12" | sort -V | head -n1)" = "3.1 RUN python -m venv --system-site-packages ~/venv RUN . ~/venv/bin/activate \ - && pip install --no-cache-dir -r requirements.txt -c https://deepnote-staging-runtime-artifactory.s3.us-east-1.amazonaws.com/deepnote-toolkit/b41caa1/constraints${PYTHON_VERSION}.txt \ No newline at end of file + && pip install --no-cache-dir -r requirements.txt -c https://deepnote-staging-runtime-artifactory.s3.us-east-1.amazonaws.com/deepnote-toolkit/2f6a15c/constraints${PYTHON_VERSION}.txt \ No newline at end of file From a60ab8a127eee9692c6c2e6e2cce7e3123b6b019 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrej=20Chud=C3=BD?= Date: Tue, 6 May 2025 08:34:35 +0200 Subject: [PATCH 9/9] Revert constraints string --- python/datascience/Dockerfile.datascience | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/datascience/Dockerfile.datascience b/python/datascience/Dockerfile.datascience index a3d83b7..be2e44c 100644 --- a/python/datascience/Dockerfile.datascience +++ b/python/datascience/Dockerfile.datascience @@ -20,4 +20,4 @@ RUN if [ "$(printf '%s\n' "$PYTHON_VERSION" "3.12" | sort -V | head -n1)" = "3.1 RUN python -m venv --system-site-packages ~/venv RUN . ~/venv/bin/activate \ - && pip install --no-cache-dir -r requirements.txt -c https://deepnote-staging-runtime-artifactory.s3.us-east-1.amazonaws.com/deepnote-toolkit/2f6a15c/constraints${PYTHON_VERSION}.txt \ No newline at end of file + && pip install --no-cache-dir -r requirements.txt -c https://tk.deepnote.com/constraints${PYTHON_VERSION}.txt \ No newline at end of file