From dfd482f80bd53b38e62d9c08eac3dfd4875210f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrej=20Chud=C3=BD?= Date: Fri, 22 Nov 2024 23:46:22 +0100 Subject: [PATCH 1/4] Fix installation python --- python/base/Dockerfile.base | 36 +++++++++++++-------------- python/python/Dockerfile.python | 43 ++++++++++----------------------- 2 files changed, 31 insertions(+), 48 deletions(-) diff --git a/python/base/Dockerfile.base b/python/base/Dockerfile.base index b4c3801..c3f3e22 100644 --- a/python/base/Dockerfile.base +++ b/python/base/Dockerfile.base @@ -3,24 +3,24 @@ ENV DEBIAN_FRONTEND noninteractive # Install OS dependencies RUN apt-get update && apt-get -yq dist-upgrade \ -&& apt-get install -yq --no-install-recommends \ -build-essential \ -bzip2 \ -cmake \ -curl \ -git \ -graphviz \ -libgtk2.0-dev \ -locales \ -sudo \ -unzip \ -vim \ -wget \ -ssh \ -gnupg2 \ -ca-certificates \ -&& apt-get clean \ -&& rm -rf /var/lib/apt/lists/* + && apt-get install -yq --no-install-recommends \ + build-essential \ + bzip2 \ + cmake \ + curl \ + git \ + graphviz \ + libgtk2.0-dev \ + locales \ + sudo \ + unzip \ + vim \ + wget \ + ssh \ + gnupg2 \ + ca-certificates \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* ENV SHELL=/bin/bash \ LC_ALL=en_US.UTF-8 \ diff --git a/python/python/Dockerfile.python b/python/python/Dockerfile.python index 93e595e..5ed7552 100644 --- a/python/python/Dockerfile.python +++ b/python/python/Dockerfile.python @@ -1,50 +1,33 @@ ARG CIRCLE_PULL_REQUEST -FROM debian:bullseye-20241111-slim AS builder +FROM deepnote/python:base${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}} # Install dependencies for building Python RUN apt-get update && apt-get install -y --no-install-recommends \ - wget \ - cmake \ build-essential \ - libssl-dev \ zlib1g-dev \ libncurses5-dev \ - libncursesw5-dev \ - libreadline-dev \ - libsqlite3-dev \ libgdbm-dev \ - libdb5.3-dev \ - libbz2-dev \ - libexpat1-dev \ - liblzma-dev \ - tk-dev \ + libnss3-dev \ + libssl-dev \ + libreadline-dev \ libffi-dev \ - uuid-dev \ + libsqlite3-dev \ ca-certificates \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* # Download and extract the Python source code -WORKDIR /tmp -ARG PYTHON_VERSION_PATCH -RUN wget -qO- "https://www.python.org/ftp/python/${PYTHON_VERSION_PATCH}/Python-${PYTHON_VERSION_PATCH}.tgz" | tar xvz \ - && cd "/tmp/Python-${PYTHON_VERSION_PATCH}" \ - && ./configure --enable-optimizations --with-ensurepip=install \ +ARG PYTHON_VERSION +RUN cd "/tmp" \ + && wget -qO- "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz" | tar xvz \ + && cd "/tmp/Python-${PYTHON_VERSION}" \ + && ./configure --enable-optimizations --with-ensurepip=install \ && make -j "$(nproc)" \ && make altinstall - -FROM deepnote/python:base${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}} -# Copy Python from the builder stage -ARG PYTHON_VERSION -# Layers will be different between python versions from here onwards because of the build-arg - -COPY --from=builder "/usr/local/bin/python${PYTHON_VERSION}" "/usr/local/bin/python${PYTHON_VERSION}" -COPY --from=builder "/usr/local/bin/pip${PYTHON_VERSION}" "/usr/local/bin/pip${PYTHON_VERSION}" -COPY --from=builder "/usr/local/lib/python${PYTHON_VERSION}" "/usr/local/lib/python${PYTHON_VERSION}" - -RUN update-alternatives --install /usr/bin/python python "/usr/local/bin/python${PYTHON_VERSION}" 1 -RUN update-alternatives --install /usr/bin/pip pip "/usr/local/bin/pip${PYTHON_VERSION}" 1 +RUN PYTHON_MAJOR_MINOR_VERSION="${PYTHON_VERSION%.*}" \ + && ln -s /usr/local/bin/python${PYTHON_MAJOR_MINOR_VERSION} /usr/local/bin/python \ + && ln -s /usr/local/bin/pip${PYTHON_MAJOR_MINOR_VERSION} /usr/local/bin/pip # We create the virtual environment in the home directory in the Dockerfile # for performance improvement. From 70597e8bee05f6b25c684022a50bace4e48ba985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrej=20Chud=C3=BD?= Date: Fri, 22 Nov 2024 23:51:57 +0100 Subject: [PATCH 2/4] PYTHON_VERSION_PATCH docker argument --- python/python/Dockerfile.python | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/python/python/Dockerfile.python b/python/python/Dockerfile.python index 5ed7552..4d3e986 100644 --- a/python/python/Dockerfile.python +++ b/python/python/Dockerfile.python @@ -17,17 +17,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* # Download and extract the Python source code +ARG PYTHON_VERSION_PATCH ARG PYTHON_VERSION + RUN cd "/tmp" \ - && wget -qO- "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz" | tar xvz \ - && cd "/tmp/Python-${PYTHON_VERSION}" \ + && wget -qO- "https://www.python.org/ftp/python/${PYTHON_VERSION_PATCH}/Python-${PYTHON_VERSION_PATCH}.tgz" | tar xvz \ + && cd "/tmp/Python-${PYTHON_VERSION_PATCH}" \ && ./configure --enable-optimizations --with-ensurepip=install \ && make -j "$(nproc)" \ && make altinstall -RUN PYTHON_MAJOR_MINOR_VERSION="${PYTHON_VERSION%.*}" \ - && ln -s /usr/local/bin/python${PYTHON_MAJOR_MINOR_VERSION} /usr/local/bin/python \ - && ln -s /usr/local/bin/pip${PYTHON_MAJOR_MINOR_VERSION} /usr/local/bin/pip +RUN ln -s /usr/local/bin/python${PYTHON_VERSION} /usr/local/bin/python \ + && ln -s /usr/local/bin/pip${PYTHON_VERSION} /usr/local/bin/pip # We create the virtual environment in the home directory in the Dockerfile # for performance improvement. From 02aaaf9f86561fdb710c94dfa68f59d9914c3281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrej=20Chud=C3=BD?= Date: Sat, 23 Nov 2024 00:12:18 +0100 Subject: [PATCH 3/4] Fix symlinks --- python/python/Dockerfile.python | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/python/Dockerfile.python b/python/python/Dockerfile.python index 4d3e986..eb8e8d6 100644 --- a/python/python/Dockerfile.python +++ b/python/python/Dockerfile.python @@ -27,7 +27,8 @@ RUN cd "/tmp" \ && make -j "$(nproc)" \ && make altinstall -RUN ln -s /usr/local/bin/python${PYTHON_VERSION} /usr/local/bin/python \ +RUN ln -s /usr/local/bin/python${PYTHON_VERSION} /usr/local/bin/python3 \ + && ln -s /usr/local/bin/python3 /usr/local/bin/python \ && ln -s /usr/local/bin/pip${PYTHON_VERSION} /usr/local/bin/pip # We create the virtual environment in the home directory in the Dockerfile From 173fbfdbf9f9c23f595c01c140f3c82ee4a68154 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrej=20Chud=C3=BD?= Date: Mon, 25 Nov 2024 17:11:02 +0100 Subject: [PATCH 4/4] Copy dockerfiles from official python repo --- .circleci/config.yml | 31 +++----- python/base/Dockerfile.base | 7 +- python/python/Dockerfile.python | 36 --------- python/python/Dockerfile.python3.10 | 117 ++++++++++++++++++++++++++++ python/python/Dockerfile.python3.11 | 116 +++++++++++++++++++++++++++ python/python/Dockerfile.python3.8 | 117 ++++++++++++++++++++++++++++ python/python/Dockerfile.python3.9 | 116 +++++++++++++++++++++++++++ 7 files changed, 480 insertions(+), 60 deletions(-) delete mode 100644 python/python/Dockerfile.python create mode 100644 python/python/Dockerfile.python3.10 create mode 100644 python/python/Dockerfile.python3.11 create mode 100644 python/python/Dockerfile.python3.8 create mode 100644 python/python/Dockerfile.python3.9 diff --git a/.circleci/config.yml b/.circleci/config.yml index f6f72bc..b72bd98 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,13 +5,6 @@ orbs: docker: circleci/docker@2.8 commands: - split_python_version: - steps: - - run: - name: Split python version - command: | - echo "export PYTHON_VERSION=$(echo $PYTHON_VERSION_PATCH | cut -d'.' -f1,2)" >> $BASH_ENV - source $BASH_ENV setup_buildkit_builder: steps: - run: @@ -69,11 +62,10 @@ jobs: python-version: type: string environment: - PYTHON_VERSION_PATCH: << parameters.python-version >> + PYTHON_VERSION: << parameters.python-version >> steps: - checkout - docker_login - - split_python_version # This provides PYTHON_VERSION - setup_buildkit_builder - run: name: Build and push python image @@ -81,13 +73,12 @@ jobs: REPOSITORY="deepnote/python" TAG="${PYTHON_VERSION}" TAG_SUFFIX="${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}" + docker buildx build \ - --file=./python/python/Dockerfile.python \ + --file=./python/python/Dockerfile.python${PYTHON_VERSION} \ --tag=978928340082.dkr.ecr.us-east-1.amazonaws.com/${REPOSITORY}:${TAG}${TAG_SUFFIX} \ --tag=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX} \ --build-arg CIRCLE_PULL_REQUEST=${CIRCLE_PULL_REQUEST} \ - --build-arg PYTHON_VERSION=${PYTHON_VERSION} \ - --build-arg PYTHON_VERSION_PATCH=${PYTHON_VERSION_PATCH} \ --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache \ --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX}-buildcache \ --cache-to type=registry,ref=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX}-buildcache,mode=max,image-manifest=true,oci-mediatypes=true \ @@ -105,11 +96,10 @@ jobs: python-version: type: string environment: - PYTHON_VERSION_PATCH: << parameters.python-version >> + PYTHON_VERSION: << parameters.python-version >> steps: - checkout - docker_login - - split_python_version # This provides PYTHON_VERSION - setup_buildkit_builder - run: name: Build and push python-datascience image @@ -140,11 +130,10 @@ jobs: python-version: type: string environment: - PYTHON_VERSION_PATCH: << parameters.python-version >> + PYTHON_VERSION: << parameters.python-version >> steps: - checkout - docker_login - - split_python_version # This provides PYTHON_VERSION - setup_buildkit_builder - run: name: Build and push python-conda image @@ -275,10 +264,10 @@ jobs: ./gpu python-versions: &python-versions - - "3.8.19" - - "3.9.19" - - "3.10.15" - - "3.11.10" + - "3.8" + - "3.9" + - "3.10" + - "3.11" r-versions: &r-versions - "4.0.4" @@ -316,7 +305,7 @@ workflows: parameters: r-version: *r-versions requires: - - Python 3.9.19 + - Python 3.9 - build-and-push-ir-with-libs: name: R with libs << matrix.r-version >> matrix: diff --git a/python/base/Dockerfile.base b/python/base/Dockerfile.base index c3f3e22..3153f9c 100644 --- a/python/base/Dockerfile.base +++ b/python/base/Dockerfile.base @@ -1,8 +1,10 @@ -FROM debian:bullseye-20241111-slim +## Source of the image with list included dependencies! +# https://github.com/docker-library/buildpack-deps/blob/master/debian/bullseye/Dockerfile +FROM buildpack-deps:bullseye ENV DEBIAN_FRONTEND noninteractive # Install OS dependencies -RUN apt-get update && apt-get -yq dist-upgrade \ +RUN apt-get update \ && apt-get install -yq --no-install-recommends \ build-essential \ bzip2 \ @@ -21,7 +23,6 @@ RUN apt-get update && apt-get -yq dist-upgrade \ ca-certificates \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* - ENV SHELL=/bin/bash \ LC_ALL=en_US.UTF-8 \ LANG=en_US.UTF-8 \ diff --git a/python/python/Dockerfile.python b/python/python/Dockerfile.python deleted file mode 100644 index eb8e8d6..0000000 --- a/python/python/Dockerfile.python +++ /dev/null @@ -1,36 +0,0 @@ -ARG CIRCLE_PULL_REQUEST -FROM deepnote/python:base${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}} - -# Install dependencies for building Python -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - zlib1g-dev \ - libncurses5-dev \ - libgdbm-dev \ - libnss3-dev \ - libssl-dev \ - libreadline-dev \ - libffi-dev \ - libsqlite3-dev \ - ca-certificates \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Download and extract the Python source code -ARG PYTHON_VERSION_PATCH -ARG PYTHON_VERSION - -RUN cd "/tmp" \ - && wget -qO- "https://www.python.org/ftp/python/${PYTHON_VERSION_PATCH}/Python-${PYTHON_VERSION_PATCH}.tgz" | tar xvz \ - && cd "/tmp/Python-${PYTHON_VERSION_PATCH}" \ - && ./configure --enable-optimizations --with-ensurepip=install \ - && make -j "$(nproc)" \ - && make altinstall - -RUN ln -s /usr/local/bin/python${PYTHON_VERSION} /usr/local/bin/python3 \ - && ln -s /usr/local/bin/python3 /usr/local/bin/python \ - && ln -s /usr/local/bin/pip${PYTHON_VERSION} /usr/local/bin/pip - -# We create the virtual environment in the home directory in the Dockerfile -# for performance improvement. -RUN python -m venv --system-site-packages ~/venv diff --git a/python/python/Dockerfile.python3.10 b/python/python/Dockerfile.python3.10 new file mode 100644 index 0000000..2ef64d3 --- /dev/null +++ b/python/python/Dockerfile.python3.10 @@ -0,0 +1,117 @@ +## WARNING: This image is plane copy of: +## https://github.com/docker-library/python/blob/7c8595e8e2b1c8bca0b6d9d146675b94c2a37ec7/3.10/bullseye/Dockerfile +## This is temporary solution and we come with proper solution in the future. +## Ticket: https://linear.app/deepnote/issue/PLA-3219/cleanup-build-pipeline-for-deepnote-python-images + +ARG CIRCLE_PULL_REQUEST +FROM deepnote/python:base${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}} + + +# ensure local python is preferred over distribution python +ENV PATH /usr/local/bin:$PATH + +# cannot remove LANG even though https://bugs.python.org/issue19846 is fixed +# last attempted removal of LANG broke many users: +# https://github.com/docker-library/python/pull/570 +ENV LANG C.UTF-8 + +# runtime dependencies +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + libbluetooth-dev \ + tk-dev \ + uuid-dev \ + ; \ + rm -rf /var/lib/apt/lists/* + +ENV GPG_KEY A035C8C19219BA821ECEA86B64E628F8D684696D +ENV PYTHON_VERSION 3.10.15 + +RUN set -eux; \ + \ + wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz"; \ + wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc"; \ + GNUPGHOME="$(mktemp -d)"; export GNUPGHOME; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY"; \ + gpg --batch --verify python.tar.xz.asc python.tar.xz; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" python.tar.xz.asc; \ + mkdir -p /usr/src/python; \ + tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz; \ + rm python.tar.xz; \ + \ + cd /usr/src/python; \ + gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ + ./configure \ + --build="$gnuArch" \ + --enable-loadable-sqlite-extensions \ + --enable-optimizations \ + --enable-option-checking=fatal \ + --enable-shared \ + --with-lto \ + --with-system-expat \ + --with-ensurepip \ + ; \ + nproc="$(nproc)"; \ + EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)"; \ + LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"; \ + make -j "$nproc" \ + "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ + "LDFLAGS=${LDFLAGS:-}" \ + "PROFILE_TASK=${PROFILE_TASK:-}" \ + ; \ +# https://github.com/docker-library/python/issues/784 +# prevent accidental usage of a system installed libpython of the same version + rm python; \ + make -j "$nproc" \ + "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ + "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" \ + "PROFILE_TASK=${PROFILE_TASK:-}" \ + python \ + ; \ + make install; \ + \ +# enable GDB to load debugging data: https://github.com/docker-library/python/pull/701 + bin="$(readlink -ve /usr/local/bin/python3)"; \ + dir="$(dirname "$bin")"; \ + mkdir -p "/usr/share/gdb/auto-load/$dir"; \ + cp -vL Tools/gdb/libpython.py "/usr/share/gdb/auto-load/$bin-gdb.py"; \ + \ + cd /; \ + rm -rf /usr/src/python; \ + \ + find /usr/local -depth \ + \( \ + \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ + -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \ + \) -exec rm -rf '{}' + \ + ; \ + \ + ldconfig; \ + \ + export PYTHONDONTWRITEBYTECODE=1; \ + python3 --version; \ + \ + pip3 install \ + --disable-pip-version-check \ + --no-cache-dir \ + --no-compile \ + 'setuptools==65.5.1' \ + wheel \ + ; \ + pip3 --version + +# make some useful symlinks that are expected to exist ("/usr/local/bin/python" and friends) +RUN set -eux; \ + for src in idle3 pip3 pydoc3 python3 python3-config; do \ + dst="$(echo "$src" | tr -d 3)"; \ + [ -s "/usr/local/bin/$src" ]; \ + [ ! -e "/usr/local/bin/$dst" ]; \ + ln -svT "$src" "/usr/local/bin/$dst"; \ + done + + +# We create the virtual environment in the home directory in the Dockerfile +# for performance improvement. +RUN python -m venv --system-site-packages ~/venv diff --git a/python/python/Dockerfile.python3.11 b/python/python/Dockerfile.python3.11 new file mode 100644 index 0000000..50d9a85 --- /dev/null +++ b/python/python/Dockerfile.python3.11 @@ -0,0 +1,116 @@ +## WARNING: This image is plane copy of: +## https://github.com/docker-library/python/blob/7c8595e8e2b1c8bca0b6d9d146675b94c2a37ec7/3.11/bullseye/Dockerfile +## This is temporary solution and we come with proper solution in the future. +## Ticket: https://linear.app/deepnote/issue/PLA-3219/cleanup-build-pipeline-for-deepnote-python-images + +ARG CIRCLE_PULL_REQUEST +FROM deepnote/python:base${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}} + +# ensure local python is preferred over distribution python +ENV PATH /usr/local/bin:$PATH + +# cannot remove LANG even though https://bugs.python.org/issue19846 is fixed +# last attempted removal of LANG broke many users: +# https://github.com/docker-library/python/pull/570 +ENV LANG C.UTF-8 + +# runtime dependencies +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + libbluetooth-dev \ + tk-dev \ + uuid-dev \ + ; \ + rm -rf /var/lib/apt/lists/* + +ENV GPG_KEY A035C8C19219BA821ECEA86B64E628F8D684696D +ENV PYTHON_VERSION 3.11.10 + +RUN set -eux; \ + \ + wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz"; \ + wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc"; \ + GNUPGHOME="$(mktemp -d)"; export GNUPGHOME; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY"; \ + gpg --batch --verify python.tar.xz.asc python.tar.xz; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" python.tar.xz.asc; \ + mkdir -p /usr/src/python; \ + tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz; \ + rm python.tar.xz; \ + \ + cd /usr/src/python; \ + gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ + ./configure \ + --build="$gnuArch" \ + --enable-loadable-sqlite-extensions \ + --enable-optimizations \ + --enable-option-checking=fatal \ + --enable-shared \ + --with-lto \ + --with-system-expat \ + --with-ensurepip \ + ; \ + nproc="$(nproc)"; \ + EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)"; \ + LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"; \ + make -j "$nproc" \ + "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ + "LDFLAGS=${LDFLAGS:-}" \ + "PROFILE_TASK=${PROFILE_TASK:-}" \ + ; \ +# https://github.com/docker-library/python/issues/784 +# prevent accidental usage of a system installed libpython of the same version + rm python; \ + make -j "$nproc" \ + "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ + "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" \ + "PROFILE_TASK=${PROFILE_TASK:-}" \ + python \ + ; \ + make install; \ + \ +# enable GDB to load debugging data: https://github.com/docker-library/python/pull/701 + bin="$(readlink -ve /usr/local/bin/python3)"; \ + dir="$(dirname "$bin")"; \ + mkdir -p "/usr/share/gdb/auto-load/$dir"; \ + cp -vL Tools/gdb/libpython.py "/usr/share/gdb/auto-load/$bin-gdb.py"; \ + \ + cd /; \ + rm -rf /usr/src/python; \ + \ + find /usr/local -depth \ + \( \ + \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ + -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \ + \) -exec rm -rf '{}' + \ + ; \ + \ + ldconfig; \ + \ + export PYTHONDONTWRITEBYTECODE=1; \ + python3 --version; \ + \ + pip3 install \ + --disable-pip-version-check \ + --no-cache-dir \ + --no-compile \ + 'setuptools==65.5.1' \ + wheel \ + ; \ + pip3 --version + +# make some useful symlinks that are expected to exist ("/usr/local/bin/python" and friends) +RUN set -eux; \ + for src in idle3 pip3 pydoc3 python3 python3-config; do \ + dst="$(echo "$src" | tr -d 3)"; \ + [ -s "/usr/local/bin/$src" ]; \ + [ ! -e "/usr/local/bin/$dst" ]; \ + ln -svT "$src" "/usr/local/bin/$dst"; \ + done + + +# We create the virtual environment in the home directory in the Dockerfile +# for performance improvement. +RUN python -m venv --system-site-packages ~/venv diff --git a/python/python/Dockerfile.python3.8 b/python/python/Dockerfile.python3.8 new file mode 100644 index 0000000..5f627e8 --- /dev/null +++ b/python/python/Dockerfile.python3.8 @@ -0,0 +1,117 @@ +## WARNING: This image is plane copy of: +## https://github.com/docker-library/python/blob/7c8595e8e2b1c8bca0b6d9d146675b94c2a37ec7/3.8/bullseye/Dockerfile +## This is temporary solution and we come with proper solution in the future. +## Ticket: https://linear.app/deepnote/issue/PLA-3219/cleanup-build-pipeline-for-deepnote-python-images + +ARG CIRCLE_PULL_REQUEST +FROM deepnote/python:base${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}} + +# ensure local python is preferred over distribution python +ENV PATH /usr/local/bin:$PATH + +# cannot remove LANG even though https://bugs.python.org/issue19846 is fixed +# last attempted removal of LANG broke many users: +# https://github.com/docker-library/python/pull/570 +ENV LANG C.UTF-8 + +# runtime dependencies +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + libbluetooth-dev \ + tk-dev \ + uuid-dev \ + ; \ + rm -rf /var/lib/apt/lists/* + +ENV GPG_KEY E3FF2839C048B25C084DEBE9B26995E310250568 +ENV PYTHON_VERSION 3.8.20 + +RUN set -eux; \ + \ + wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz"; \ + wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc"; \ + GNUPGHOME="$(mktemp -d)"; export GNUPGHOME; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY"; \ + gpg --batch --verify python.tar.xz.asc python.tar.xz; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" python.tar.xz.asc; \ + mkdir -p /usr/src/python; \ + tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz; \ + rm python.tar.xz; \ + \ + cd /usr/src/python; \ + gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ + ./configure \ + --build="$gnuArch" \ + --enable-loadable-sqlite-extensions \ + --enable-optimizations \ + --enable-option-checking=fatal \ + --enable-shared \ + --with-system-expat \ + --with-ensurepip \ + ; \ + nproc="$(nproc)"; \ + EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)"; \ + LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"; \ + make -j "$nproc" \ + "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ + "LDFLAGS=${LDFLAGS:-}" \ + "PROFILE_TASK=${PROFILE_TASK:-}" \ + ; \ +# https://github.com/docker-library/python/issues/784 +# prevent accidental usage of a system installed libpython of the same version + rm python; \ + make -j "$nproc" \ + "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ + "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" \ + "PROFILE_TASK=${PROFILE_TASK:-}" \ + python \ + ; \ + make install; \ + \ +# enable GDB to load debugging data: https://github.com/docker-library/python/pull/701 + bin="$(readlink -ve /usr/local/bin/python3)"; \ + dir="$(dirname "$bin")"; \ + mkdir -p "/usr/share/gdb/auto-load/$dir"; \ + cp -vL Tools/gdb/libpython.py "/usr/share/gdb/auto-load/$bin-gdb.py"; \ + \ + cd /; \ + rm -rf /usr/src/python; \ + \ + find /usr/local -depth \ + \( \ + \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ + -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \ + -o \( -type f -a -name 'wininst-*.exe' \) \ + \) -exec rm -rf '{}' + \ + ; \ + \ + ldconfig; \ + \ + export PYTHONDONTWRITEBYTECODE=1; \ + python3 --version; \ + \ + pip3 install \ + --disable-pip-version-check \ + --no-cache-dir \ + --no-compile \ + 'setuptools==57.5.0' \ + wheel \ + ; \ + pip3 --version + +# make some useful symlinks that are expected to exist ("/usr/local/bin/python" and friends) +RUN set -eux; \ + for src in idle3 pip3 pydoc3 python3 python3-config; do \ + dst="$(echo "$src" | tr -d 3)"; \ + [ -s "/usr/local/bin/$src" ]; \ + [ ! -e "/usr/local/bin/$dst" ]; \ + ln -svT "$src" "/usr/local/bin/$dst"; \ + done + + + +# We create the virtual environment in the home directory in the Dockerfile +# for performance improvement. +RUN python -m venv --system-site-packages ~/venv diff --git a/python/python/Dockerfile.python3.9 b/python/python/Dockerfile.python3.9 new file mode 100644 index 0000000..b69c43a --- /dev/null +++ b/python/python/Dockerfile.python3.9 @@ -0,0 +1,116 @@ +## WARNING: This image is plane copy of: +## https://github.com/docker-library/python/blob/7c8595e8e2b1c8bca0b6d9d146675b94c2a37ec7/3.9/bullseye/Dockerfile +## This is temporary solution and we come with proper solution in the future. +## Ticket: https://linear.app/deepnote/issue/PLA-3219/cleanup-build-pipeline-for-deepnote-python-images + + + +ARG CIRCLE_PULL_REQUEST +FROM deepnote/python:base${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}} +# ensure local python is preferred over distribution python +ENV PATH /usr/local/bin:$PATH + +# cannot remove LANG even though https://bugs.python.org/issue19846 is fixed +# last attempted removal of LANG broke many users: +# https://github.com/docker-library/python/pull/570 +ENV LANG C.UTF-8 + +# runtime dependencies +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + libbluetooth-dev \ + tk-dev \ + uuid-dev \ + ; \ + rm -rf /var/lib/apt/lists/* + +ENV GPG_KEY E3FF2839C048B25C084DEBE9B26995E310250568 +ENV PYTHON_VERSION 3.9.20 + +RUN set -eux; \ + \ + wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz"; \ + wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc"; \ + GNUPGHOME="$(mktemp -d)"; export GNUPGHOME; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY"; \ + gpg --batch --verify python.tar.xz.asc python.tar.xz; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" python.tar.xz.asc; \ + mkdir -p /usr/src/python; \ + tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz; \ + rm python.tar.xz; \ + \ + cd /usr/src/python; \ + gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ + ./configure \ + --build="$gnuArch" \ + --enable-loadable-sqlite-extensions \ + --enable-optimizations \ + --enable-option-checking=fatal \ + --enable-shared \ + --with-system-expat \ + --with-ensurepip \ + ; \ + nproc="$(nproc)"; \ + EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)"; \ + LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"; \ + make -j "$nproc" \ + "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ + "LDFLAGS=${LDFLAGS:-}" \ + "PROFILE_TASK=${PROFILE_TASK:-}" \ + ; \ +# https://github.com/docker-library/python/issues/784 +# prevent accidental usage of a system installed libpython of the same version + rm python; \ + make -j "$nproc" \ + "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ + "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" \ + "PROFILE_TASK=${PROFILE_TASK:-}" \ + python \ + ; \ + make install; \ + \ +# enable GDB to load debugging data: https://github.com/docker-library/python/pull/701 + bin="$(readlink -ve /usr/local/bin/python3)"; \ + dir="$(dirname "$bin")"; \ + mkdir -p "/usr/share/gdb/auto-load/$dir"; \ + cp -vL Tools/gdb/libpython.py "/usr/share/gdb/auto-load/$bin-gdb.py"; \ + \ + cd /; \ + rm -rf /usr/src/python; \ + \ + find /usr/local -depth \ + \( \ + \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ + -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \ + \) -exec rm -rf '{}' + \ + ; \ + \ + ldconfig; \ + \ + export PYTHONDONTWRITEBYTECODE=1; \ + python3 --version; \ + \ + pip3 install \ + --disable-pip-version-check \ + --no-cache-dir \ + --no-compile \ + 'setuptools==58.1.0' \ + wheel \ + ; \ + pip3 --version + +# make some useful symlinks that are expected to exist ("/usr/local/bin/python" and friends) +RUN set -eux; \ + for src in idle3 pip3 pydoc3 python3 python3-config; do \ + dst="$(echo "$src" | tr -d 3)"; \ + [ -s "/usr/local/bin/$src" ]; \ + [ ! -e "/usr/local/bin/$dst" ]; \ + ln -svT "$src" "/usr/local/bin/$dst"; \ + done + + +# We create the virtual environment in the home directory in the Dockerfile +# for performance improvement. +RUN python -m venv --system-site-packages ~/venv