1- ARG BASE_IMAGE_REPO \
2- BASE_IMAGE_TAG \
3- CPU_BASE_IMAGE_NAME \
4- GPU_BASE_IMAGE_NAME \
5- LIGHTGBM_VERSION \
6- TORCH_VERSION \
7- TORCHAUDIO_VERSION \
8- TORCHVISION_VERSION \
9- JAX_VERSION
10-
11- {{ if eq .Accelerator " gpu" }}
12- FROM gcr.io /kaggle-images/python-lightgbm-whl:$ {GPU_BASE_IMAGE_NAME}-$ {BASE_IMAGE_TAG}-$ {LIGHTGBM_VERSION} AS lightgbm_whl
13- FROM gcr.io /kaggle-images/python-torch-whl:$ {GPU_BASE_IMAGE_NAME}-$ {BASE_IMAGE_TAG}-$ {TORCH_VERSION} AS torch_whl
14- FROM gcr.io /kaggle-images/python-jaxlib-whl:$ {GPU_BASE_IMAGE_NAME}-$ {BASE_IMAGE_TAG}-$ {JAX_VERSION} AS jaxlib_whl
15- FROM $ {BASE_IMAGE_REPO}/$ {GPU_BASE_IMAGE_NAME}:$ {BASE_IMAGE_TAG}
16- {{ else }}
17- FROM $ {BASE_IMAGE_REPO}/$ {CPU_BASE_IMAGE_NAME}:$ {BASE_IMAGE_TAG}
18- {{ end }}
19-
20- # Ensures shared libraries installed with conda can be found by the dynamic link loader.
21- ENV LIBRARY_PATH= " $LIBRARY_PATH:/opt/conda/lib" \
22- LD_LIBRARY_PATH= " $LD_LIBRARY_PATH:/opt/conda/lib"
23-
24- {{ if eq .Accelerator " gpu" }}
25- ARG CUDA_MAJOR_VERSION \
26- CUDA_MINOR_VERSION
27- ENV CUDA_MAJOR_VERSION= $ {CUDA_MAJOR_VERSION} \
28- CUDA_MINOR_VERSION= $ {CUDA_MINOR_VERSION}
29- # Make sure we are on the right version of CUDA
30- RUN update-alternatives --set cuda /usr/local/cuda-$CUDA_MAJOR_VERSION . $CUDA_MINOR_VERSION
31- # NVIDIA binaries from the host are mounted to /opt/bin.
32- ENV PATH= /opt/bin:$ {PATH} \
33- # Add CUDA stubs to LD_LIBRARY_PATH to support building the GPU image on a CPU machine.
34- LD_LIBRARY_PATH_NO_STUBS= " $LD_LIBRARY_PATH" \
35- LD_LIBRARY_PATH= " $LD_LIBRARY_PATH:/usr/local/cuda/lib64/stubs"
36- RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
37- {{ end }}
38-
39- # Keep these variables in sync if base image is updated.
40- ENV TENSORFLOW_VERSION= 2.16.1 \
41- # See https://github.com /tensorflow/io#tensorflow-version-compatibility
42- TENSORFLOW_IO_VERSION= 0.37.0
43-
44- # We need to redefine the ARG here to get the ARG value defined above the FROM instruction.
45- # See: https://docs.docker.com /engine/reference/builder/#understand-how-arg-and -from-interact
46- ARG LIGHTGBM_VERSION \
47- TORCH_VERSION \
48- TORCHAUDIO_VERSION \
49- TORCHVISION_VERSION \
50- JAX_VERSION
1+ FROM us-docker.pkg.dev /colab-images/public/runtime
512
523# Disable pesky logs like: KMP_AFFINITY: pid 6121 tid 6121 thread 0 bound to OS proc set 0
534# See: https://stackoverflow.com /questions/57385766/disable-tensorflow-log-information
@@ -78,99 +29,28 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list &
7829 apt-get install -y graphviz && pip install graphviz && \
7930 /tmp/clean-layer.sh
8031
81- # b/128333086: Set PROJ_DATA to points to the proj4 cartographic library.
82- ENV PROJ_DATA= /opt/conda/share/proj
83-
84- # Install micromamba, setup channels, and replace conda with micromamba
85- ENV MAMBA_ROOT_PREFIX= /opt/conda
86- RUN curl -L " https://micro.mamba.pm/install.sh" -o /tmp/micromamba-install.sh \
87- && bash /tmp/micromamba-install.sh \
88- && rm /tmp/micromamba-install.sh \
89- && mv ~/.local /bin/micromamba /usr/bin/micromamba \
90- && (!(which conda) || cp /usr/bin/micromamba $ (which conda)) \
91- && micromamba config append channels nvidia \
92- && micromamba config append channels rapidsai \
93- && micromamba config append channels conda-forge \
94- && micromamba config set channel_priority flexible \
95- && python -m nb_conda_kernels.install --disable
32+ # # b/128333086: Set PROJ_DATA to points to the proj4 cartographic library.
33+ # ENV PROJ_DATA= /opt/conda/share/proj
34+
35+ # # Install micromamba, setup channels, and replace conda with micromamba
36+ # ENV MAMBA_ROOT_PREFIX= /opt/conda
37+ # RUN curl -L " https://micro.mamba.pm/install.sh" -o /tmp/micromamba-install.sh \
38+ # && bash /tmp/micromamba-install.sh \
39+ # && rm /tmp/micromamba-install.sh \
40+ # && mv ~/.local /bin/micromamba /usr/bin/micromamba \
41+ # && (!(which conda) || cp /usr/bin/micromamba $ (which conda)) \
42+ # && micromamba config append channels nvidia \
43+ # && micromamba config append channels rapidsai \
44+ # && micromamba config append channels conda-forge \
45+ # && micromamba config set channel_priority flexible \
46+ # && python -m nb_conda_kernels.install --disable
9647
9748# Install conda packages not available on pip.
9849# When using pip in a conda environment, conda commands should be ran first and then
9950# the remaining pip commands: https://www.anaconda.com /using-pip-in-a-conda-environment/
100- RUN micromamba install -y mkl cartopy imagemagick pyproj " shapely<2" && \
101- rm -rf /opt/conda/lib/python3.10 /site-packages/pyproj/proj_dir/ && \
102- /tmp/clean-layer.sh
103-
104- # Install spacy
105- # b/232247930: uninstall pyarrow to avoid double installation with the GPU specific version.
106- # b/341938540: unistall grpc-cpp to allow >= v24.4 cudf and cuml to be installed.
107- {{ if eq .Accelerator " gpu" }}
108- RUN pip uninstall -y pyarrow && \
109- micromamba install -vvvy spacy " cudf>=24.4" " cuml>=24.4" cupy cuda-version= $CUDA_MAJOR_VERSION . $CUDA_MINOR_VERSION && \
110- /tmp/clean-layer.sh
111- {{ else }}
112- RUN pip install spacy && \
113- /tmp/clean-layer.sh
114- {{ end }}
115-
116- # Install PyTorch
117- # b/356397043: magma-cuda121 is the latest version
118- {{ if eq .Accelerator " gpu" }}
119- COPY --from= torch_whl /tmp/whl/* .whl /tmp/torch/
120- # b/356397043: We are currently using cuda 12.3,
121- # but magma-cuda121 is the latest compatible version
122- RUN micromamba install -y -c pytorch magma-cuda121 && \
123- pip install /tmp/torch/*.whl && \
124- sudo apt -y install libsox-dev && \
125- rm -rf /tmp/torch && \
126- /tmp/clean-layer.sh
127- {{ else }}
128- RUN pip install \
129- torch==$TORCH_VERSION+cpu \
130- torchvision==$TORCHVISION_VERSION+cpu \
131- torchaudio==$TORCHAUDIO_VERSION+cpu \
132- --index-url https://download.pytorch.org/whl/cpu && \
133- /tmp/clean-layer.sh
134- {{ end }}
135-
136- # Install LightGBM
137- {{ if eq .Accelerator "gpu" }}
138- COPY --from=lightgbm_whl /tmp/whl/*.whl /tmp/lightgbm/
139- # Install OpenCL (required by LightGBM GPU version)
140- RUN apt-get install -y ocl-icd-libopencl1 clinfo && \
141- mkdir -p /etc/OpenCL/vendors && \
142- echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd && \
143- pip install /tmp/lightgbm/*.whl && \
144- rm -rf /tmp/lightgbm && \
145- /tmp/clean-layer.sh
146- {{ else }}
147- RUN pip install lightgbm==$LIGHTGBM_VERSION && \
148- /tmp/clean-layer.sh
149- {{ end }}
150-
151- # Install JAX
152- {{ if eq .Accelerator "gpu" }}
153- COPY --from=jaxlib_whl /tmp/whl/*.whl /tmp/jax/
154- # b/319722433#comment9: Use pip wheels once versions matches our CUDA version.
155- RUN pip install /tmp/jax/*.whl jax==$JAX_VERSION && \
156- /tmp/clean-layer.sh
157- {{ else }}
158- RUN pip install jax[cpu] && \
159- /tmp/clean-layer.sh
160- {{ end }}
161-
162-
163- # Install GPU specific packages
164- {{ if eq .Accelerator "gpu" }}
165- # Install GPU-only packages
166- # No specific package for nnabla-ext-cuda 12.x minor versions.
167- RUN export PATH=/usr/local/cuda/bin:$PATH && \
168- export CUDA_ROOT=/usr/local/cuda && \
169- pip install pycuda \
170- pynvrtc \
171- pynvml && \
172- /tmp/clean-layer.sh
173- {{ end }}
51+ # RUN micromamba install -y mkl cartopy imagemagick pyproj " shapely<2" && \
52+ # rm -rf /opt/conda/lib/python3.10 /site-packages/pyproj/proj_dir/ && \
53+ # /tmp/clean-layer.sh
17454
17555# b/308525631: Pin Matplotlib until seaborn can be upgraded
17656# to >0.13.0 (now it's stuck by a package conflict with ydata-profiling 4.5.1 ).
@@ -195,28 +75,31 @@ RUN apt-get update && \
19575
19676RUN pip install -f http://h2o-release.s3.amazonaws.com /h2o/latest_stable_Py.html h2o && /tmp/clean-layer.sh
19777
78+
79+ # Keep these variables in sync if base image is updated.
80+ ENV TENSORFLOW_VERSION= 2.17.0 \
81+ # See https://github.com /tensorflow/io#tensorflow-version-compatibility
82+ TENSORFLOW_IO_VERSION= 0.37.1
83+
19884RUN pip install \
19985 " tensorflow==${TENSORFLOW_VERSION}" \
20086 " tensorflow-io==${TENSORFLOW_IO_VERSION}" \
87+ " tensorflow_hub>=0.16.0" \
20188 tensorflow-probability \
20289 tensorflow_decision_forests \
20390 tensorflow-text \
204- "tensorflow_hub>=0.16.0" \
20591 tf-keras \
206- "keras>3" \
20792 keras-cv \
20893 keras-nlp && \
20994 /tmp/clean-layer.sh
21095
21196ADD patches/keras_internal.py \
21297 patches/keras_internal_test.py \
213- /opt/conda /lib/python3.10/site -packages/tensorflow_decision_forests/keras/
98+ /usr/local /lib/python3.10 /dist -packages/tensorflow_decision_forests/keras/
21499
215100# b/350573866: xgboost v2.1.0 breaks learntools
216101RUN apt-get install -y libfreetype6-dev && \
217102 apt-get install -y libglib2.0 -0 libxext6 libsm6 libxrender1 libfontconfig1 --fix-missing && \
218- rm -rf /opt/conda/lib/python3.10/site-packages/numpy* && \
219- pip install "numpy==1.26.4" && \
220103 pip install gensim \
221104 textblob \
222105 wordcloud \
@@ -225,10 +108,7 @@ RUN apt-get install -y libfreetype6-dev && \
225108 hep_ml && \
226109 # NLTK Project datasets
227110 mkdir -p /usr/share/nltk_data && \
228- # NLTK Downloader no longer continues smoothly after an error, so we explicitly list
229- # the corpuses that work
230- # "yes | ..." answers yes to the retry prompt in case of an error. See b/133762095.
231- yes | python -m nltk.downloader -d /usr/share/nltk_data abc alpino averaged_perceptron_tagger \
111+ python -m nltk.downloader -d /usr/share/nltk_data abc alpino averaged_perceptron_tagger \
232112 basque_grammars biocreative_ppi bllip_wsj_no_aux \
233113 book_grammars brown brown_tei cess_cat cess_esp chat80 city_database cmudict \
234114 comtrans conll2000 conll2002 conll2007 crubadan dependency_treebank \
@@ -377,10 +257,9 @@ RUN pip install annoy \
377257 mlcrate && \
378258 /tmp/clean-layer.sh
379259
380- # b/273059949: The pre-installed nbconvert is slow on html conversions and has to be force-uninstalled.
381- # b/274619697: learntools also requires a specific nbconvert right now
382- RUN rm -rf /opt/conda/lib/python3.10/site-packages/{nbconvert,nbclient,mistune,platformdirs}*
383-
260+ # # b/273059949: The pre-installed nbconvert is slow on html conversions and has to be force-uninstalled.
261+ # # b/274619697: learntools also requires a specific nbconvert right now
262+ # RUN rm -rf /opt/conda/lib/python3.10 /site-packages/{nbconvert,nbclient,mistune,platformdirs}*
384263RUN pip install bleach \
385264 certifi \
386265 cycler \
@@ -446,8 +325,8 @@ RUN python -m spacy download en_core_web_sm && python -m spacy download en_core_
446325 #
447326 ###########
448327
449- RUN rm /opt/conda/lib/python3.10/site-packages/google*/ direct_url.json && \
450- rm /opt/conda/lib/python3.10 /site-packages/google*/REQUESTED
328+ # RUN rm /opt/conda/lib/python3.10/site-packages/google*/direct_url.json && \
329+ # rm /opt/conda/lib/python3.10/site-packages/google*/REQUESTED
451330# dlib has a libmkl incompatibility:
452331# test_dlib_face_detector (test_dlib.TestDLib) ... INTEL MKL ERROR: /opt/conda/bin/../lib/libmkl_avx512.so.2: undefined symbol: mkl_sparse_optimize_bsr_trsm_i8.
453332# Intel MKL FATAL ERROR: Cannot load libmkl_avx512.so.2 or libmkl_def.so.2.
@@ -476,9 +355,6 @@ RUN pip install wandb \
476355 Rtree \
477356 accelerate && \
478357 apt-get -y install libspatialindex-dev && \
479- # b/370860329: newer versions are not capable with current tensorflow
480- rm -rf /opt/conda/lib/python3.10 /site-packages/numpy* && \
481- pip install " numpy==1.26.4" && \
482358 pip install pytorch-ignite \
483359 qgrid \
484360 bqplot \
@@ -510,9 +386,6 @@ RUN pip install wandb \
510386 pip install git+https://github.com/facebookresearch/segment-anything.git && \
511387 # b/370860329: newer versions are not capable with current tensorflow
512388 pip install --no-dependencies fastai fastdownload && \
513- # b/343971718: remove duplicate aiohttp installs, and reinstall it
514- rm -rf /opt/conda/lib/python3.10 /site-packages/aiohttp* && \
515- micromamba install --force-reinstall -y aiohttp && \
516389 /tmp/clean-layer.sh
517390
518391# Download base easyocr models.
@@ -543,66 +416,51 @@ ENV TESSERACT_PATH=/usr/bin/tesseract \
543416 # For Theano with MKL
544417 MKL_THREADING_LAYER=GNU
545418
546- # Temporary fixes and patches
547- # Temporary patch for Dask getting downgraded, which breaks Keras
548- RUN pip install --upgrade dask && \
549- # Stop jupyter nbconvert trying to rewrite its folder hierarchy
550- mkdir -p /root/.jupyter && touch /root/.jupyter /jupyter_nbconvert_config.py && touch /root/.jupyter /migrated && \
551- mkdir -p /.jupyter && touch /.jupyter /jupyter_nbconvert_config.py && touch /.jupyter /migrated && \
552- # Stop Matplotlib printing junk to the console on first load
553- sed -i " s/^.*Matplotlib is building the font cache using fc-list.*$/# Warning removed by Kaggle/g" /opt/conda/lib/python3.10 /site-packages/matplotlib/font_manager.py && \
554- # Make matplotlib output in Jupyter notebooks display correctly
555- mkdir -p /etc/ipython/ && echo " c = get_config(); c.IPKernelApp.matplotlib = 'inline'" > /etc/ipython/ipython_config.py && \
556- # Temporary patch for broken libpixman 0.38 in conda-forge, symlink to system libpixman 0.34 untile conda package gets updated to 0.38.5 or higher.
557- ln -sf /usr/lib/x86_64-linux-gnu/libpixman-1.so.0.34.0 /opt/conda/lib/libpixman-1.so.0.38.0 && \
558- # b/333854354: pin jupyter-server to version 2.12.5 ; later versions break LSP (b/333854354)
559- pip install --force-reinstall --no-deps jupyter_server== 2.12.5 && \
560- /tmp/clean-layer.sh
419+ # # Temporary fixes and patches
420+ # # Temporary patch for Dask getting downgraded, which breaks Keras
421+ # RUN pip install --upgrade dask && \
422+ # # Stop jupyter nbconvert trying to rewrite its folder hierarchy
423+ # mkdir -p /root/.jupyter && touch /root/.jupyter/jupyter_nbconvert_config.py && touch /root/.jupyter/migrated && \
424+ # mkdir -p /.jupyter && touch /.jupyter/jupyter_nbconvert_config.py && touch /.jupyter/migrated && \
425+ # # Stop Matplotlib printing junk to the console on first load
426+ # sed -i " s/^. *Matplotlib is building the font cache using fc-list. *$ /# Warning removed by Kaggle/g" /opt/conda/lib/python3.10/site-packages/matplotlib/font_manager.py && \
427+ # # Make matplotlib output in Jupyter notebooks display correctly
428+ # mkdir -p /etc/ipython/ && echo " c = get_config(); c.IPKernelApp.matplotlib = 'inline'" > /etc/ipython/ipython_config.py && \
429+ # # Temporary patch for broken libpixman 0.38 in conda-forge, symlink to system libpixman 0.34 untile conda package gets updated to 0.38.5 or higher.
430+ # ln -sf /usr/lib/x86_64-linux-gnu/libpixman-1.so.0.34.0 /opt/conda/lib/libpixman-1.so.0.38.0 && \
431+ # # b/333854354: pin jupyter-server to version 2.12.5; later versions break LSP (b/333854354)
432+ # pip install --force-reinstall --no-deps jupyter_server==2.12.5 && \
433+ # /tmp/clean-layer.sh
561434
562435# Fix to import bq_helper library without downgrading setuptools
563436RUN mkdir -p ~/src && git clone https://github.com/SohierDane/BigQuery_Helper ~/src/BigQuery_Helper && \
564437 mkdir -p ~/src/BigQuery_Helper/bq_helper && \
565438 mv ~/src/BigQuery_Helper/bq_helper.py ~/src/BigQuery_Helper/bq_helper/__init__.py && \
566439 mv ~/src/BigQuery_Helper/test_helper.py ~/src/BigQuery_Helper/bq_helper/ && \
567440 sed -i 's/)/packages=[" bq_helper" ])/g' ~/src/BigQuery_Helper/setup.py && \
441+ pip install setuptools==70.0.0 && \
568442 pip install -e ~/src/BigQuery_Helper && \
569443 /tmp/clean-layer.sh
570444
571- # Add BigQuery client proxy settings
572- ENV PYTHONUSERBASE " /root/.local"
573- ADD patches/kaggle_gcp.py \
574- patches/kaggle_secrets.py \
575- patches/kaggle_session.py \
576- patches/kaggle_web_client.py \
577- patches/kaggle_datasets.py \
578- patches/log.py \
579- patches/sitecustomize.py \
580- /root/.local /lib/python3.10 /site-packages/
581-
582- # Override default imagemagick policies
583- ADD patches/imagemagick-policy.xml /etc/ImageMagick-6/policy.xml
445+ # These patch are not working as intended:
446+ # # Add BigQuery client proxy settings
447+ # ENV PYTHONUSERBASE " /usr/local"
448+ # ADD patches/kaggle_gcp.py \
449+ # patches/kaggle_secrets.py \
450+ # patches/kaggle_session.py \
451+ # patches/kaggle_web_client.py \
452+ # patches/kaggle_datasets.py \
453+ # patches/log.py \
454+ # patches/sitecustomize.py \
455+ # /root/.local/lib/python3.10/site-packages/
456+
457+ # # Override default imagemagick policies
458+ # ADD patches/imagemagick-policy.xml /etc/ImageMagick-6/policy.xml
584459
585460# Add Kaggle module resolver
586- ADD patches/kaggle_module_resolver.py /opt/conda/lib/python3.10 /site-packages/tensorflow_hub/kaggle_module_resolver.py
587- RUN sed -i '/from tensorflow_hub import uncompressed_module_resolver/a from tensorflow_hub import kaggle_module_resolver' /opt/conda/lib/python3.10 /site-packages/tensorflow_hub/config.py && \
588- sed -i '/_install_default_resolvers()/a \ \ registry.resolver.add_implementation (kaggle_module_resolver.KaggleFileResolver ())' /opt/conda/lib/python3.10 /site-packages/tensorflow_hub/config.py && \
589- # Disable preloaded jupyter modules (they add to startup, and break when they are missing)
590- sed -i /bq_stats/d /etc/ipython/ipython_kernel_config.py && \
591- sed -i /beatrix/d /etc/ipython/ipython_kernel_config.py && \
592- sed -i /bigquery/d /etc/ipython/ipython_kernel_config.py && \
593- sed -i /sql/d /etc/ipython/ipython_kernel_config.py
594-
595- # Force only one libcusolver
596- {{ if eq .Accelerator " gpu" }}
597- RUN rm /opt/conda/bin/.. /lib/libcusolver.so.11 && ln -s /usr/local/cuda/lib64/libcusolver.so.11 /opt/conda/bin/.. /lib/libcusolver.so.11
598- {{ else }}
599- RUN ln -s /usr/local/cuda/lib64/libcusolver.so.11 /opt/conda/bin/.. /lib/libcusolver.so.11
600- {{ end }}
601-
602- # b/270147159: conda ships with a version of libtinfo which is missing version info causing warnings, replace it with a good version.
603- RUN rm /opt/conda/lib/libtinfo.so.6 && ln -s /usr/lib/x86_64-linux-gnu/libtinfo.so.6 /opt/conda/lib/libtinfo.so.6 && \
604- # b/276358430: fix Jupyter lsp freezing up the jupyter server
605- pip install " jupyter-lsp==1.5.1"
461+ ADD patches/kaggle_module_resolver.py /usr/local/lib/python3.10/dist-packages/tensorflow_hub/kaggle_module_resolver.py
462+ RUN sed -i '/from tensorflow_hub import uncompressed_module_resolver/a from tensorflow_hub import kaggle_module_resolver' /usr/local/lib/python3.10/dist-packages/tensorflow_hub/config.py && \
463+ sed -i '/_install_default_resolvers()/a \ \ registry.resolver.add_implementation(kaggle_module_resolver.KaggleFileResolver())' /usr/local/lib/python3.10/dist-packages/tensorflow_hub/config.py
606464
607465# Set backend for matplotlib
608466ENV MPLBACKEND=" agg" \
@@ -626,9 +484,3 @@ LABEL tensorflow-version=$TENSORFLOW_VERSION \
626484# Correlate current release with the git hash inside the kernel editor by running `!cat /etc/git_commit`.
627485RUN echo " $GIT_COMMIT " > /etc/git_commit && echo " $BUILD_DATE " > /etc/build_date
628486
629- {{ if eq .Accelerator " gpu" }}
630- # Remove the CUDA stubs.
631- ENV LD_LIBRARY_PATH= " $LD_LIBRARY_PATH_NO_STUBS" \
632- # Add the CUDA home.
633- CUDA_HOME= /usr/local/cuda
634- {{ end }}
0 commit comments