diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f8923f663..24b1e6706 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -79,3 +79,11 @@ updates: package-ecosystem: pip schedule: interval: weekly + - directory: /preset + groups: + preset: + patterns: + - "*requirements.txt" + package-ecosystem: pip + schedule: + interval: weekly diff --git a/docs/matrix.md b/docs/matrix.md index 6f5c47d82..0af0cf4c9 100644 --- a/docs/matrix.md +++ b/docs/matrix.md @@ -23,13 +23,13 @@ {{ read_csv('assets/max-pvc.csv') }} === "[Preset Containers](https://github.com/intel/ai-containers/blob/main/preset/README.md)" - === "Data Analytics" + === "[Data Analytics](https://hub.docker.com/r/intel/data-analytics)" {{ read_csv('assets/data_analytics.csv') }} - === "Classical ML" + === "[Classical ML](https://hub.docker.com/r/intel/classical-ml)" {{ read_csv('assets/classical_ml.csv') }} - === "Deep Learning" + === "[Deep Learning](https://hub.docker.com/r/intel/deep-learning)" {{ read_csv('assets/deep_learning.csv') }} - === "Inference Optimization" + === "[Inference Optimization](https://hub.docker.com/r/intel/inference-optimization)" {{ read_csv('assets/inference_optimization.csv') }} === "[Workflows](https://hub.docker.com/r/intel/ai-workflows)" diff --git a/docs/scripts/hook.py b/docs/scripts/hook.py index fffa2b00e..3b862bdfd 100644 --- a/docs/scripts/hook.py +++ b/docs/scripts/hook.py @@ -36,10 +36,10 @@ def create_support_matrix(): compose_to_csv("classical-ml", None) # get_repo(models) - # compose_to_csv("preset/data-analytics", "data_analytics") - # compose_to_csv("preset/classical-ml", "classical_ml") - # compose_to_csv("preset/deep-learning", "deep_learning") - # compose_to_csv("preset/inference-optimization", "inference_optimization") + compose_to_csv("preset/data-analytics", "data_analytics") + compose_to_csv("preset/classical-ml", "classical_ml") + compose_to_csv("preset/deep-learning", "deep_learning") + compose_to_csv("preset/inference-optimization", "inference_optimization") compose_to_csv("workflows/charts/huggingface-llm", "genai") diff --git a/preset/classical-ml/.actions.json b/preset/classical-ml/.actions.json index aa899a13a..639f025c2 100644 --- a/preset/classical-ml/.actions.json +++ b/preset/classical-ml/.actions.json @@ -1,6 +1,5 @@ { "PYTHON_VERSION": ["3.9", "3.10"], "experimental": [true], - "RELEASE": ["2024.1.0"], "runner_label": ["PVC"] } diff --git a/preset/classical-ml/Dockerfile b/preset/classical-ml/Dockerfile index 4f13c34f2..01820bcc8 100644 --- a/preset/classical-ml/Dockerfile +++ b/preset/classical-ml/Dockerfile @@ -67,10 +67,10 @@ RUN wget --progress=dot:giga --no-check-certificate https://github.com/conda-for conda config --add channels intel && \ conda init --all && \ conda install -y \ - jupyterlab \ - notebook \ - jupyterhub \ - jupyter-server-proxy \ + 'jupyterlab>=4.1.8' \ + 'notebook>=7.1.3' \ + 'jupyterhub>=4.1.5' \ + 'jupyter-server-proxy>=4.1.2' \ 'mako>=1.2.2' \ 'pyjwt>=2.4.0' \ 'cryptography>=42.0.5' \ @@ -99,15 +99,15 @@ RUN conda create -yn classical-ml -c ${INTEL_CHANNEL} -c conda-forge \ scikit-learn-intelex==${SKLEARNEX_VERSION} \ xgboost=${XGBOOST_VERSION} \ modin-ray=${MODIN_VERSION} \ - python-dotenv \ - tqdm \ - matplotlib-base \ - dataset_librarian \ - threadpoolctl \ - ipython \ - ipykernel \ - kernda \ - protobuf=4.24 \ + 'python-dotenv>=1.0.1' \ + 'tqdm>=4.66.2' \ + 'matplotlib-base>=3.4.3' \ + 'dataset_librarian>=1.0.4' \ + 'threadpoolctl>=3.3.0' \ + 'ipython>=8.18.1' \ + 'ipykernel>=6.29.3' \ + 'kernda>=0.3.0' \ + 'protobuf>=4.24' \ 'pillow>=10.2.0' \ 'tornado>=6.3.3' && \ conda clean -y --all @@ -116,7 +116,7 @@ RUN conda create -yn classical-ml -c ${INTEL_CHANNEL} -c conda-forge \ # PyPI packages RUN conda run -n classical-ml python -m pip install --no-deps --no-cache-dir \ - cloud-data-connector + 'cloud-data-connector==1.0.3' ENV PYTHONSTARTUP=~/.patch_sklearn.py diff --git a/preset/classical-ml/docker-compose.yaml b/preset/classical-ml/docker-compose.yaml index e42e2cac4..a82fbd6d8 100644 --- a/preset/classical-ml/docker-compose.yaml +++ b/preset/classical-ml/docker-compose.yaml @@ -37,11 +37,59 @@ services: https_proxy: ${https_proxy} no_proxy: '' context: . + labels: + docs: classical_ml + org.opencontainers.image.title: "Intel® AI Tools Selector Preset Containers - Classical ML" + org.opencontainers.base.name: "ubuntu:22.04" + org.opencontainers.image.name: "intel/classical-ml" + org.opencontainers.image.version: 2024.1.0-py${PYTHON_VERSION:-3.10} + dependency.python: ${PYTHON_VERSION:-3.10} + dependency.python.pip: requirements.txt + dependency.apt.bzip2: true + dependency.apt.ca-certificates: true + dependency.apt.diffutils: true + dependency.apt.gcc: true + dependency.apt.git: true + dependency.apt.gzip: true + dependency.apt.make: true + dependency.apt.patch: true + dependency.apt.rsync: true + dependency.apt.unzip: true + dependency.apt.wget: true + dependency.apt.xz-utils: true + dependency.conda.jupyterlab: '>=4.1.8' + dependency.conda.notebook: '>=7.1.3' + dependency.conda.jupyterhub: '>=4.1.5' + dependency.conda.jupyter-server-proxy: '>=4.1.2' + dependency.conda.mako: '>=1.2.2' + dependency.conda.pyjwt: '>=2.4.0' + dependency.conda.cryptography: '>=42.0.5' + dependency.conda.nodejs: '>=20.12.2' + dependency.conda.aiohttp: '>=3.9.4' + dependency.conda.idna: '>=3.7' + dependency.conda.oauthlib: '>=3.2.2' + dependency.conda.dpnp: '>=0.14.0' + dependency.conda.numpy: '>=1.26.4' + dependency.conda.python: "=${PYTHON_VERSION:-3.10}" + dependency.conda.scikit-learn-intelex: '>=2024.2.0' + dependency.conda.xgboost: '>=2.0.3' + dependency.conda.modin-ray: '>=0.26.1' + dependency.conda.python-dotenv: '>=1.0.1' + dependency.conda.tqdm: '>=4.66.2' + dependency.conda.matplotlib-base: '>=3.4.3' + dependency.conda.dataset_librarian: '>=1.0.4' + dependency.conda.threadpoolctl: '>=3.3.0' + dependency.conda.ipython: '>=8.18.1' + dependency.conda.ipykernel: '>=6.29.3' + dependency.conda.kernda: '>=0.3.0' + dependency.conda.protobuf: '>=4.24' + dependency.conda.pillow: '>=10.2.0' + dependency.conda.tornado: '>=6.3.3' target: classical-ml-jupyter command: | bash -c "conda run -n classical-ml python -c 'import sklearn; import xgboost; print(\"SciKit:\", sklearn.__version__, \" XGBoost:\",xgboost.__version__)' && \ conda run -n classical-ml python -c 'import modin.pandas as pd, modin.config as cfg; cfg.Engine.put(\"Ray\"); df = pd.DataFrame([1]);print(df+1)'" - image: ${REGISTRY}/${REPO}:b-${GITHUB_RUN_NUMBER:-0}-classical-ml-${IDP_VERSION:-2024.1.0}-py${PYTHON_VERSION:-3.10} + image: ${REGISTRY}/${REPO}:b-${GITHUB_RUN_NUMBER:-0}-classical-ml-2024.1.0-py${PYTHON_VERSION:-3.10} environment: http_proxy: ${http_proxy} https_proxy: ${https_proxy} diff --git a/preset/classical-ml/requirements.txt b/preset/classical-ml/requirements.txt new file mode 100644 index 000000000..d231202db --- /dev/null +++ b/preset/classical-ml/requirements.txt @@ -0,0 +1 @@ +cloud-data-connector==1.0.3 diff --git a/preset/data-analytics/.actions.json b/preset/data-analytics/.actions.json index aa899a13a..639f025c2 100644 --- a/preset/data-analytics/.actions.json +++ b/preset/data-analytics/.actions.json @@ -1,6 +1,5 @@ { "PYTHON_VERSION": ["3.9", "3.10"], "experimental": [true], - "RELEASE": ["2024.1.0"], "runner_label": ["PVC"] } diff --git a/preset/data-analytics/Dockerfile b/preset/data-analytics/Dockerfile index 7fdb37495..c3a50d6c6 100644 --- a/preset/data-analytics/Dockerfile +++ b/preset/data-analytics/Dockerfile @@ -68,16 +68,16 @@ RUN wget --progress=dot:giga --no-check-certificate "https://github.com/conda-fo conda config --add channels intel && \ conda init --all && \ conda install -y \ - jupyterlab \ - notebook \ - jupyterhub \ - jupyter-server-proxy \ + 'jupyterlab>=4.1.8' \ + 'notebook>=7.1.3' \ + 'jupyterhub>=4.1.5' \ + 'jupyter-server-proxy>=4.1.2' \ 'mako>=1.2.2' \ 'pyjwt>=2.4.0' \ 'cryptography>=42.0.5' \ + 'nodejs>=20.12.2' \ 'idna>=3.7' \ 'tqdm>=4.66.2' \ - 'nodejs>=20.12.2' \ && \ jupyter labextension disable "@jupyterlab/apputils-extension:announcements" && \ conda clean -y --all @@ -95,22 +95,22 @@ RUN conda create -yn data-analytics -c "${INTEL_CHANNEL}" -c conda-forge \ numpy="${NUMPY_VERSION}" \ python="${PYTHON_VERSION}" \ modin-ray="${MODIN_VERSION}" \ - python-dotenv \ - tqdm \ - matplotlib-base \ - dataset_librarian \ - threadpoolctl \ - ipython \ - ipykernel \ - kernda \ - protobuf=4.24 \ + 'python-dotenv>=1.0.1' \ + 'tqdm>=4.66.2' \ + 'matplotlib-base>=3.4.3' \ + 'dataset_librarian>=1.0.4' \ + 'threadpoolctl>=3.3.0' \ + 'ipython>=8.18.1' \ + 'ipykernel>=6.29.3' \ + 'kernda>=0.3.0' \ + 'protobuf>=4.24.4' \ 'pillow>=10.2.0' \ 'idna>=3.7' \ 'tornado>=6.3.3' && \ conda clean -y --all RUN conda run -n data-analytics python -m pip install --no-deps --no-cache-dir \ - cloud-data-connector + 'cloud-data-connector==1.0.3' FROM data-analytics-python as data-analytics-jupyter diff --git a/preset/data-analytics/docker-compose.yaml b/preset/data-analytics/docker-compose.yaml index d71b1d2b3..259caceb2 100644 --- a/preset/data-analytics/docker-compose.yaml +++ b/preset/data-analytics/docker-compose.yaml @@ -17,7 +17,7 @@ version: '3' services: - classical-ml: + data-analytics: build: args: BASE_IMAGE: ${BASE_IMAGE:-ubuntu} @@ -35,10 +35,54 @@ services: https_proxy: ${https_proxy} no_proxy: '' context: . + labels: + docs: data_analytics + org.opencontainers.image.title: "Intel® AI Tools Selector Preset Containers - Data Analytics" + org.opencontainers.base.name: "ubuntu:22.04" + org.opencontainers.image.name: "intel/data-analytics" + org.opencontainers.image.version: 2024.1.0-py${PYTHON_VERSION:-3.10} + dependency.python: ${PYTHON_VERSION:-3.10} + dependency.python.pip: requirements.txt + dependency.apt.bzip2: true + dependency.apt.ca-certificates: true + dependency.apt.diffutils: true + dependency.apt.gcc: true + dependency.apt.git: true + dependency.apt.gzip: true + dependency.apt.make: true + dependency.apt.patch: true + dependency.apt.rsync: true + dependency.apt.unzip: true + dependency.apt.wget: true + dependency.apt.xz-utils: true + dependency.conda.jupyterlab: '>=4.1.8' + dependency.conda.notebook: '>=7.1.3' + dependency.conda.jupyterhub: '>=4.1.5' + dependency.conda.jupyter-server-proxy: '>=4.1.2' + dependency.conda.mako: '>=1.2.2' + dependency.conda.pyjwt: '>=2.4.0' + dependency.conda.cryptography: '>=42.0.5' + dependency.conda.nodejs: '>=20.12.2' + dependency.conda.idna: '>=3.7' + dependency.conda.tqdm: '>=4.66.2' + dependency.conda.dpnp: '>=0.14.0' + dependency.conda.numpy: '>=1.26.4' + dependency.conda.python: "=${PYTHON_VERSION:-3.10}" + dependency.conda.modin-ray: '>=0.26.1' + dependency.conda.python-dotenv: '>=1.0.1' + dependency.conda.matplotlib-base: '>=3.4.3' + dependency.conda.dataset_librarian: '>=1.0.4' + dependency.conda.threadpoolctl: '>=3.3.0' + dependency.conda.ipython: '>=8.18.1' + dependency.conda.ipykernel: '>=6.29.3' + dependency.conda.kernda: '>=0.3.0' + dependency.conda.protobuf: '>=4.24.4' + dependency.conda.pillow: '>=10.2.0' + dependency.conda.tornado: '>=6.3.3' target: data-analytics-jupyter command: > bash -c "conda run -n data-analytics python -c 'import modin.pandas as pd, modin.config as cfg; cfg.Engine.put(\"Ray\"); df = pd.DataFrame([1]);print(df+1)'" - image: ${REGISTRY}/${REPO}:b-${GITHUB_RUN_NUMBER:-0}-data-analytics-${IDP_VERSION:-2024.1.0}-py${PYTHON_VERSION:-3.10} + image: ${REGISTRY}/${REPO}:b-${GITHUB_RUN_NUMBER:-0}-data-analytics-2024.1.0-py${PYTHON_VERSION:-3.10} environment: http_proxy: ${http_proxy} https_proxy: ${https_proxy} diff --git a/preset/data-analytics/requirements.txt b/preset/data-analytics/requirements.txt new file mode 100644 index 000000000..d231202db --- /dev/null +++ b/preset/data-analytics/requirements.txt @@ -0,0 +1 @@ +cloud-data-connector==1.0.3 diff --git a/preset/deep-learning/.actions.json b/preset/deep-learning/.actions.json index aa899a13a..639f025c2 100644 --- a/preset/deep-learning/.actions.json +++ b/preset/deep-learning/.actions.json @@ -1,6 +1,5 @@ { "PYTHON_VERSION": ["3.9", "3.10"], "experimental": [true], - "RELEASE": ["2024.1.0"], "runner_label": ["PVC"] } diff --git a/preset/deep-learning/Dockerfile b/preset/deep-learning/Dockerfile index ef62392bf..c4f5e7f6c 100644 --- a/preset/deep-learning/Dockerfile +++ b/preset/deep-learning/Dockerfile @@ -76,13 +76,35 @@ RUN apt-get update && \ RUN apt-get update && \ apt-get install -y --no-install-recommends --fix-missing \ - intel-media-va-driver-non-free libmfx1 libmfxgen1 libvpl2 \ - libegl-mesa0 libegl1-mesa libegl1-mesa-dev libgbm1 libgl1-mesa-dev libgl1-mesa-dri \ - libglapi-mesa libgles2-mesa-dev libglx-mesa0 libigdgmm12 libxatracker2 mesa-va-drivers \ - mesa-vdpau-drivers mesa-vulkan-drivers va-driver-all vainfo hwinfo clinfo + intel-media-va-driver-non-free \ + libmfx1 \ + libmfxgen1 \ + libvpl2 \ + libegl-mesa0 \ + libegl1-mesa \ + libegl1-mesa-dev \ + libgbm1 \ + libgl1-mesa-dev \ + libgl1-mesa-dri \ + libglapi-mesa \ + libgles2-mesa-dev \ + libglx-mesa0 \ + libigdgmm12 \ + libxatracker2 \ + mesa-va-drivers \ + mesa-vdpau-drivers \ + mesa-vulkan-drivers \ + va-driver-all \ + vainfo \ + hwinfo \ + clinfo RUN apt-get install -y --no-install-recommends --fix-missing \ - libigc-dev intel-igc-cm libigdfcl-dev libigfxcmrt-dev level-zero-dev="${LEVEL_ZERO_DEV_VER}" && \ + libigc-dev \ + intel-igc-cm \ + libigdfcl-dev \ + libigfxcmrt-dev \ + level-zero-dev="${LEVEL_ZERO_DEV_VER}" && \ rm -rf /var/lib/apt/lists/* RUN rm /etc/apt/sources.list.d/*list @@ -118,10 +140,10 @@ RUN wget --progress=dot:giga --no-check-certificate "https://github.com/conda-fo conda config --add channels intel && \ conda init --all && \ conda install -c conda-forge \ - jupyterlab \ - notebook \ - jupyterhub \ - jupyter-server-proxy \ + 'jupyterlab>=4.1.8' \ + 'notebook>=7.1.3' \ + 'jupyterhub>=4.1.5' \ + 'jupyter-server-proxy>=4.1.2' \ 'mako>=1.2.2' \ 'pyjwt>=2.4.0' \ 'cryptography>=42.0.5' \ @@ -160,36 +182,36 @@ RUN conda create -yn pytorch-cpu -c intel/label/oneapi -c "${INTEL_CHANNEL}" -c intel-extension-for-pytorch="${IPEX_CPU_VERSION}" \ torchvision="${TORCHVISION_CPU_VERSION}" \ torchaudio="${TORCHAUDIO_CPU_VERSION}" \ - matplotlib-base \ - ipykernel \ - kernda \ + 'matplotlib-base>=3.4.3' \ + 'ipykernel>=6.29.3' \ + 'kernda>=0.3.0' \ 'pillow>=10.2.0' \ 'aiohttp>=3.9.0' \ 'tornado>=6.3.3' \ 'jinja2>=3.1.3' \ 'idna>=3.7' \ - onnx \ + 'onnx>=1.15.0' \ && \ conda clean -y --all # PyPI packages RUN conda run -n pytorch-cpu pip install --no-deps --no-cache-dir --ignore-installed \ - ninja \ - python-dotenv \ + 'ninja>=1.11.1.1' \ + 'python-dotenv>=1.0.1' \ 'tqdm>=4.66.2' \ - cloud-data-connector \ - dataset-librarian && \ + 'cloud-data-connector>=1.0.3' \ + 'dataset-librarian>=1.0.4' && \ conda run -n pytorch-cpu pip install --no-cache-dir --ignore-installed \ - transformers \ - datasets \ - evaluate && \ - conda run -n pytorch-cpu pip install --no-cache-dir -U accelerate && \ + 'transformers>=4.40.2' \ + 'datasets>=2.19.1' \ + 'evaluate>=0.4.2' && \ + conda run -n pytorch-cpu pip install --no-cache-dir -U 'accelerate>=0.30.0' && \ conda run -n pytorch-cpu pip install --no-cache-dir "git+https://github.com/huggingface/optimum-intel.git" && \ conda clean -y --all -RUN conda run -n pytorch-cpu conda install protobuf=4.24 -c conda-forge --override --force-reinstall -y +RUN conda run -n pytorch-cpu conda install 'protobuf=4.24' -c conda-forge --override --force-reinstall -y # PyTorch Installation ARG IDP_VERSION @@ -219,41 +241,41 @@ RUN conda create -yn pytorch-gpu -c intel/label/oneapi -c "${INTEL_CHANNEL}" -c intel-extension-for-pytorch="${IPEX_GPU_VERSION}" \ torchvision="${TORCHVISION_GPU_VERSION}" \ torchaudio="${TORCHAUDIO_GPU_VERSION}" \ - tensorboardx \ - matplotlib-base \ - pandas \ - ipython \ - ipykernel \ - kernda \ + 'tensorboardx>=2.6.2.2' \ + 'matplotlib-base>=3.4.3' \ + 'pandas>=2.2.2' \ + 'ipython>=8.18.1' \ + 'ipykernel>=6.29.3' \ + 'kernda>=0.3.0' \ 'pillow>=10.2.0' \ 'aiohttp>=3.9.0' \ 'tornado>=6.3.3' \ 'jinja2>=3.1.3' \ 'idna>=3.7' \ - onnx \ - packaging=23.2 \ - setuptools=69.1.0 \ + 'onnx>=1.15.0' \ + 'packaging=23.2' \ + 'setuptools=69.1.0' \ && \ conda clean -y --all # PyPI packages RUN conda run -n pytorch-gpu pip install --no-deps --no-cache-dir --ignore-installed \ - ninja \ - python-dotenv \ + 'ninja>=1.11.1.1' \ + 'python-dotenv>=1.0.1' \ 'tqdm>=4.66.2' \ - cloud-data-connector \ - dataset-librarian && \ + 'cloud-data-connector>=1.0.3' \ + 'dataset-librarian>=1.0.4' && \ conda run -n pytorch-gpu pip install --no-cache-dir --ignore-installed \ - transformers \ - datasets \ - evaluate && \ - conda run -n pytorch-gpu pip install --no-cache-dir -U accelerate && \ + 'transformers>=4.40.2' \ + 'datasets>=2.19.1' \ + 'evaluate>=0.4.2' && \ + conda run -n pytorch-gpu pip install --no-cache-dir -U 'accelerate>=0.30.0' && \ conda run -n pytorch-gpu pip install --no-cache-dir "git+https://github.com/huggingface/optimum-intel.git" && \ conda clean -y --all -RUN conda run -n pytorch-gpu conda install protobuf=4.24 -c conda-forge --override --force-reinstall -y +RUN conda run -n pytorch-gpu conda install 'protobuf=4.24' -c conda-forge --override --force-reinstall -y # TensorFlow Installation @@ -283,11 +305,11 @@ RUN conda create -yn tensorflow-cpu -c "${INTEL_CHANNEL}" -c conda-forge \ intel-optimization-for-horovod="${INTEL_HOROVOD}" \ tensorflow="${TF_VERSION}" \ impi-devel="${IMPI_VERSION}" \ - matplotlib-base \ - dataset_librarian \ - ipython \ - ipykernel \ - kernda \ + 'matplotlib-base>=3.4.3' \ + 'dataset_librarian>=1.0.4' \ + 'ipython>=8.18.1' \ + 'ipykernel>=6.29.3' \ + 'kernda>=0.3.0' \ 'pillow>=10.2.0' \ 'cryptography>=42.0.4' \ 'werkzeug>=2.2.3' \ @@ -296,19 +318,19 @@ RUN conda create -yn tensorflow-cpu -c "${INTEL_CHANNEL}" -c conda-forge \ 'pyjwt>=2.8.0' \ 'oauthlib>=3.2.2' \ 'idna>=3.7' \ - onnx \ + 'onnx>=1.14.1' \ && \ conda clean -y --all # PyPI packages RUN conda run -n tensorflow-cpu pip install --no-cache-dir --ignore-installed \ - py-cpuinfo \ - requests \ - cryptography + 'py-cpuinfo>=9.0.0' \ + 'requests>=2.31.0' \ + 'cryptography>=42.0.7' RUN conda run -n tensorflow-cpu pip install --no-deps --no-cache-dir --ignore-installed \ - tensorflow_hub \ + 'tensorflow-hub>=0.16.1' \ 'tqdm>=4.66.2' \ - cloud-data-connector && \ + 'cloud-data-connector>=1.0.3' && \ conda clean -y --all # Tensorflow Env - conda packages @@ -322,11 +344,11 @@ RUN conda create -yn tensorflow-gpu -c "${INTEL_CHANNEL}" -c conda-forge \ intel-optimization-for-horovod="${INTEL_HOROVOD}" \ tensorflow="${TF_VERSION}" \ impi-devel="${IMPI_VERSION}" \ - matplotlib-base \ - dataset_librarian \ - ipython \ - ipykernel \ - kernda \ + 'matplotlib-base>=3.4.3' \ + 'dataset_librarian>=1.0.4' \ + 'ipython>=8.18.1' \ + 'ipykernel>=6.29.3' \ + 'kernda>=0.3.0' \ 'pillow>=10.2.0' \ 'cryptography>=42.0.4' \ 'werkzeug>=2.2.3' \ @@ -335,21 +357,21 @@ RUN conda create -yn tensorflow-gpu -c "${INTEL_CHANNEL}" -c conda-forge \ 'pyjwt>=2.8.0' \ 'oauthlib>=3.2.2' \ 'idna>=3.7' \ - onnx \ - packaging=23.2 \ - setuptools=69.1.0 \ + 'onnx>=1.14.1' \ + 'packaging=23.2' \ + 'setuptools=69.1.0' \ && \ conda clean -y --all # PyPI packages RUN conda run -n tensorflow-gpu pip install --no-cache-dir --ignore-installed \ - py-cpuinfo \ - requests \ - cryptography + 'py-cpuinfo>=9.0.0' \ + 'requests>=2.31.0' \ + 'cryptography>=42.0.7' RUN conda run -n tensorflow-gpu pip install --no-deps --no-cache-dir --ignore-installed \ - tensorflow_hub \ + 'tensorflow-hub>=0.16.1' \ 'tqdm>=4.66.2' \ - cloud-data-connector && \ + 'cloud-data-connector>=1.0.3' && \ conda clean -y --all FROM deep-learning-python as deep-learning-jupyter @@ -408,11 +430,11 @@ USER dev RUN conda install -n pytorch-cpu -c intel/label/oneapi -c "${INTEL_CHANNEL}" -c conda-forge \ deepspeed="${DEEPSPEED_VERSION}" \ - tensorboardx + 'tensorboardx>=2.6.2.2' RUN conda install -n pytorch-gpu -c intel/label/oneapi -c "${INTEL_CHANNEL}" -c conda-forge \ deepspeed="${DEEPSPEED_VERSION}" \ - tensorboardx + 'tensorboardx>=2.6.2.2' COPY --chown=dev notebooks /home/dev/jupyter COPY --chown=dev tests /home/dev/sample-tests diff --git a/preset/deep-learning/docker-compose.yaml b/preset/deep-learning/docker-compose.yaml index 26180671f..12de056fe 100644 --- a/preset/deep-learning/docker-compose.yaml +++ b/preset/deep-learning/docker-compose.yaml @@ -54,6 +54,8 @@ services: https_proxy: ${https_proxy} no_proxy: '' context: . + labels: + docs: false target: deep-learning-jupyter command: | bash -c "conda run -n pytorch-cpu python -c 'import torch;print(torch.__version__);import intel_extension_for_pytorch as ipex;print(ipex.__version__);' && \ @@ -67,7 +69,124 @@ services: - /dev/dri/by-path:/dev/dri/by-path deep-learning: build: + labels: + docs: deep_learning + org.opencontainers.image.title: "Intel® AI Tools Selector Preset Containers - Deep Learning" + org.opencontainers.base.name: "ubuntu:22.04" + org.opencontainers.image.name: "intel/deep-learning" + org.opencontainers.image.version: 2024.1.0-py${PYTHON_VERSION:-3.10} + dependency.python: ${PYTHON_VERSION:-3.10} + dependency.python.pip: requirements.txt + dependency.apt.apt-utils: true + dependency.apt.build-essential: true + dependency.apt.bzip2: true + dependency.apt.ca-certificates: true + dependency.apt.clinfo: true + dependency.apt.cmake: true + dependency.apt.diffutils: true + dependency.apt.g++: true + dependency.apt.gcc: true + dependency.apt.git: true + dependency.apt.gnupg2: true + dependency.apt.gpg-agent: true + dependency.apt.gzip: true + dependency.apt.hwinfo: true + dependency.apt.intel-igc-cm: true + dependency.apt.intel-level-zero-gpu: '=1.3.27642.40-803~22.04' + dependency.apt.intel-media-va-driver-non-free: true + dependency.apt.intel-opencl-icd: '=23.43.27642.40-803~22.04' + dependency.apt.level-zero: '=1.14.0-744~22.04' + dependency.apt.level-zero-dev: '=1.14.0-744~22.04' + dependency.apt.libegl1-mesa: true + dependency.apt.libegl1-mesa-dev: true + dependency.apt.libegl-mesa0: true + dependency.apt.libgbm1: true + dependency.apt.libgl1-mesa-dev: true + dependency.apt.libgl1-mesa-dri: true + dependency.apt.libglapi-mesa: true + dependency.apt.libgles2-mesa-dev: true + dependency.apt.libglx-mesa0: true + dependency.apt.libigc-dev: true + dependency.apt.libigdfcl-dev: true + dependency.apt.libigdgmm12: true + dependency.apt.libigfxcmrt-dev: true + dependency.apt.libmfx1: true + dependency.apt.libmfxgen1: true + dependency.apt.libopenmpi-dev: true + dependency.apt.libvpl2: true + dependency.apt.libxatracker2: true + dependency.apt.make: true + dependency.apt.mesa-va-drivers: true + dependency.apt.mesa-vdpau-drivers: true + dependency.apt.mesa-vulkan-drivers: true + dependency.apt.numactl: true + dependency.apt.openmpi-bin: true + dependency.apt.openmpi-common: true + dependency.apt.openssh-client: true + dependency.apt.openssh-server: true + dependency.apt.patch: true + dependency.apt.rsync: true + dependency.apt.sudo: true + dependency.apt.unzip: true + dependency.apt.va-driver-all: true + dependency.apt.vainfo: true + dependency.apt.wget: true + dependency.apt.xz-utils: true + dependency.conda.jupyterlab: '>=4.1.8' + dependency.conda.aiohttp: '>=3.9.0' + dependency.conda.cryptography: '>=42.0.4' + dependency.conda.dataset_librarian: '>=1.0.4' + dependency.conda.deepspeed: '=0.14.0' + dependency.conda.dpcpp_impl_linux-64: '=2024.1.0' + dependency.conda.dpcpp-cpp-rt: '=2024.1.0' + dependency.conda.dpnp: '=0.14.0' + dependency.conda.idna: '>=3.7' + dependency.conda.impi-devel: '=2021.12' + dependency.conda.intel-extension-for-pytorch_cpu: '=2.2.0=*cpu*' + dependency.conda.intel-extension-for-pytorch_gpu: '=2.1.20=*xpu*' + dependency.conda.intel-extension-for-tensorflow_cpu: '=2.15=*cpu*' + dependency.conda.intel-extension-for-tensorflow_gpu: '=2.15=*xpu*' + dependency.conda.intel-openmp: '=2024.1.0' + dependency.conda.intel-optimization-for-horovod: '=0.28.1.4' + dependency.conda.ipykernel: '>=6.29.3' + dependency.conda.ipython: '>=8.18.1' + dependency.conda.jinja2: '>=3.1.3' + dependency.conda.jupyterhub: '>=4.1.5' + dependency.conda.jupyter-server-proxy: '>=4.1.2' + dependency.conda.kernda: '>=0.3.0' + dependency.conda.mako: '>=1.2.2' + dependency.conda.matplotlib-base: '>=3.4.3' + dependency.conda.mkl-dpcpp: '2024.1.0' + dependency.conda.nodejs: '>=20.12.2' + dependency.conda.notebook: '>=7.1.3' + dependency.conda.numpy: '=1.26.4' + dependency.conda.oauthlib: '>=3.2.2' + dependency.conda.oneccl_bind_pt_cpu: '=2.2.0=*cpu*' + dependency.conda.oneccl_bind_pt_gpu: '=2.1.200=*xpu*' + dependency.conda.onnx: '>=1.14.1' + dependency.conda.packaging: '=23.2' + dependency.conda.pandas: '>=2.2.2' + dependency.conda.pillow: '>=10.2.0' + dependency.conda.protobuf: '=4.24' + dependency.conda.pyjwt: '>=2.4.0' + dependency.conda.python: "=${PYTHON_VERSION:-3.10}" + dependency.conda.pytorch_cpu: '=2.2.0=*cpu*' + dependency.conda.pytorch_gpu: '=2.1.0=*xpu*' + dependency.conda.setuptools: '=69.1.0' + dependency.conda.tensorboardx: '>=2.6.2.2' + dependency.conda.tensorflow: '=2.15' + dependency.conda.torchaudio_cpu: '=2.2.0=*cpu*' + dependency.conda.torchaudio_gpu: '=2.1.0=*xpu*' + dependency.conda.torchvision_cpu: '=0.17=*cpu*' + dependency.conda.torchvision_gpu: '=0.16.0=*xpu*' + dependency.conda.tornado: '>=6.3.3' + dependency.conda.tqdm: '>=4.66.2' + dependency.conda.werkzeug: '>=2.2.3' target: distributed-deep-learning + depends_on: + - dl-base + extends: dl-base + image: ${REGISTRY}/${REPO}:b-${GITHUB_RUN_NUMBER:-0}-deep-learning-2024.1.0-py${PYTHON_VERSION:-3.10} command: | bash -c "conda run -n pytorch-cpu python -c 'import torch;print(torch.__version__);import intel_extension_for_pytorch as ipex;print(ipex.__version__);' && \ conda run -n pytorch-cpu bash -c 'mpirun --version' && \ @@ -79,7 +198,3 @@ services: conda run -n tensorflow-gpu python -c 'from tensorflow.python.client import device_lib; print(device_lib.list_local_devices())' && \ conda run -n tensorflow-gpu bash -c 'horovodrun --check-build && mpirun --version' && \ conda run -n tensorflow-gpu python -c 'import horovod.tensorflow as hvd;hvd.init();import horovod.tensorflow'" - depends_on: - - dl-base - extends: dl-base - image: ${REGISTRY}/${REPO}:b-${GITHUB_RUN_NUMBER:-0}-deep-learning-${IDP_VERSION:-2024.1.0}-py${PYTHON_VERSION:-3.10} diff --git a/preset/deep-learning/requirements.txt b/preset/deep-learning/requirements.txt new file mode 100644 index 000000000..4122126b5 --- /dev/null +++ b/preset/deep-learning/requirements.txt @@ -0,0 +1,14 @@ +accelerate>=0.30.0 +cloud-data-connector>=1.0.3 +cryptography>=42.0.7 +dataset-librarian>=1.0.4 +datasets>=2.19.1 +evaluate>=0.4.2 +git+https://github.com/huggingface/optimum-intel.git +ninja>=1.11.1.1 +py-cpuinfo>=9.0.0 +python-dotenv>=1.0.1 +requests>=2.31.0 +tensorflow-hub>=0.16.1 +tqdm>=4.66.2 +transformers>=4.40.2 diff --git a/preset/inference-optimization/.actions.json b/preset/inference-optimization/.actions.json index aa899a13a..639f025c2 100644 --- a/preset/inference-optimization/.actions.json +++ b/preset/inference-optimization/.actions.json @@ -1,6 +1,5 @@ { "PYTHON_VERSION": ["3.9", "3.10"], "experimental": [true], - "RELEASE": ["2024.1.0"], "runner_label": ["PVC"] } diff --git a/preset/inference-optimization/Dockerfile b/preset/inference-optimization/Dockerfile index 0d982ab2b..6689b4370 100644 --- a/preset/inference-optimization/Dockerfile +++ b/preset/inference-optimization/Dockerfile @@ -34,13 +34,13 @@ RUN conda install -yn tensorflow-gpu -c "${INTEL_CHANNEL}" -c conda-forge \ neural-compressor="${NEURAL_COMPRESSOR_VERSION}" RUN conda run -n tensorflow-cpu python -m pip install --no-deps --no-cache-dir \ - tf2onnx \ - onnxruntime && \ + 'tf2onnx>=1.16.1' \ + 'onnxruntime>=1.17.3' && \ conda clean -y --all RUN conda run -n tensorflow-gpu python -m pip install --no-deps --no-cache-dir \ - tf2onnx \ - onnxruntime && \ + 'tf2onnx>=1.16.1' \ + 'onnxruntime>=1.17.3' && \ conda clean -y --all COPY --chown=dev notebooks /home/dev/jupyter diff --git a/preset/inference-optimization/docker-compose.yaml b/preset/inference-optimization/docker-compose.yaml index 2fbe59f0e..0d2760b57 100644 --- a/preset/inference-optimization/docker-compose.yaml +++ b/preset/inference-optimization/docker-compose.yaml @@ -22,6 +22,7 @@ services: args: BASE_IMAGE: ${BASE_IMAGE:-ubuntu} BASE_TAG: ${BASE_TAG:-22.04} + DEEPSPEED_VERSION: ${DEEPSPEED_VERSION:-0.14.0} DEVICE: ${DEVICE:-flex} DPNP_VERSION: ${NUMBA_DPEX_VERSION:-0.14.0} HOROVOD_VERSION: ${HOROVOD_VERSION:-0.28.1.4} @@ -54,6 +55,8 @@ services: https_proxy: ${https_proxy} no_proxy: '' context: ../deep-learning + labels: + docs: false target: deep-learning-jupyter environment: http_proxy: ${http_proxy} @@ -72,11 +75,125 @@ services: args: COMPOSE_PROJECT_NAME: ${COMPOSE_PROJECT_NAME:-preset} context: . + labels: + docs: inference_optimization + org.opencontainers.image.title: "Intel® AI Tools Selector Preset Containers - Inference Optimization" + org.opencontainers.base.name: "intel/deep-learning" + org.opencontainers.image.name: "intel/inference-optimization" + org.opencontainers.image.version: 2024.1.0-py${PYTHON_VERSION:-3.10} + dependency.python: ${PYTHON_VERSION:-3.10} + dependency.python.pip: requirements.txt + dependency.apt.apt-utils: true + dependency.apt.build-essential: true + dependency.apt.bzip2: true + dependency.apt.ca-certificates: true + dependency.apt.clinfo: true + dependency.apt.cmake: true + dependency.apt.diffutils: true + dependency.apt.g++: true + dependency.apt.gcc: true + dependency.apt.git: true + dependency.apt.gnupg2: true + dependency.apt.gpg-agent: true + dependency.apt.gzip: true + dependency.apt.hwinfo: true + dependency.apt.intel-igc-cm: true + dependency.apt.intel-level-zero-gpu: '1.3.27642.40-803~22.04' + dependency.apt.intel-media-va-driver-non-free: true + dependency.apt.intel-opencl-icd: '23.43.27642.40-803~22.04' + dependency.apt.level-zero: '1.14.0-744~22.04' + dependency.apt.level-zero-dev: '1.14.0-744~22.04' + dependency.apt.libegl1-mesa: true + dependency.apt.libegl1-mesa-dev: true + dependency.apt.libegl-mesa0: true + dependency.apt.libgbm1: true + dependency.apt.libgl1-mesa-dev: true + dependency.apt.libgl1-mesa-dri: true + dependency.apt.libglapi-mesa: true + dependency.apt.libgles2-mesa-dev: true + dependency.apt.libglx-mesa0: true + dependency.apt.libigc-dev: true + dependency.apt.libigdfcl-dev: true + dependency.apt.libigdgmm12: true + dependency.apt.libigfxcmrt-dev: true + dependency.apt.libmfx1: true + dependency.apt.libmfxgen1: true + dependency.apt.libopenmpi-dev: true + dependency.apt.libvpl2: true + dependency.apt.libxatracker2: true + dependency.apt.make: true + dependency.apt.mesa-va-drivers: true + dependency.apt.mesa-vdpau-drivers: true + dependency.apt.mesa-vulkan-drivers: true + dependency.apt.numactl: true + dependency.apt.openmpi-bin: true + dependency.apt.openmpi-common: true + dependency.apt.openssh-client: true + dependency.apt.openssh-server: true + dependency.apt.patch: true + dependency.apt.rsync: true + dependency.apt.sudo: true + dependency.apt.unzip: true + dependency.apt.va-driver-all: true + dependency.apt.vainfo: true + dependency.apt.wget: true + dependency.apt.xz-utils: true + dependency.conda.jupyterlab: '>=4.1.8' + dependency.conda.aiohttp: '>=3.9.0' + dependency.conda.cryptography: '>=42.0.4' + dependency.conda.dataset_librarian: '>=1.0.4' + dependency.conda.deepspeed: '>=0.14.0' + dependency.conda.dpcpp_impl_linux-64: '>=2024.1.' + dependency.conda.dpcpp-cpp-rt: '>=2024.1.' + dependency.conda.dpnp: '>=0.14.0' + dependency.conda.idna: '>=3.7' + dependency.conda.impi-devel: '>=2021.12' + dependency.conda.intel-extension-for-pytorch_cpu: '>=2.2.0=*cpu*' + dependency.conda.intel-extension-for-pytorch_gpu: '>=2.1.20=*xpu*' + dependency.conda.intel-extension-for-tensorflow_cpu: '>=2.15=*cpu*' + dependency.conda.intel-extension-for-tensorflow_gpu: '>=2.15=*xpu*' + dependency.conda.intel-openmp: '>=2024.1.0' + dependency.conda.intel-optimization-for-horovod: '>=0.28.1.4' + dependency.conda.ipykernel: '>=6.29.3' + dependency.conda.ipython: '>=8.18.1' + dependency.conda.jinja2: '>=3.1.3' + dependency.conda.jupyterhub: '>=4.1.5' + dependency.conda.jupyter-server-proxy: '>=4.1.2' + dependency.conda.kernda: '>=0.3.0' + dependency.conda.mako: '>=1.2.2' + dependency.conda.matplotlib-base: '>=3.4.3' + dependency.conda.mkl-dpcpp: '>=2024.1.0' + dependency.conda.neural-compressor: '>=2.4.1' + dependency.conda.nodejs: '>=20.12.2' + dependency.conda.notebook: '>=7.1.3' + dependency.conda.numpy: '>=1.26.4' + dependency.conda.oauthlib: '>=3.2.2' + dependency.conda.oneccl_bind_pt_cpu: '>=2.2.0=*cpu*' + dependency.conda.oneccl_bind_pt_gpu: '>=2.1.200=*xpu*' + dependency.conda.onnx: '>=1.14.1' + dependency.conda.packaging: '>=23.2' + dependency.conda.pandas: '>=2.2.2' + dependency.conda.pillow: '>=10.2.0' + dependency.conda.protobuf: '>=4.24' + dependency.conda.pyjwt: '>=2.4.0' + dependency.conda.python: "=${PYTHON_VERSION:-3.10}" + dependency.conda.pytorch_cpu: '>=2.2.0=*cpu*' + dependency.conda.pytorch_gpu: '>=2.1.0=*xpu*' + dependency.conda.setuptools: '>=69.1.0' + dependency.conda.tensorboardx: '>=2.6.2.2' + dependency.conda.tensorflow: '>=2.15' + dependency.conda.torchaudio_cpu: '>=2.2.0=*cpu*' + dependency.conda.torchaudio_gpu: '>=2.1.0=*xpu*' + dependency.conda.torchvision_cpu: '>=0.17=*cpu*' + dependency.conda.torchvision_gpu: '>=0.16.0=*xpu*' + dependency.conda.tornado: '>=6.3.3' + dependency.conda.tqdm: '>=4.66.2' + dependency.conda.werkzeug: '>=2.2.3' target: inference-optimization depends_on: - dl-base extends: dl-base - image: ${REGISTRY}/${REPO}:b-${GITHUB_RUN_NUMBER:-0}-inference-optimization-${IDP_VERSION:-2024.1.0}-py${PYTHON_VERSION:-3.10} + image: ${REGISTRY}/${REPO}:b-${GITHUB_RUN_NUMBER:-0}-inference-optimization-2024.1.0-py${PYTHON_VERSION:-3.10} command: | bash -c "conda run -n pytorch-cpu python -c 'import intel_extension_for_pytorch as ipex;print(ipex.__version__);' && \ conda run -n pytorch-cpu python -c 'import neural_compressor;print(\"Neural Compressor Version:\", neural_compressor.__version__)' && \ diff --git a/preset/inference-optimization/requirements.txt b/preset/inference-optimization/requirements.txt new file mode 100644 index 000000000..15dad7740 --- /dev/null +++ b/preset/inference-optimization/requirements.txt @@ -0,0 +1,16 @@ +accelerate>=0.30.0 +cloud-data-connector>=1.0.3 +cryptography>=42.0.7 +dataset-librarian>=1.0.4 +datasets>=2.19.1 +evaluate>=0.4.2 +git+https://github.com/huggingface/optimum-intel.git +ninja>=1.11.1.1 +onnxruntime>=1.17.3 +py-cpuinfo>=9.0.0 +python-dotenv>=1.0.1 +requests>=2.31.0 +tensorflow-hub>=0.16.1 +tf2onnx>==1.16.1 +tqdm>=4.66.2 +transformers>=4.40.2