From 1f18adc1f8bb349d79931768c6fd3041ab788d1a Mon Sep 17 00:00:00 2001 From: Ziqun Ye Date: Wed, 18 Oct 2023 19:24:43 -0700 Subject: [PATCH 1/7] first working version --- ads/opctl/conda/pack.py | 13 +-- ads/opctl/docker/Dockerfile.job | 143 ++++++++++++++++++++++---------- 2 files changed, 104 insertions(+), 52 deletions(-) diff --git a/ads/opctl/conda/pack.py b/ads/opctl/conda/pack.py index f64fa1f46..bad61cb99 100644 --- a/ads/opctl/conda/pack.py +++ b/ads/opctl/conda/pack.py @@ -23,7 +23,7 @@ def main(pack_folder_path): manifest_path = glob.glob(os.path.join(pack_folder_path, "*_manifest.yaml"))[0] with open(manifest_path) as f: env = yaml.safe_load(f.read()) - + with tempfile.TemporaryDirectory() as td: process = subprocess.Popen( ["conda", "env", "export", "--prefix", pack_folder_path], @@ -31,11 +31,12 @@ def main(pack_folder_path): stderr=subprocess.PIPE, ) stdout, stderr = process.communicate() - if stderr: - print(stderr) - raise Exception( - f"Error export environment information from {pack_folder_path}" - ) + # import pdb; pdb.set_trace() + # if stderr: + # print(stderr) + # raise Exception( + # f"Error export environment information from {pack_folder_path}" + # ) try: new_env_info = yaml.safe_load(StringIO(stdout.decode("utf-8"))) except Exception as e: diff --git a/ads/opctl/docker/Dockerfile.job b/ads/opctl/docker/Dockerfile.job index cbe8fe800..d9bbe6829 100644 --- a/ads/opctl/docker/Dockerfile.job +++ b/ads/opctl/docker/Dockerfile.job @@ -1,7 +1,5 @@ -# Copyright (c) 2021, 2023 Oracle and/or its affiliates. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ - -FROM ghcr.io/oracle/oraclelinux:7-slim +# Used OL8 because miniconda required a higher version of glibc that was unavoidable +FROM ocr-docker-remote.artifactory.oci.oraclecorp.com/os/oraclelinux:8-slim-fips # Configure environment ENV DATASCIENCE_USER datascience @@ -11,76 +9,139 @@ ENV DATASCIENCE_INSTALL_DIR /etc/datascience ENV LOGS_DIRECTORY /logs ARG release=19 -ARG update=13 +ARG update=10 +ARG target=cpu + +# Move this arg around the file as needed when you want to use cache for local builds/testing to speed up the process +# This will allow you to set success points where you know previous code in the dockerfile is working +# ARG RAND + +COPY common /etc/datascience +COPY etc/yum.repos.d /etc/yum.repos.d + +# source: https://confluence.oraclecorp.com/confluence/pages/viewpage.action?spaceKey=ESEDoc&title=Artifactory+FAQ +ADD Symantec_Private_SSL_SHA256_CA.cer /etc/pki/ca-trust/source/anchors +ADD Symantec_Private_SSL_SHA256_Root.cer /etc/pki/ca-trust/source/anchors + +RUN update-ca-trust + +RUN microdnf install yum yum-utils && yum clean all && rm -rf /var/cache/yum RUN \ - yum -y -q install oracle-release-el7 && \ - yum-config-manager --add-repo http://yum.oracle.com/repo/OracleLinux/OL7/developer_EPEL/x86_64 && \ - yum-config-manager --enable ol7_optional_latest --enable ol7_addons --enable ol7_software_collections --enable ol7_oracle_instantclient > /dev/null && \ - yum groupinstall -y -q 'Development Tools' && \ - yum update -y && \ - yum install -y --setopt=skip_missing_names_on_install=False \ + yum install -y \ + oracle-release-el8 \ + oraclelinux-developer-release-el8 \ + oracle-epel-release-el8 \ + https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ + yum-config-manager --enable ol8_addons --enable ol8_oracle_instantclient > /dev/null && yum -y update && \ + yum clean all && \ + rm -rf /var/cache/yum/* + +RUN \ + yum install -y \ + --setopt=skip_missing_names_on_install=False \ bzip2 \ - curl \ git \ - gcc-gfortran \ + httpd \ libcurl-devel \ libxml2-devel \ + java-11-openjdk-headless \ oracle-instantclient${release}.${update}-basic \ oracle-instantclient${release}.${update}-sqlplus \ openssl \ openssl-devel \ patch \ + snappy \ + gcc-c++ \ sudo \ unzip \ + vi \ zip \ - gcc-c++ \ wget \ + jq \ gcc \ + htop \ + git-lfs \ + pcsc-lite-libs \ + perl-Env \ + perl-CPAN \ + perl-devel \ + curl-devel \ + expat-devel \ + asciidoc \ + xmlto \ + supervisor && \ + yum clean all && \ + rm -rf /var/cache/yum/* + +RUN \ + yum groupinstall -y -q "development tools" \ && yum clean all \ && rm -rf /var/cache/yum/* # setup user RUN \ - mkdir -p /home/$DATASCIENCE_USER && \ - useradd -m -s /bin/bash -N -u $DATASCIENCE_UID $DATASCIENCE_USER && \ - chown -R $DATASCIENCE_USER /home/$DATASCIENCE_USER && \ - chown -R $DATASCIENCE_USER:users /usr/local/ && \ - touch /etc/sudoers.d/$DATASCIENCE_USER && echo "$DATASCIENCE_USER ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers.d/$DATASCIENCE_USER && \ - mkdir -p $DATASCIENCE_INSTALL_DIR && chown $DATASCIENCE_USER $DATASCIENCE_INSTALL_DIR && \ - mkdir -p $LOGS_DIRECTORY && chown -R $DATASCIENCE_USER:users $LOGS_DIRECTORY && \ - mkdir -p $LOGS_DIRECTORY/harness && chown -R $DATASCIENCE_USER:users $LOGS_DIRECTORY/harness - -RUN mkdir -p /etc/datascience/build -RUN mkdir -p $DATASCIENCE_INSTALL_DIR/{pre-build-ds,post-build-ds,pre-run-ds,pre-run-user} - -#conda + mkdir -p /home/$DATASCIENCE_USER && \ + useradd -m -s /bin/bash -N -u $DATASCIENCE_UID $DATASCIENCE_USER && \ + chown $DATASCIENCE_USER /home/$DATASCIENCE_USER && \ + chown -R $DATASCIENCE_USER:users /usr/local/ && \ + touch /etc/sudoers.d/$DATASCIENCE_USER && echo "$DATASCIENCE_USER ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers.d/$DATASCIENCE_USER && \ + mkdir -p $DATASCIENCE_INSTALL_DIR && chown $DATASCIENCE_USER $DATASCIENCE_INSTALL_DIR + +# COPY build-tool-entrypoint.sh /etc/datascience/build-tool-entrypoint.sh +# COPY install-capture-entrypoint.sh /etc/datascience/install-capture-entrypoint.sh + # set a default language for localization. necessary for oci cli ARG LANG=en_US.utf8 ENV LANG=$LANG ENV SHELL=/bin/bash +############### SETUP ROOT (BASE) ENVIRONMENT WITH CONDA ###################### +ARG PYTHON_MAJOR='3' +ARG PYTHON_MINOR='8' +ARG PYTHON_PATCH='' +ARG MINICONDA_VER=py${PYTHON_MAJOR}${PYTHON_MINOR}_23.5.2-0 +ARG MAMBAFORGE_VER=23.3.1-1 + # set /opt folder permissions for $DATASCIENCE_USER. Conda is going to live in this folder. -RUN chown -R $DATASCIENCE_USER /opt +RUN chown $DATASCIENCE_USER /opt USER $DATASCIENCE_USER WORKDIR /home/datascience -RUN wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /home/datascience/miniconda.sh \ - && /bin/bash /home/datascience/miniconda.sh -f -b -p /opt/conda \ - && rm /home/datascience/miniconda.sh \ - && /opt/conda/bin/conda clean -yaf -WORKDIR / +# Note in order to run sudo commands as a non root user, you must specify --credential yes if using qemu static to build the image +RUN wget -nv https://artifactory.oci.oraclecorp.com/odsc-dev-generic-local/datascience/Miniconda3-${MINICONDA_VER}-$(uname)-$(uname -m).sh \ + -O /home/datascience/Miniconda3.sh \ + && /bin/bash /home/datascience/Miniconda3.sh -f -b -p /opt/conda \ + && rm /home/datascience/Miniconda3.sh \ + && wget -nv https://artifactory.oci.oraclecorp.com/odsc-dev-generic-local/datascience/Mambaforge-${MAMBAFORGE_VER}-$(uname)-$(uname -m).sh \ + -O /home/datascience/Mambaforge3.sh \ + && /bin/bash /home/datascience/Mambaforge3.sh -f -b -p /opt/conda \ + && rm /home/datascience/Mambaforge3.sh \ + && ls /opt/**/* + +# Adding conda to the path +ENV PATH="/opt/conda/bin:${PATH}" + +# Set activate conda by default USER root RUN printf "#!/bin/bash\nsource /opt/conda/bin/activate\n" > /etc/profile.d/enableconda.sh \ && chmod +x /etc/profile.d/enableconda.sh USER $DATASCIENCE_USER -ENV PATH="/opt/conda/bin:${PATH}" WORKDIR /home/datascience -COPY docker/base-env.yaml /opt/base-env.yaml -RUN conda env update -q -n root -f /opt/base-env.yaml && conda clean -yaf && rm -rf /home/datascience/.cache/pip +COPY base-env-arm.yaml /opt/base-env.yaml +COPY jupyter-arm.yaml /opt/jupyter.yaml + +# Needs to be set to prevent a hang with mamba installations if this dockerfile +# is being built by running qemu-user-static to emulate an ARM machine +# +ENV G_SLICE=always-malloc + +# Replaced conda with mamba. Conda wasn't able to resolve conflicts +RUN mamba env update -n root -f /opt/jupyter.yaml && mamba clean -yaf && rm -rf /home/datascience/.cache/pip + USER $DATASCIENCE_USER @@ -96,13 +157,3 @@ ARG PIP_INDEX_URL ############# Setup Conda environment tools ########################### ARG RAND=1 - -ARG RUN_WORKING_DIR="/home/datascience" -WORKDIR $RUN_WORKING_DIR - -# clean tmp folder -RUN rm -rf /tmp/* - -RUN mkdir -p /etc/datascience/operators - -USER datascience From f3f189d0d662400154fe0478ba1b89eeec033607 Mon Sep 17 00:00:00 2001 From: Ziqun Ye Date: Mon, 23 Oct 2023 22:07:40 -0700 Subject: [PATCH 2/7] use arm base image when local machine is detected as m# machine --- ads/opctl/conda/cmds.py | 3 +- ads/opctl/conda/pack.py | 12 +- ads/opctl/docker/Dockerfile.job | 143 +++++----------- ads/opctl/docker/Dockerfile_arm.job | 162 ++++++++++++++++++ ads/opctl/docker/base-env-arm.yaml | 13 ++ .../docker/common/install-runtime-deps.sh | 25 +++ ads/opctl/docker/jupyter-arm.yaml | 19 ++ ads/opctl/utils.py | 25 ++- pyproject.toml | 1 + 9 files changed, 291 insertions(+), 112 deletions(-) create mode 100644 ads/opctl/docker/Dockerfile_arm.job create mode 100644 ads/opctl/docker/base-env-arm.yaml create mode 100755 ads/opctl/docker/common/install-runtime-deps.sh create mode 100644 ads/opctl/docker/jupyter-arm.yaml diff --git a/ads/opctl/conda/cmds.py b/ads/opctl/conda/cmds.py index e002336ca..3fd3459c9 100644 --- a/ads/opctl/conda/cmds.py +++ b/ads/opctl/conda/cmds.py @@ -602,7 +602,8 @@ def _publish( ) except Exception: raise RuntimeError(f"Could not pack environment {conda_slug}.") - + if "/" in conda_slug: + raise ValueError("Invalid conda_slug. found `/` in slug name. Please use a different slug name.") pack_file = os.path.join(pack_folder_path, f"{conda_slug}.tar.gz") if not os.path.exists(pack_file): raise RuntimeError(f"Pack {pack_file} was not created.") diff --git a/ads/opctl/conda/pack.py b/ads/opctl/conda/pack.py index bad61cb99..68b66dc69 100644 --- a/ads/opctl/conda/pack.py +++ b/ads/opctl/conda/pack.py @@ -31,12 +31,12 @@ def main(pack_folder_path): stderr=subprocess.PIPE, ) stdout, stderr = process.communicate() - # import pdb; pdb.set_trace() - # if stderr: - # print(stderr) - # raise Exception( - # f"Error export environment information from {pack_folder_path}" - # ) + + if process.returncode and stderr: + print(stderr) + raise Exception( + f"Error export environment information from {pack_folder_path}" + ) try: new_env_info = yaml.safe_load(StringIO(stdout.decode("utf-8"))) except Exception as e: diff --git a/ads/opctl/docker/Dockerfile.job b/ads/opctl/docker/Dockerfile.job index d9bbe6829..c2eb3fa03 100644 --- a/ads/opctl/docker/Dockerfile.job +++ b/ads/opctl/docker/Dockerfile.job @@ -1,5 +1,7 @@ -# Used OL8 because miniconda required a higher version of glibc that was unavoidable -FROM ocr-docker-remote.artifactory.oci.oraclecorp.com/os/oraclelinux:8-slim-fips +# Copyright (c) 2021, 2023 Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +FROM ghcr.io/oracle/oraclelinux:7-slim # Configure environment ENV DATASCIENCE_USER datascience @@ -9,139 +11,76 @@ ENV DATASCIENCE_INSTALL_DIR /etc/datascience ENV LOGS_DIRECTORY /logs ARG release=19 -ARG update=10 -ARG target=cpu - -# Move this arg around the file as needed when you want to use cache for local builds/testing to speed up the process -# This will allow you to set success points where you know previous code in the dockerfile is working -# ARG RAND - -COPY common /etc/datascience -COPY etc/yum.repos.d /etc/yum.repos.d - -# source: https://confluence.oraclecorp.com/confluence/pages/viewpage.action?spaceKey=ESEDoc&title=Artifactory+FAQ -ADD Symantec_Private_SSL_SHA256_CA.cer /etc/pki/ca-trust/source/anchors -ADD Symantec_Private_SSL_SHA256_Root.cer /etc/pki/ca-trust/source/anchors - -RUN update-ca-trust - -RUN microdnf install yum yum-utils && yum clean all && rm -rf /var/cache/yum +ARG update=13 RUN \ - yum install -y \ - oracle-release-el8 \ - oraclelinux-developer-release-el8 \ - oracle-epel-release-el8 \ - https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ - yum-config-manager --enable ol8_addons --enable ol8_oracle_instantclient > /dev/null && yum -y update && \ - yum clean all && \ - rm -rf /var/cache/yum/* - -RUN \ - yum install -y \ - --setopt=skip_missing_names_on_install=False \ + yum -y -q install oracle-release-el7 && \ + yum-config-manager --add-repo http://yum.oracle.com/repo/OracleLinux/OL7/developer_EPEL/x86_64 && \ + yum-config-manager --enable ol7_optional_latest --enable ol7_addons --enable ol7_software_collections --enable ol7_oracle_instantclient > /dev/null && \ + yum groupinstall -y -q 'Development Tools' && \ + yum update -y && \ + yum install -y --setopt=skip_missing_names_on_install=False \ bzip2 \ + curl \ git \ - httpd \ + gcc-gfortran \ libcurl-devel \ libxml2-devel \ - java-11-openjdk-headless \ oracle-instantclient${release}.${update}-basic \ oracle-instantclient${release}.${update}-sqlplus \ openssl \ openssl-devel \ patch \ - snappy \ - gcc-c++ \ sudo \ unzip \ - vi \ zip \ + gcc-c++ \ wget \ - jq \ gcc \ - htop \ - git-lfs \ - pcsc-lite-libs \ - perl-Env \ - perl-CPAN \ - perl-devel \ - curl-devel \ - expat-devel \ - asciidoc \ - xmlto \ - supervisor && \ - yum clean all && \ - rm -rf /var/cache/yum/* - -RUN \ - yum groupinstall -y -q "development tools" \ && yum clean all \ && rm -rf /var/cache/yum/* # setup user RUN \ - mkdir -p /home/$DATASCIENCE_USER && \ - useradd -m -s /bin/bash -N -u $DATASCIENCE_UID $DATASCIENCE_USER && \ - chown $DATASCIENCE_USER /home/$DATASCIENCE_USER && \ - chown -R $DATASCIENCE_USER:users /usr/local/ && \ - touch /etc/sudoers.d/$DATASCIENCE_USER && echo "$DATASCIENCE_USER ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers.d/$DATASCIENCE_USER && \ - mkdir -p $DATASCIENCE_INSTALL_DIR && chown $DATASCIENCE_USER $DATASCIENCE_INSTALL_DIR - -# COPY build-tool-entrypoint.sh /etc/datascience/build-tool-entrypoint.sh -# COPY install-capture-entrypoint.sh /etc/datascience/install-capture-entrypoint.sh - + mkdir -p /home/$DATASCIENCE_USER && \ + useradd -m -s /bin/bash -N -u $DATASCIENCE_UID $DATASCIENCE_USER && \ + chown -R $DATASCIENCE_USER /home/$DATASCIENCE_USER && \ + chown -R $DATASCIENCE_USER:users /usr/local/ && \ + touch /etc/sudoers.d/$DATASCIENCE_USER && echo "$DATASCIENCE_USER ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers.d/$DATASCIENCE_USER && \ + mkdir -p $DATASCIENCE_INSTALL_DIR && chown $DATASCIENCE_USER $DATASCIENCE_INSTALL_DIR && \ + mkdir -p $LOGS_DIRECTORY && chown -R $DATASCIENCE_USER:users $LOGS_DIRECTORY && \ + mkdir -p $LOGS_DIRECTORY/harness && chown -R $DATASCIENCE_USER:users $LOGS_DIRECTORY/harness + +RUN mkdir -p /etc/datascience/build +RUN mkdir -p $DATASCIENCE_INSTALL_DIR/{pre-build-ds,post-build-ds,pre-run-ds,pre-run-user} + +#conda # set a default language for localization. necessary for oci cli ARG LANG=en_US.utf8 ENV LANG=$LANG ENV SHELL=/bin/bash -############### SETUP ROOT (BASE) ENVIRONMENT WITH CONDA ###################### -ARG PYTHON_MAJOR='3' -ARG PYTHON_MINOR='8' -ARG PYTHON_PATCH='' -ARG MINICONDA_VER=py${PYTHON_MAJOR}${PYTHON_MINOR}_23.5.2-0 -ARG MAMBAFORGE_VER=23.3.1-1 - # set /opt folder permissions for $DATASCIENCE_USER. Conda is going to live in this folder. -RUN chown $DATASCIENCE_USER /opt +RUN chown -R $DATASCIENCE_USER /opt USER $DATASCIENCE_USER WORKDIR /home/datascience +RUN wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /home/datascience/miniconda.sh \ + && /bin/bash /home/datascience/miniconda.sh -f -b -p /opt/conda \ + && rm /home/datascience/miniconda.sh \ + && /opt/conda/bin/conda clean -yaf -# Note in order to run sudo commands as a non root user, you must specify --credential yes if using qemu static to build the image -RUN wget -nv https://artifactory.oci.oraclecorp.com/odsc-dev-generic-local/datascience/Miniconda3-${MINICONDA_VER}-$(uname)-$(uname -m).sh \ - -O /home/datascience/Miniconda3.sh \ - && /bin/bash /home/datascience/Miniconda3.sh -f -b -p /opt/conda \ - && rm /home/datascience/Miniconda3.sh \ - && wget -nv https://artifactory.oci.oraclecorp.com/odsc-dev-generic-local/datascience/Mambaforge-${MAMBAFORGE_VER}-$(uname)-$(uname -m).sh \ - -O /home/datascience/Mambaforge3.sh \ - && /bin/bash /home/datascience/Mambaforge3.sh -f -b -p /opt/conda \ - && rm /home/datascience/Mambaforge3.sh \ - && ls /opt/**/* - -# Adding conda to the path -ENV PATH="/opt/conda/bin:${PATH}" - -# Set activate conda by default +WORKDIR / USER root RUN printf "#!/bin/bash\nsource /opt/conda/bin/activate\n" > /etc/profile.d/enableconda.sh \ && chmod +x /etc/profile.d/enableconda.sh USER $DATASCIENCE_USER +ENV PATH="/opt/conda/bin:${PATH}" WORKDIR /home/datascience -COPY base-env-arm.yaml /opt/base-env.yaml -COPY jupyter-arm.yaml /opt/jupyter.yaml - -# Needs to be set to prevent a hang with mamba installations if this dockerfile -# is being built by running qemu-user-static to emulate an ARM machine -# -ENV G_SLICE=always-malloc - -# Replaced conda with mamba. Conda wasn't able to resolve conflicts -RUN mamba env update -n root -f /opt/jupyter.yaml && mamba clean -yaf && rm -rf /home/datascience/.cache/pip - +COPY docker/base-env.yaml /opt/base-env.yaml +RUN conda env update -q -n root -f /opt/base-env.yaml && conda clean -yaf && rm -rf /home/datascience/.cache/pip USER $DATASCIENCE_USER @@ -157,3 +96,13 @@ ARG PIP_INDEX_URL ############# Setup Conda environment tools ########################### ARG RAND=1 + +ARG RUN_WORKING_DIR="/home/datascience" +WORKDIR $RUN_WORKING_DIR + +# clean tmp folder +RUN rm -rf /tmp/* + +RUN mkdir -p /etc/datascience/operators + +USER datascience \ No newline at end of file diff --git a/ads/opctl/docker/Dockerfile_arm.job b/ads/opctl/docker/Dockerfile_arm.job new file mode 100644 index 000000000..ec9004c5e --- /dev/null +++ b/ads/opctl/docker/Dockerfile_arm.job @@ -0,0 +1,162 @@ +# Copyright (c) 2023 Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +# Used OL8 because miniconda required a higher version of glibc that was unavoidable +FROM ocr-docker-remote.artifactory.oci.oraclecorp.com/os/oraclelinux:8-slim-fips + +# Configure environment +ENV DATASCIENCE_USER datascience +ENV DATASCIENCE_UID 1000 +ENV HOME /home/$DATASCIENCE_USER +ENV DATASCIENCE_INSTALL_DIR /etc/datascience +ENV LOGS_DIRECTORY /logs + +ARG release=19 +ARG update=10 +ARG target=cpu + +# Move this arg around the file as needed when you want to use cache for local builds/testing to speed up the process +# This will allow you to set success points where you know previous code in the dockerfile is working +# ARG RAND + +COPY common /etc/datascience +COPY etc/yum.repos.d /etc/yum.repos.d + +# source: https://confluence.oraclecorp.com/confluence/pages/viewpage.action?spaceKey=ESEDoc&title=Artifactory+FAQ +ADD Symantec_Private_SSL_SHA256_CA.cer /etc/pki/ca-trust/source/anchors +ADD Symantec_Private_SSL_SHA256_Root.cer /etc/pki/ca-trust/source/anchors + +RUN update-ca-trust + +RUN microdnf install yum yum-utils && yum clean all && rm -rf /var/cache/yum + +RUN \ + yum install -y \ + oracle-release-el8 \ + oraclelinux-developer-release-el8 \ + oracle-epel-release-el8 \ + https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ + yum-config-manager --enable ol8_addons --enable ol8_oracle_instantclient > /dev/null && yum -y update && \ + yum clean all && \ + rm -rf /var/cache/yum/* + +RUN \ + yum install -y \ + --setopt=skip_missing_names_on_install=False \ + bzip2 \ + git \ + httpd \ + libcurl-devel \ + libxml2-devel \ + java-11-openjdk-headless \ + oracle-instantclient${release}.${update}-basic \ + oracle-instantclient${release}.${update}-sqlplus \ + openssl \ + openssl-devel \ + patch \ + snappy \ + gcc-c++ \ + sudo \ + unzip \ + vi \ + zip \ + wget \ + jq \ + gcc \ + htop \ + git-lfs \ + pcsc-lite-libs \ + perl-Env \ + perl-CPAN \ + perl-devel \ + curl-devel \ + expat-devel \ + asciidoc \ + xmlto \ + supervisor && \ + yum clean all && \ + rm -rf /var/cache/yum/* + +RUN \ + yum groupinstall -y -q "development tools" \ + && yum clean all \ + && rm -rf /var/cache/yum/* + +# setup user +RUN \ + mkdir -p /home/$DATASCIENCE_USER && \ + useradd -m -s /bin/bash -N -u $DATASCIENCE_UID $DATASCIENCE_USER && \ + chown $DATASCIENCE_USER /home/$DATASCIENCE_USER && \ + chown -R $DATASCIENCE_USER:users /usr/local/ && \ + touch /etc/sudoers.d/$DATASCIENCE_USER && echo "$DATASCIENCE_USER ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers.d/$DATASCIENCE_USER && \ + mkdir -p $DATASCIENCE_INSTALL_DIR && chown $DATASCIENCE_USER $DATASCIENCE_INSTALL_DIR + +# COPY build-tool-entrypoint.sh /etc/datascience/build-tool-entrypoint.sh +# COPY install-capture-entrypoint.sh /etc/datascience/install-capture-entrypoint.sh + +# set a default language for localization. necessary for oci cli +ARG LANG=en_US.utf8 +ENV LANG=$LANG +ENV SHELL=/bin/bash + +############### SETUP ROOT (BASE) ENVIRONMENT WITH CONDA ###################### +ARG PYTHON_MAJOR='3' +ARG PYTHON_MINOR='8' +ARG PYTHON_PATCH='' +ARG MINICONDA_VER=py${PYTHON_MAJOR}${PYTHON_MINOR}_23.5.2-0 +ARG MAMBAFORGE_VER=23.3.1-1 + +# set /opt folder permissions for $DATASCIENCE_USER. Conda is going to live in this folder. +RUN chown $DATASCIENCE_USER /opt + +USER $DATASCIENCE_USER +WORKDIR /home/datascience + +# Note in order to run sudo commands as a non root user, you must specify --credential yes if using qemu static to build the image +RUN wget -nv https://artifactory.oci.oraclecorp.com/odsc-dev-generic-local/datascience/Miniconda3-${MINICONDA_VER}-$(uname)-$(uname -m).sh \ + -O /home/datascience/Miniconda3.sh \ + && /bin/bash /home/datascience/Miniconda3.sh -f -b -p /opt/conda \ + && rm /home/datascience/Miniconda3.sh \ + && wget -nv https://artifactory.oci.oraclecorp.com/odsc-dev-generic-local/datascience/Mambaforge-${MAMBAFORGE_VER}-$(uname)-$(uname -m).sh \ + -O /home/datascience/Mambaforge3.sh \ + && /bin/bash /home/datascience/Mambaforge3.sh -f -b -p /opt/conda \ + && rm /home/datascience/Mambaforge3.sh \ + && ls /opt/**/* + +# Adding conda to the path +ENV PATH="/opt/conda/bin:${PATH}" + +# Set activate conda by default +USER root +RUN printf "#!/bin/bash\nsource /opt/conda/bin/activate\n" > /etc/profile.d/enableconda.sh \ + && chmod +x /etc/profile.d/enableconda.sh + +USER $DATASCIENCE_USER +WORKDIR /home/datascience + +COPY base-env-arm.yaml /opt/base-env.yaml +COPY jupyter-arm.yaml /opt/jupyter.yaml + +# Needs to be set to prevent a hang with mamba installations if this dockerfile +# is being built by running qemu-user-static to emulate an ARM machine +# +ENV G_SLICE=always-malloc + +# Replaced conda with mamba. Conda wasn't able to resolve conflicts +RUN mamba env update -n root -f /opt/jupyter.yaml && mamba clean -yaf && rm -rf /home/datascience/.cache/pip + + +USER $DATASCIENCE_USER + +####### WRAP UP ############################### +RUN python -c 'import sys; assert(sys.version_info[:2]) == (3, 8), "Python 3.8 is not detected"' +WORKDIR / + +RUN conda list + +USER root + +ARG PIP_INDEX_URL + +############# Setup Conda environment tools ########################### +ARG RAND=1 diff --git a/ads/opctl/docker/base-env-arm.yaml b/ads/opctl/docker/base-env-arm.yaml new file mode 100644 index 000000000..58dd62c47 --- /dev/null +++ b/ads/opctl/docker/base-env-arm.yaml @@ -0,0 +1,13 @@ +dependencies: + - main::ipykernel=6.15.2 + - main::ipython<=8.8.0 + - main::nb_conda_kernels=2.3.1 + - main::pip + - main::python=3.8.17 + - main::libffi=3.4.2 + - main::PyYAML=5.4.1 + - pip: + - oci + - oci-cli + - cffi>=1.15.1 + - oracledb diff --git a/ads/opctl/docker/common/install-runtime-deps.sh b/ads/opctl/docker/common/install-runtime-deps.sh new file mode 100755 index 000000000..30174b04b --- /dev/null +++ b/ads/opctl/docker/common/install-runtime-deps.sh @@ -0,0 +1,25 @@ +#!/bin/bash --login +# install reqs +cd $REPO_PATH + +USER_PIP=$(which pip) + +if [ ! -z ${YUM_REQUIREMENTS+x} ] +then + # apt install them + xargs -d '\n' -- sudo yum install -y < $YUM_REQUIREMENTS +fi + +if [ ! -z ${PYTHON_REQUIREMENTS+x} ] +then + # pip install them + $USER_PIP install -r $PYTHON_REQUIREMENTS +fi + +if [ ! -z ${CONDA_REQUIREMENTS+x} ] +then + # conda install them + conda env update -f $CONDA_REQUIREMENTS +fi + +cd /home/datascience/ diff --git a/ads/opctl/docker/jupyter-arm.yaml b/ads/opctl/docker/jupyter-arm.yaml new file mode 100644 index 000000000..bb9bac4ee --- /dev/null +++ b/ads/opctl/docker/jupyter-arm.yaml @@ -0,0 +1,19 @@ +channels: +- conda-forge +dependencies: + - conda-forge::conda-pack=0.5.0 + - main::ipykernel=6.15.2 + - main::jupyterlab=2.2.10 + - main::nb_conda_kernels=2.3.1 + - main::nodejs>=16,<=17 + - main::pip + - main::python=3.8.17 + - main::PyYAML=5.4.1 + - conda-forge::pillow=9.2.0 + - conda-forge::bokeh=2.4.3 + - conda-forge::jupyter_bokeh=2.0.4 + - pip: + - oracle-ads[opctl] + - oci # install from artifactory if possible + - oci-cli # install from artifactory if possible + - cffi>=1.15.1 \ No newline at end of file diff --git a/ads/opctl/utils.py b/ads/opctl/utils.py index 9ca13bc4c..fa3c01896 100644 --- a/ads/opctl/utils.py +++ b/ads/opctl/utils.py @@ -17,6 +17,7 @@ from subprocess import Popen, PIPE, STDOUT from typing import Union, List, Tuple, Dict import yaml +import re import ads from ads.common.oci_client import OCIClientFactory @@ -161,7 +162,14 @@ def build_image( ) proc = _build_custom_operator_image(gpu, source_folder, dst_image) else: - image, dockerfile, target = _get_image_name_dockerfile_target(image_type, gpu) + # https://stackoverflow.com/questions/66842004/get-the-processor-type-using-python-for-apple-m1-processor-gives-me-an-intel-pro + import cpuinfo + # Just get the manufacturer of the processors + manufacturer = cpuinfo.get_cpu_info().get('brand_raw') + arch = 'arm' if re.search("apple m\d ", manufacturer, re.IGNORECASE) else 'other' + print(f"The local machine's platform is {arch}.") + image, dockerfile, target = _get_image_name_dockerfile_target(image_type, gpu, arch) + print(f"dockerfile used is {dockerfile}") command = [ "docker", "build", @@ -180,21 +188,22 @@ def build_image( command += ["--build-arg", f"https_proxy={os.environ['https_proxy']}"] if os.environ.get(CONTAINER_NETWORK): command += ["--network", os.environ[CONTAINER_NETWORK]] - command += [os.path.abspath(curr_dir)] + command += [os.path.join(os.path.abspath(curr_dir), "docker")] logger.info("Build image with command %s", command) proc = run_command(command) if proc.returncode != 0: raise RuntimeError("Docker build failed.") -def _get_image_name_dockerfile_target(type: str, gpu: bool) -> str: +def _get_image_name_dockerfile_target(type: str, gpu: bool, arch: str) -> str: look_up = { - ("job-local", False): (ML_JOB_IMAGE, "Dockerfile.job", None), - ("job-local", True): (ML_JOB_GPU_IMAGE, "Dockerfile.job.gpu", None), - ("ads-ops-base", False): (OPS_IMAGE_BASE, "Dockerfile", "base"), - ("ads-ops-base", True): (OPS_IMAGE_GPU_BASE, "Dockerfile.gpu", "base"), + ("job-local", False, "arm"): (ML_JOB_IMAGE, "Dockerfile_arm.job", None), + ("job-local", False, "other"): (ML_JOB_IMAGE, "Dockerfile.job", None), + ("job-local", True, "other"): (ML_JOB_GPU_IMAGE, "Dockerfile.job.gpu", None), + ("ads-ops-base", False, "other"): (OPS_IMAGE_BASE, "Dockerfile", "base"), + ("ads-ops-base", True, "other"): (OPS_IMAGE_GPU_BASE, "Dockerfile.gpu", "base"), } - return look_up[(type, gpu)] + return look_up[(type, gpu, arch)] @runtime_dependency(module="docker", install_from=OptionalDependency.OPCTL) diff --git a/pyproject.toml b/pyproject.toml index 0f337f229..e3a7443e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,6 +123,7 @@ opctl = [ "nbconvert", "nbformat", "oci-cli", + "py-cpuinfo", ] optuna = [ "optuna==2.9.0", From c29bd5d75bfc8f72393abdbe9a0bfab061cdb3d0 Mon Sep 17 00:00:00 2001 From: Ziqun Ye Date: Mon, 23 Oct 2023 22:07:40 -0700 Subject: [PATCH 3/7] use arm base image when local machine is detected as m# machine --- ads/opctl/conda/cmds.py | 3 +- ads/opctl/conda/pack.py | 12 +- ads/opctl/docker/Dockerfile.job | 143 +++++----------- ads/opctl/docker/Dockerfile_arm.job | 162 ++++++++++++++++++ .../docker/Symantec_Private_SSL_SHA256_CA.cer | 24 +++ .../Symantec_Private_SSL_SHA256_Root.cer | 23 +++ ads/opctl/docker/base-env-arm.yaml | 13 ++ .../docker/common/install-runtime-deps.sh | 25 +++ ads/opctl/docker/etc/supervisord.conf | 37 ++++ .../docker/etc/supervisord.d/jupyterlab.ini | 17 ++ .../docker/etc/supervisord.d/jupyterlab/run | 14 ++ .../docker/etc/supervisord.d/nbruntime.ini | 17 ++ .../docker/etc/supervisord.d/nbruntime/conf | 16 ++ .../docker/etc/supervisord.d/nbruntime/run | 113 ++++++++++++ .../etc/yum.repos.d/ol7x-base-default.repo | 8 + ads/opctl/docker/jupyter-arm.yaml | 19 ++ ads/opctl/utils.py | 25 ++- pyproject.toml | 1 + 18 files changed, 560 insertions(+), 112 deletions(-) create mode 100644 ads/opctl/docker/Dockerfile_arm.job create mode 100755 ads/opctl/docker/Symantec_Private_SSL_SHA256_CA.cer create mode 100755 ads/opctl/docker/Symantec_Private_SSL_SHA256_Root.cer create mode 100644 ads/opctl/docker/base-env-arm.yaml create mode 100755 ads/opctl/docker/common/install-runtime-deps.sh create mode 100644 ads/opctl/docker/etc/supervisord.conf create mode 100644 ads/opctl/docker/etc/supervisord.d/jupyterlab.ini create mode 100644 ads/opctl/docker/etc/supervisord.d/jupyterlab/run create mode 100644 ads/opctl/docker/etc/supervisord.d/nbruntime.ini create mode 100644 ads/opctl/docker/etc/supervisord.d/nbruntime/conf create mode 100644 ads/opctl/docker/etc/supervisord.d/nbruntime/run create mode 100644 ads/opctl/docker/etc/yum.repos.d/ol7x-base-default.repo create mode 100644 ads/opctl/docker/jupyter-arm.yaml diff --git a/ads/opctl/conda/cmds.py b/ads/opctl/conda/cmds.py index e002336ca..3fd3459c9 100644 --- a/ads/opctl/conda/cmds.py +++ b/ads/opctl/conda/cmds.py @@ -602,7 +602,8 @@ def _publish( ) except Exception: raise RuntimeError(f"Could not pack environment {conda_slug}.") - + if "/" in conda_slug: + raise ValueError("Invalid conda_slug. found `/` in slug name. Please use a different slug name.") pack_file = os.path.join(pack_folder_path, f"{conda_slug}.tar.gz") if not os.path.exists(pack_file): raise RuntimeError(f"Pack {pack_file} was not created.") diff --git a/ads/opctl/conda/pack.py b/ads/opctl/conda/pack.py index bad61cb99..68b66dc69 100644 --- a/ads/opctl/conda/pack.py +++ b/ads/opctl/conda/pack.py @@ -31,12 +31,12 @@ def main(pack_folder_path): stderr=subprocess.PIPE, ) stdout, stderr = process.communicate() - # import pdb; pdb.set_trace() - # if stderr: - # print(stderr) - # raise Exception( - # f"Error export environment information from {pack_folder_path}" - # ) + + if process.returncode and stderr: + print(stderr) + raise Exception( + f"Error export environment information from {pack_folder_path}" + ) try: new_env_info = yaml.safe_load(StringIO(stdout.decode("utf-8"))) except Exception as e: diff --git a/ads/opctl/docker/Dockerfile.job b/ads/opctl/docker/Dockerfile.job index d9bbe6829..c2eb3fa03 100644 --- a/ads/opctl/docker/Dockerfile.job +++ b/ads/opctl/docker/Dockerfile.job @@ -1,5 +1,7 @@ -# Used OL8 because miniconda required a higher version of glibc that was unavoidable -FROM ocr-docker-remote.artifactory.oci.oraclecorp.com/os/oraclelinux:8-slim-fips +# Copyright (c) 2021, 2023 Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +FROM ghcr.io/oracle/oraclelinux:7-slim # Configure environment ENV DATASCIENCE_USER datascience @@ -9,139 +11,76 @@ ENV DATASCIENCE_INSTALL_DIR /etc/datascience ENV LOGS_DIRECTORY /logs ARG release=19 -ARG update=10 -ARG target=cpu - -# Move this arg around the file as needed when you want to use cache for local builds/testing to speed up the process -# This will allow you to set success points where you know previous code in the dockerfile is working -# ARG RAND - -COPY common /etc/datascience -COPY etc/yum.repos.d /etc/yum.repos.d - -# source: https://confluence.oraclecorp.com/confluence/pages/viewpage.action?spaceKey=ESEDoc&title=Artifactory+FAQ -ADD Symantec_Private_SSL_SHA256_CA.cer /etc/pki/ca-trust/source/anchors -ADD Symantec_Private_SSL_SHA256_Root.cer /etc/pki/ca-trust/source/anchors - -RUN update-ca-trust - -RUN microdnf install yum yum-utils && yum clean all && rm -rf /var/cache/yum +ARG update=13 RUN \ - yum install -y \ - oracle-release-el8 \ - oraclelinux-developer-release-el8 \ - oracle-epel-release-el8 \ - https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ - yum-config-manager --enable ol8_addons --enable ol8_oracle_instantclient > /dev/null && yum -y update && \ - yum clean all && \ - rm -rf /var/cache/yum/* - -RUN \ - yum install -y \ - --setopt=skip_missing_names_on_install=False \ + yum -y -q install oracle-release-el7 && \ + yum-config-manager --add-repo http://yum.oracle.com/repo/OracleLinux/OL7/developer_EPEL/x86_64 && \ + yum-config-manager --enable ol7_optional_latest --enable ol7_addons --enable ol7_software_collections --enable ol7_oracle_instantclient > /dev/null && \ + yum groupinstall -y -q 'Development Tools' && \ + yum update -y && \ + yum install -y --setopt=skip_missing_names_on_install=False \ bzip2 \ + curl \ git \ - httpd \ + gcc-gfortran \ libcurl-devel \ libxml2-devel \ - java-11-openjdk-headless \ oracle-instantclient${release}.${update}-basic \ oracle-instantclient${release}.${update}-sqlplus \ openssl \ openssl-devel \ patch \ - snappy \ - gcc-c++ \ sudo \ unzip \ - vi \ zip \ + gcc-c++ \ wget \ - jq \ gcc \ - htop \ - git-lfs \ - pcsc-lite-libs \ - perl-Env \ - perl-CPAN \ - perl-devel \ - curl-devel \ - expat-devel \ - asciidoc \ - xmlto \ - supervisor && \ - yum clean all && \ - rm -rf /var/cache/yum/* - -RUN \ - yum groupinstall -y -q "development tools" \ && yum clean all \ && rm -rf /var/cache/yum/* # setup user RUN \ - mkdir -p /home/$DATASCIENCE_USER && \ - useradd -m -s /bin/bash -N -u $DATASCIENCE_UID $DATASCIENCE_USER && \ - chown $DATASCIENCE_USER /home/$DATASCIENCE_USER && \ - chown -R $DATASCIENCE_USER:users /usr/local/ && \ - touch /etc/sudoers.d/$DATASCIENCE_USER && echo "$DATASCIENCE_USER ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers.d/$DATASCIENCE_USER && \ - mkdir -p $DATASCIENCE_INSTALL_DIR && chown $DATASCIENCE_USER $DATASCIENCE_INSTALL_DIR - -# COPY build-tool-entrypoint.sh /etc/datascience/build-tool-entrypoint.sh -# COPY install-capture-entrypoint.sh /etc/datascience/install-capture-entrypoint.sh - + mkdir -p /home/$DATASCIENCE_USER && \ + useradd -m -s /bin/bash -N -u $DATASCIENCE_UID $DATASCIENCE_USER && \ + chown -R $DATASCIENCE_USER /home/$DATASCIENCE_USER && \ + chown -R $DATASCIENCE_USER:users /usr/local/ && \ + touch /etc/sudoers.d/$DATASCIENCE_USER && echo "$DATASCIENCE_USER ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers.d/$DATASCIENCE_USER && \ + mkdir -p $DATASCIENCE_INSTALL_DIR && chown $DATASCIENCE_USER $DATASCIENCE_INSTALL_DIR && \ + mkdir -p $LOGS_DIRECTORY && chown -R $DATASCIENCE_USER:users $LOGS_DIRECTORY && \ + mkdir -p $LOGS_DIRECTORY/harness && chown -R $DATASCIENCE_USER:users $LOGS_DIRECTORY/harness + +RUN mkdir -p /etc/datascience/build +RUN mkdir -p $DATASCIENCE_INSTALL_DIR/{pre-build-ds,post-build-ds,pre-run-ds,pre-run-user} + +#conda # set a default language for localization. necessary for oci cli ARG LANG=en_US.utf8 ENV LANG=$LANG ENV SHELL=/bin/bash -############### SETUP ROOT (BASE) ENVIRONMENT WITH CONDA ###################### -ARG PYTHON_MAJOR='3' -ARG PYTHON_MINOR='8' -ARG PYTHON_PATCH='' -ARG MINICONDA_VER=py${PYTHON_MAJOR}${PYTHON_MINOR}_23.5.2-0 -ARG MAMBAFORGE_VER=23.3.1-1 - # set /opt folder permissions for $DATASCIENCE_USER. Conda is going to live in this folder. -RUN chown $DATASCIENCE_USER /opt +RUN chown -R $DATASCIENCE_USER /opt USER $DATASCIENCE_USER WORKDIR /home/datascience +RUN wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /home/datascience/miniconda.sh \ + && /bin/bash /home/datascience/miniconda.sh -f -b -p /opt/conda \ + && rm /home/datascience/miniconda.sh \ + && /opt/conda/bin/conda clean -yaf -# Note in order to run sudo commands as a non root user, you must specify --credential yes if using qemu static to build the image -RUN wget -nv https://artifactory.oci.oraclecorp.com/odsc-dev-generic-local/datascience/Miniconda3-${MINICONDA_VER}-$(uname)-$(uname -m).sh \ - -O /home/datascience/Miniconda3.sh \ - && /bin/bash /home/datascience/Miniconda3.sh -f -b -p /opt/conda \ - && rm /home/datascience/Miniconda3.sh \ - && wget -nv https://artifactory.oci.oraclecorp.com/odsc-dev-generic-local/datascience/Mambaforge-${MAMBAFORGE_VER}-$(uname)-$(uname -m).sh \ - -O /home/datascience/Mambaforge3.sh \ - && /bin/bash /home/datascience/Mambaforge3.sh -f -b -p /opt/conda \ - && rm /home/datascience/Mambaforge3.sh \ - && ls /opt/**/* - -# Adding conda to the path -ENV PATH="/opt/conda/bin:${PATH}" - -# Set activate conda by default +WORKDIR / USER root RUN printf "#!/bin/bash\nsource /opt/conda/bin/activate\n" > /etc/profile.d/enableconda.sh \ && chmod +x /etc/profile.d/enableconda.sh USER $DATASCIENCE_USER +ENV PATH="/opt/conda/bin:${PATH}" WORKDIR /home/datascience -COPY base-env-arm.yaml /opt/base-env.yaml -COPY jupyter-arm.yaml /opt/jupyter.yaml - -# Needs to be set to prevent a hang with mamba installations if this dockerfile -# is being built by running qemu-user-static to emulate an ARM machine -# -ENV G_SLICE=always-malloc - -# Replaced conda with mamba. Conda wasn't able to resolve conflicts -RUN mamba env update -n root -f /opt/jupyter.yaml && mamba clean -yaf && rm -rf /home/datascience/.cache/pip - +COPY docker/base-env.yaml /opt/base-env.yaml +RUN conda env update -q -n root -f /opt/base-env.yaml && conda clean -yaf && rm -rf /home/datascience/.cache/pip USER $DATASCIENCE_USER @@ -157,3 +96,13 @@ ARG PIP_INDEX_URL ############# Setup Conda environment tools ########################### ARG RAND=1 + +ARG RUN_WORKING_DIR="/home/datascience" +WORKDIR $RUN_WORKING_DIR + +# clean tmp folder +RUN rm -rf /tmp/* + +RUN mkdir -p /etc/datascience/operators + +USER datascience \ No newline at end of file diff --git a/ads/opctl/docker/Dockerfile_arm.job b/ads/opctl/docker/Dockerfile_arm.job new file mode 100644 index 000000000..ec9004c5e --- /dev/null +++ b/ads/opctl/docker/Dockerfile_arm.job @@ -0,0 +1,162 @@ +# Copyright (c) 2023 Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +# Used OL8 because miniconda required a higher version of glibc that was unavoidable +FROM ocr-docker-remote.artifactory.oci.oraclecorp.com/os/oraclelinux:8-slim-fips + +# Configure environment +ENV DATASCIENCE_USER datascience +ENV DATASCIENCE_UID 1000 +ENV HOME /home/$DATASCIENCE_USER +ENV DATASCIENCE_INSTALL_DIR /etc/datascience +ENV LOGS_DIRECTORY /logs + +ARG release=19 +ARG update=10 +ARG target=cpu + +# Move this arg around the file as needed when you want to use cache for local builds/testing to speed up the process +# This will allow you to set success points where you know previous code in the dockerfile is working +# ARG RAND + +COPY common /etc/datascience +COPY etc/yum.repos.d /etc/yum.repos.d + +# source: https://confluence.oraclecorp.com/confluence/pages/viewpage.action?spaceKey=ESEDoc&title=Artifactory+FAQ +ADD Symantec_Private_SSL_SHA256_CA.cer /etc/pki/ca-trust/source/anchors +ADD Symantec_Private_SSL_SHA256_Root.cer /etc/pki/ca-trust/source/anchors + +RUN update-ca-trust + +RUN microdnf install yum yum-utils && yum clean all && rm -rf /var/cache/yum + +RUN \ + yum install -y \ + oracle-release-el8 \ + oraclelinux-developer-release-el8 \ + oracle-epel-release-el8 \ + https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ + yum-config-manager --enable ol8_addons --enable ol8_oracle_instantclient > /dev/null && yum -y update && \ + yum clean all && \ + rm -rf /var/cache/yum/* + +RUN \ + yum install -y \ + --setopt=skip_missing_names_on_install=False \ + bzip2 \ + git \ + httpd \ + libcurl-devel \ + libxml2-devel \ + java-11-openjdk-headless \ + oracle-instantclient${release}.${update}-basic \ + oracle-instantclient${release}.${update}-sqlplus \ + openssl \ + openssl-devel \ + patch \ + snappy \ + gcc-c++ \ + sudo \ + unzip \ + vi \ + zip \ + wget \ + jq \ + gcc \ + htop \ + git-lfs \ + pcsc-lite-libs \ + perl-Env \ + perl-CPAN \ + perl-devel \ + curl-devel \ + expat-devel \ + asciidoc \ + xmlto \ + supervisor && \ + yum clean all && \ + rm -rf /var/cache/yum/* + +RUN \ + yum groupinstall -y -q "development tools" \ + && yum clean all \ + && rm -rf /var/cache/yum/* + +# setup user +RUN \ + mkdir -p /home/$DATASCIENCE_USER && \ + useradd -m -s /bin/bash -N -u $DATASCIENCE_UID $DATASCIENCE_USER && \ + chown $DATASCIENCE_USER /home/$DATASCIENCE_USER && \ + chown -R $DATASCIENCE_USER:users /usr/local/ && \ + touch /etc/sudoers.d/$DATASCIENCE_USER && echo "$DATASCIENCE_USER ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers.d/$DATASCIENCE_USER && \ + mkdir -p $DATASCIENCE_INSTALL_DIR && chown $DATASCIENCE_USER $DATASCIENCE_INSTALL_DIR + +# COPY build-tool-entrypoint.sh /etc/datascience/build-tool-entrypoint.sh +# COPY install-capture-entrypoint.sh /etc/datascience/install-capture-entrypoint.sh + +# set a default language for localization. necessary for oci cli +ARG LANG=en_US.utf8 +ENV LANG=$LANG +ENV SHELL=/bin/bash + +############### SETUP ROOT (BASE) ENVIRONMENT WITH CONDA ###################### +ARG PYTHON_MAJOR='3' +ARG PYTHON_MINOR='8' +ARG PYTHON_PATCH='' +ARG MINICONDA_VER=py${PYTHON_MAJOR}${PYTHON_MINOR}_23.5.2-0 +ARG MAMBAFORGE_VER=23.3.1-1 + +# set /opt folder permissions for $DATASCIENCE_USER. Conda is going to live in this folder. +RUN chown $DATASCIENCE_USER /opt + +USER $DATASCIENCE_USER +WORKDIR /home/datascience + +# Note in order to run sudo commands as a non root user, you must specify --credential yes if using qemu static to build the image +RUN wget -nv https://artifactory.oci.oraclecorp.com/odsc-dev-generic-local/datascience/Miniconda3-${MINICONDA_VER}-$(uname)-$(uname -m).sh \ + -O /home/datascience/Miniconda3.sh \ + && /bin/bash /home/datascience/Miniconda3.sh -f -b -p /opt/conda \ + && rm /home/datascience/Miniconda3.sh \ + && wget -nv https://artifactory.oci.oraclecorp.com/odsc-dev-generic-local/datascience/Mambaforge-${MAMBAFORGE_VER}-$(uname)-$(uname -m).sh \ + -O /home/datascience/Mambaforge3.sh \ + && /bin/bash /home/datascience/Mambaforge3.sh -f -b -p /opt/conda \ + && rm /home/datascience/Mambaforge3.sh \ + && ls /opt/**/* + +# Adding conda to the path +ENV PATH="/opt/conda/bin:${PATH}" + +# Set activate conda by default +USER root +RUN printf "#!/bin/bash\nsource /opt/conda/bin/activate\n" > /etc/profile.d/enableconda.sh \ + && chmod +x /etc/profile.d/enableconda.sh + +USER $DATASCIENCE_USER +WORKDIR /home/datascience + +COPY base-env-arm.yaml /opt/base-env.yaml +COPY jupyter-arm.yaml /opt/jupyter.yaml + +# Needs to be set to prevent a hang with mamba installations if this dockerfile +# is being built by running qemu-user-static to emulate an ARM machine +# +ENV G_SLICE=always-malloc + +# Replaced conda with mamba. Conda wasn't able to resolve conflicts +RUN mamba env update -n root -f /opt/jupyter.yaml && mamba clean -yaf && rm -rf /home/datascience/.cache/pip + + +USER $DATASCIENCE_USER + +####### WRAP UP ############################### +RUN python -c 'import sys; assert(sys.version_info[:2]) == (3, 8), "Python 3.8 is not detected"' +WORKDIR / + +RUN conda list + +USER root + +ARG PIP_INDEX_URL + +############# Setup Conda environment tools ########################### +ARG RAND=1 diff --git a/ads/opctl/docker/Symantec_Private_SSL_SHA256_CA.cer b/ads/opctl/docker/Symantec_Private_SSL_SHA256_CA.cer new file mode 100755 index 000000000..7597ceb7f --- /dev/null +++ b/ads/opctl/docker/Symantec_Private_SSL_SHA256_CA.cer @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIIEGDCCAwCgAwIBAgIQPqGO3UAruXNc+m3tNaUO2TANBgkqhkiG9w0BAQsFADBl +MQswCQYDVQQGEwJVUzEdMBsGA1UEChMUU3ltYW50ZWMgQ29ycG9yYXRpb24xNzA1 +BgNVBAMTLlN5bWFudGVjIFRydXN0IFNlcnZpY2VzIFByaXZhdGUgU0hBMjU2IFJv +b3QgQ0EwHhcNMTQwNjEzMDAwMDAwWhcNMjQwNjEyMjM1OTU5WjBVMQswCQYDVQQG +EwJVUzEdMBsGA1UEChMUU3ltYW50ZWMgQ29ycG9yYXRpb24xJzAlBgNVBAMTHlN5 +bWFudGVjIFByaXZhdGUgU1NMIFNIQTI1NiBDQTCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBAJDQExo+maVP+rm3NO6FQg/Vrgd+ipzfgmvlXfa118kDuhrD +uYl5b+1sJVWdosBicoBAHCKWDxyJ2yEDcIoX4m2w11bQ4s/TrpnEJdF3RxhfiLSR +H1fTEi1jxIgqPdnEr3r4PcHHluYYQ+Xf3RscpwHmLM7unZx7jv3kIKCb/5MKpWQz +vSA0zAsS+Hq3OniL2BhSg57qb2wc3duQkHUSLMTK9hD4Y9b5bmFx1G+oYf8I2B/O +EqhhU7dwUqbZEGRSGPVa51CMwUD5MsydR6BeGyHHsfWwK7b6DqXmmuawgt00r09p +vRQ0a/1Vzw6JphofgpNu5ShCi8PkQ8GqDYqahFkCAwEAAaOB0zCB0DASBgNVHRMB +Af8ECDAGAQH/AgEAMA4GA1UdDwEB/wQEAwIBBjA/BgNVHR8EODA2MDSgMqAwhi5o +dHRwOi8vcy5zeW1jYi5jb20vdHJ1c3RzZXJ2aWNlc3NoYTI1NnJvb3QuY3JsMCkG +A1UdEQQiMCCkHjAcMRowGAYDVQQDExFTeW1hbnRlY1BLSS0xLTcwNTAdBgNVHQ4E +FgQUbmgiNqSsgdJMqOH4Gq/I/ruWnGwwHwYDVR0jBBgwFoAUCVLSAP0XLXXGd3EV +oei3kdFPGVIwDQYJKoZIhvcNAQELBQADggEBAKqkuviIecTZ4GUBCRtwXxcMV0UE +I+vi2CBmbOBpMdUv9UuC8OFl1pZlODm8XN09hZn3NVNytQP70fIvNLB4VRsik/Rl ++ncUeoSusGSxjbaYoxaHNU1mAbj4XCteEL/e6e+LnVzKIdhBGDF5tFUqhr5J7Zid +CreVSSRfz4qvpMdBRYiP1EnKCf6lILf10JBQ1C383m/OOjdlBqDx61tusTNT9NW8 +hO9rWVwT75hMLf5UKQyPLUQVMj7EnKqzkcwUtMfR9AealkBmNKLgDRXI86j1EGBJ +s+L7czprGyRW5PLqtLLC8GEQaDTax+EHS3O9Z1zT8fj8F9tzzJNm2s1DtsE= +-----END CERTIFICATE----- diff --git a/ads/opctl/docker/Symantec_Private_SSL_SHA256_Root.cer b/ads/opctl/docker/Symantec_Private_SSL_SHA256_Root.cer new file mode 100755 index 000000000..404336924 --- /dev/null +++ b/ads/opctl/docker/Symantec_Private_SSL_SHA256_Root.cer @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIDvzCCAqegAwIBAgIQFyg3mT6+zOG+taeI9QWQvjANBgkqhkiG9w0BAQsFADBl +MQswCQYDVQQGEwJVUzEdMBsGA1UEChMUU3ltYW50ZWMgQ29ycG9yYXRpb24xNzA1 +BgNVBAMTLlN5bWFudGVjIFRydXN0IFNlcnZpY2VzIFByaXZhdGUgU0hBMjU2IFJv +b3QgQ0EwHhcNMTQwNjEzMDAwMDAwWhcNNDQwNjEyMjM1OTU5WjBlMQswCQYDVQQG +EwJVUzEdMBsGA1UEChMUU3ltYW50ZWMgQ29ycG9yYXRpb24xNzA1BgNVBAMTLlN5 +bWFudGVjIFRydXN0IFNlcnZpY2VzIFByaXZhdGUgU0hBMjU2IFJvb3QgQ0EwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDe4QajAWFjd5si27U0NFpYRRTy +9PGAeHwDFJ+377QfFt9JQpX853latmNBKUq6SSqQKCZvje18mk7++R6tsd4gfSSG ++p16O1WIMc27FukqWtB8DXWHZ7in4HAWIB80kW/WUR8QKhmIbZEVJnGXYly+yCy/ +ngbJ5ZGD/p1QYZ5eLAgevkS4sXlT/BVtu75t6F6UbzAavy8WIc9DxTFObyWw4txz +GNiiNVrNez0CN7EXpFF+ho1T78gNk6UV21CiYPO7JjdOc/HW5Cibba7Qdtq7pSa5 +4gt/72M7DV17Co2tTZRAl9hhhxzXuOmvvu8Wf7qugv2W+WEZ37VqWnYYkRvHAgMB +AAGjazBpMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMCcGA1UdEQQg +MB6kHDAaMRgwFgYDVQQDEw9NUEtJLTIwNDgtMS0xNTcwHQYDVR0OBBYEFAlS0gD9 +Fy11xndxFaHot5HRTxlSMA0GCSqGSIb3DQEBCwUAA4IBAQCjXpkp7aCHaJIlYCEh +ofs+o38auCuWp/XRGaCHm82ogEweuOndPELkK9fsdca704mXK4PZ5B/6BJKMh0aJ +AnIDrS7Q/iOcyXEx/v1uNqaYEUVxLJa4nGGg7d6e+HZH0vas24HdPo28UjGCifHX +iSE9ZmFBGa9w5H7lgbChki2SE5ilR2m28wFko+v3U9u0tipdhDrBq5j5LSGpI75D +Vbj9BGXgo7AJ9+s0uW2AP3WNiFg//E6gGI1bcq7Vu4I6l7ixDTsJugdFoQ65pBXs +a8oVB4RdlEXPzmpsekwcRmA5dTD1+TH6DiAJ9z5n1mn262W+rQayLGH73/arMrQT +jylR +-----END CERTIFICATE----- diff --git a/ads/opctl/docker/base-env-arm.yaml b/ads/opctl/docker/base-env-arm.yaml new file mode 100644 index 000000000..58dd62c47 --- /dev/null +++ b/ads/opctl/docker/base-env-arm.yaml @@ -0,0 +1,13 @@ +dependencies: + - main::ipykernel=6.15.2 + - main::ipython<=8.8.0 + - main::nb_conda_kernels=2.3.1 + - main::pip + - main::python=3.8.17 + - main::libffi=3.4.2 + - main::PyYAML=5.4.1 + - pip: + - oci + - oci-cli + - cffi>=1.15.1 + - oracledb diff --git a/ads/opctl/docker/common/install-runtime-deps.sh b/ads/opctl/docker/common/install-runtime-deps.sh new file mode 100755 index 000000000..30174b04b --- /dev/null +++ b/ads/opctl/docker/common/install-runtime-deps.sh @@ -0,0 +1,25 @@ +#!/bin/bash --login +# install reqs +cd $REPO_PATH + +USER_PIP=$(which pip) + +if [ ! -z ${YUM_REQUIREMENTS+x} ] +then + # apt install them + xargs -d '\n' -- sudo yum install -y < $YUM_REQUIREMENTS +fi + +if [ ! -z ${PYTHON_REQUIREMENTS+x} ] +then + # pip install them + $USER_PIP install -r $PYTHON_REQUIREMENTS +fi + +if [ ! -z ${CONDA_REQUIREMENTS+x} ] +then + # conda install them + conda env update -f $CONDA_REQUIREMENTS +fi + +cd /home/datascience/ diff --git a/ads/opctl/docker/etc/supervisord.conf b/ads/opctl/docker/etc/supervisord.conf new file mode 100644 index 000000000..62c0c426e --- /dev/null +++ b/ads/opctl/docker/etc/supervisord.conf @@ -0,0 +1,37 @@ +[unix_http_server] +file=/run/supervisor/supervisor.sock ; (the path to the socket file) +chmod=0750 ; sockef file mode (default 0700) +chown=1000:100 ; socket file uid:gid owner + +[supervisord] +logfile=%(ENV_JUPYTER_SERVER_LOG_DIR)s/%(ENV_SUPERVISORD_LOG_FILE)s ; (main log file;default $CWD/supervisord.log) +logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB) +logfile_backups=2 ; (num of main logfile rotation backups;default 10) +loglevel=info ; (log level;default info; others: debug,warn,trace) +pidfile=/run/supervisor/supervisord.pid ; (supervisord pidfile;default supervisord.pid) +nodaemon=true ; (start in foreground if true;default false) +minfds=1024 ; (min. avail startup file descriptors;default 1024) +minprocs=200 ; (min. avail process descriptors;default 200) +;umask=022 ; (process file creation umask;default 022) +;identifier=supervisor ; (supervisord identifier, default is 'supervisor') +;directory=/tmp ; (default is not to cd during start) +;nocleanup=true ; (don't clean up tempfiles at start;default false) +;childlogdir=/tmp ; ('AUTO' child log dir, default $TEMP) +;environment=KEY=value ; (key value pairs to add to environment) +;strip_ansi=false ; (strip ansi escape codes in logs; def. false) +user=datascience +;environment= + +; the below section must remain in the config file for RPC +; (supervisorctl/web interface) to work, additional interfaces may be +; added by defining them in separate rpcinterface: sections +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///run/supervisor/supervisor.sock ; use a unix:// URL for a unix socket +;prompt=mysupervisor ; cmd line prompt (default "supervisor") +;history_file=~/.sc_history ; use readline history if available + +[include] +files = supervisord.d/*.ini \ No newline at end of file diff --git a/ads/opctl/docker/etc/supervisord.d/jupyterlab.ini b/ads/opctl/docker/etc/supervisord.d/jupyterlab.ini new file mode 100644 index 000000000..87b43269f --- /dev/null +++ b/ads/opctl/docker/etc/supervisord.d/jupyterlab.ini @@ -0,0 +1,17 @@ +[program:jupyterlab] +directory=/home/datascience +command=/etc/supervisord.d/jupyterlab/run +autostart=true +autorestart=true +startretries=5 +numprocs=1 +startsecs=3 +process_name=%(program_name)s_%(process_num)02d +stdout_logfile=%(ENV_JUPYTER_SERVER_LOG_DIR)s/%(ENV_JUPYTER_SERVER_LOG_FILE)s +stdout_logfile_maxbytes=20MB +stderr_logfile_maxbytes=20MB +stdout_logfile_backups=10 +stderr_logfile_backups=10 +stopsignal=TERM +stopwaitsecs=60 +redirect_stderr=true \ No newline at end of file diff --git a/ads/opctl/docker/etc/supervisord.d/jupyterlab/run b/ads/opctl/docker/etc/supervisord.d/jupyterlab/run new file mode 100644 index 000000000..fb427de23 --- /dev/null +++ b/ads/opctl/docker/etc/supervisord.d/jupyterlab/run @@ -0,0 +1,14 @@ +#!/bin/bash + +for file in $DATASCIENCE_INSTALL_DIR/pre-run-ds/*.sh; do + [ -f "$file" ] && [ -x "$file" ] && "$file" +done + +for file in $DATASCIENCE_INSTALL_DIR/pre-run-user/*.sh; do + [ -f "$file" ] && [ -x "$file" ] && "$file" +done + +JUPYTER_STOP_COMMAND="jupyter notebook stop" + +echo "Jupyter Lab command: $JUPYTER_EXEC_COMMAND" +exec $JUPYTER_EXEC_COMMAND \ No newline at end of file diff --git a/ads/opctl/docker/etc/supervisord.d/nbruntime.ini b/ads/opctl/docker/etc/supervisord.d/nbruntime.ini new file mode 100644 index 000000000..79c5a8c72 --- /dev/null +++ b/ads/opctl/docker/etc/supervisord.d/nbruntime.ini @@ -0,0 +1,17 @@ +[program:nbruntime] +directory=/home/datascience +command=/bin/bash -c "exec /etc/supervisord.d/nbruntime/run" +autostart=true +autorestart=true +startretries=5 +numprocs=1 +startsecs=3 +process_name=%(ENV_NBRUNTIME_SERVICE_NAME)s +stdout_logfile=%(ENV_JUPYTER_SERVER_LOG_DIR)s/%(ENV_NBRUNTIME_SERVICE_LOG_FILE)s +stdout_logfile_maxbytes=20MB +stderr_logfile_maxbytes=20MB +stdout_logfile_backups=10 +stderr_logfile_backups=10 +stopsignal=TERM +stopwaitsecs=%(ENV_MAX_SCRIPT_EXECUTION_SECONDS)s +redirect_stderr=true \ No newline at end of file diff --git a/ads/opctl/docker/etc/supervisord.d/nbruntime/conf b/ads/opctl/docker/etc/supervisord.d/nbruntime/conf new file mode 100644 index 000000000..64015a55d --- /dev/null +++ b/ads/opctl/docker/etc/supervisord.d/nbruntime/conf @@ -0,0 +1,16 @@ +#VM-AGENT can override all the values +USER_ACTION="${CUSTOMER_ACTION}" +CREATE_NOTEBOOK_ACTION="createNotebook" +ACTIVATE_NOTEBOOK_ACTION="activateNotebook" + +SCRIPTS_DIR="${NB_LIFECYCLE_SCRIPTS_DIR:-/home/datascience/.scripts}" +SCRIPT_EXECUTION_DIR="${NB_LIFECYCLE_SCRIPTS_EXEC_DIR:-/home/datascience}" +NB_LFS_ONACTIVATE_TIMEOUT_S="${NB_LFS_ONACTIVATE_TIMEOUT_S:-900}" #Default:15mins +NB_LFS_ONDEACTIVATE_TIMEOUT_S="${NB_LFS_ONDEACTIVATE_TIMEOUT_S:-900}" #Default:15mins +NB_LFS_ONDELETE_TIMEOUT_S="${NB_LFS_ONDELETE_TIMEOUT_S:-900}" #Default:15mins +NB_LFS_ONCREATE_TIMEOUT_S="${NB_LFS_ONCREATE_TIMEOUT_S:-900}" #Default:15mins + +CREATE_SCRIPT_FILE_NAME="${NB_ONCREATE_FILE_NAME:-NB_ONCREATE_ENTRYPOINT}" +ACTIVATE_SCRIPT_FILE_NAME="${NB_ONACTIVATE_FILE_NAME:-NB_ONACTIVATE_ENTRYPOINT}" +DEACTIVATE_SCRIPT_FILE_NAME="${NB_ONDEACTIVATE_FILE_NAME:-NB_ONDEACTIVATE_ENTRYPOINT}" +DELETE_SCRIPT_FILE_NAME="${NB_ONDELETE_FILE_NAME:-NB_ONDELETE_ENTRYPOINT}" diff --git a/ads/opctl/docker/etc/supervisord.d/nbruntime/run b/ads/opctl/docker/etc/supervisord.d/nbruntime/run new file mode 100644 index 000000000..eec5f80ed --- /dev/null +++ b/ads/opctl/docker/etc/supervisord.d/nbruntime/run @@ -0,0 +1,113 @@ +#!/bin/bash -e +exec 2>&1 + +# conf file +SCRIPT_DIR="$( dirname -- "$0"; )" +[ -r $SCRIPT_DIR/conf ] && . $SCRIPT_DIR/conf + +NOTEBOOK_STOP_ACTION="delete" + +trap execute_container_onstop_script SIGTERM +trap rundeactivatescript SIGUSR1 #Deactivate flag +trap rundeletescript SIGUSR2 #Delete flag + +execute_container_onstop_script() { + echo "Going to check and run the script on container stop...." + file_to_search="" + script_execution_timeout_seconds="" + + if [[ $NOTEBOOK_STOP_ACTION == "deactivate" ]] + then + echo "Deactivate Notebook Action......." + file_to_search=$DEACTIVATE_SCRIPT_FILE_NAME + script_execution_timeout_seconds=$NB_LFS_ONDEACTIVATE_TIMEOUT_S + elif [[ $NOTEBOOK_STOP_ACTION == "delete" ]] + then + echo "Delete Notebook Action......." + file_to_search=$DELETE_SCRIPT_FILE_NAME + script_execution_timeout_seconds=$NB_LFS_ONDELETE_TIMEOUT_S + else + echo "Invalid function argument." + exit 1 + fi + + echo "Checking if the script is present" + + [[ -z "$file_to_search" ]] && echo "Unrecognized user action or file name is missing. exit...." && exit 1 + + script_file=$(find "$SCRIPTS_DIR" -maxdepth 1 -iname "$file_to_search*" -type f) + [[ ! -n "$script_file" ]] && echo "Find command return empty list of files. exit...." && exit 1 + + file_count=`echo $script_file | wc -l` + [[ "$file_count" != 1 ]] && echo "More than 1 script file found. exit...." && exit 1 + + echo "Executing the script..$script_file" + exec odsc-notebook execute-lifecycle-script -f $script_file -d $SCRIPTS_DIR -e $SCRIPT_EXECUTION_DIR -t $script_execution_timeout_seconds + + echo "Exiting after file execution is finished.." + exit 0 +} + +execute_container_onstart_script() { + echo "Going to check and run the script on container start...." + [[ -z "$USER_ACTION" ]] && echo "No user action provided." && return + + echo "Checking if the script is present" + + file_to_search="" + script_execution_timeout_seconds="" + if [[ "$USER_ACTION" == "$CREATE_NOTEBOOK_ACTION" ]]; then + echo "Create Notebook Action..." + file_to_search=$CREATE_SCRIPT_FILE_NAME + script_execution_timeout_seconds=$NB_LFS_ONCREATE_TIMEOUT_S + elif [[ "$USER_ACTION" == "$ACTIVATE_NOTEBOOK_ACTION" ]]; then + echo "Activate Notebook Action..." + file_to_search=$ACTIVATE_SCRIPT_FILE_NAME + script_execution_timeout_seconds=$NB_LFS_ONACTIVATE_TIMEOUT_S + fi + + [[ -z "$file_to_search" ]] && echo "Unrecognized user action or file name is missing. exit...." && return + + script_file=$(find "$SCRIPTS_DIR" -maxdepth 1 -iname "$file_to_search*" -type f) + [[ ! -n "$script_file" ]] && echo "Find command return empty list of files. exit...." && return + + file_count=`echo $script_file | wc -l` + [[ "$file_count" != 1 ]] && echo "More than 1 script file found. exit...." && return + + #Execute script + echo "executing the script..$script_file" + exec odsc-notebook execute-lifecycle-script -f $script_file -d $SCRIPTS_DIR -e $SCRIPT_EXECUTION_DIR -t $script_execution_timeout_seconds & +} + +rundeactivatescript() { + echo "Received USR1, setting the stop action to deactivate." + NOTEBOOK_STOP_ACTION="deactivate" +} + +rundeletescript() { + echo "Received USR2, setting the stop action to delete." + NOTEBOOK_STOP_ACTION="delete" +} + +# enable conda commands +source /etc/profile.d/enableconda.sh + +#activating runtimeconfig conda +conda activate $OCI_INTERNAL_DS_RUNTIME_CONFIG_ENV_NAME + +# Start git cloning in background +set +H +exec \ +odsc-notebook runtime-config process_git_configuration \ +-f $OCI_INTERNAL_RUNTIME_CONFIG_FILE_PATH \ +-d $OCI_INTERNAL_DS_REPO_HOME || \ +echo "Runtime Config process git settings exit status: $?" & + +# Start the on start script +execute_container_onstart_script || echo "Container onstart script execution exit with status code: $?" & + +echo "Starting the while loop with sleep 1....." +while [[ true ]] +do + sleep 1 +done \ No newline at end of file diff --git a/ads/opctl/docker/etc/yum.repos.d/ol7x-base-default.repo b/ads/opctl/docker/etc/yum.repos.d/ol7x-base-default.repo new file mode 100644 index 000000000..58307a375 --- /dev/null +++ b/ads/opctl/docker/etc/yum.repos.d/ol7x-base-default.repo @@ -0,0 +1,8 @@ +[ol7-yum] +name=ol7-yum +baseurl=https://artifactory.oci.oraclecorp.com/ol7-yum/ +enabled=1 +gpgcheck=0 +repo_gpgcheck=1 +gpgkey=https://artifactory.oci.oraclecorp.com/ol7-yum/repodata/repomd.xml.key +assumeyes=1 \ No newline at end of file diff --git a/ads/opctl/docker/jupyter-arm.yaml b/ads/opctl/docker/jupyter-arm.yaml new file mode 100644 index 000000000..bb9bac4ee --- /dev/null +++ b/ads/opctl/docker/jupyter-arm.yaml @@ -0,0 +1,19 @@ +channels: +- conda-forge +dependencies: + - conda-forge::conda-pack=0.5.0 + - main::ipykernel=6.15.2 + - main::jupyterlab=2.2.10 + - main::nb_conda_kernels=2.3.1 + - main::nodejs>=16,<=17 + - main::pip + - main::python=3.8.17 + - main::PyYAML=5.4.1 + - conda-forge::pillow=9.2.0 + - conda-forge::bokeh=2.4.3 + - conda-forge::jupyter_bokeh=2.0.4 + - pip: + - oracle-ads[opctl] + - oci # install from artifactory if possible + - oci-cli # install from artifactory if possible + - cffi>=1.15.1 \ No newline at end of file diff --git a/ads/opctl/utils.py b/ads/opctl/utils.py index 9ca13bc4c..fa3c01896 100644 --- a/ads/opctl/utils.py +++ b/ads/opctl/utils.py @@ -17,6 +17,7 @@ from subprocess import Popen, PIPE, STDOUT from typing import Union, List, Tuple, Dict import yaml +import re import ads from ads.common.oci_client import OCIClientFactory @@ -161,7 +162,14 @@ def build_image( ) proc = _build_custom_operator_image(gpu, source_folder, dst_image) else: - image, dockerfile, target = _get_image_name_dockerfile_target(image_type, gpu) + # https://stackoverflow.com/questions/66842004/get-the-processor-type-using-python-for-apple-m1-processor-gives-me-an-intel-pro + import cpuinfo + # Just get the manufacturer of the processors + manufacturer = cpuinfo.get_cpu_info().get('brand_raw') + arch = 'arm' if re.search("apple m\d ", manufacturer, re.IGNORECASE) else 'other' + print(f"The local machine's platform is {arch}.") + image, dockerfile, target = _get_image_name_dockerfile_target(image_type, gpu, arch) + print(f"dockerfile used is {dockerfile}") command = [ "docker", "build", @@ -180,21 +188,22 @@ def build_image( command += ["--build-arg", f"https_proxy={os.environ['https_proxy']}"] if os.environ.get(CONTAINER_NETWORK): command += ["--network", os.environ[CONTAINER_NETWORK]] - command += [os.path.abspath(curr_dir)] + command += [os.path.join(os.path.abspath(curr_dir), "docker")] logger.info("Build image with command %s", command) proc = run_command(command) if proc.returncode != 0: raise RuntimeError("Docker build failed.") -def _get_image_name_dockerfile_target(type: str, gpu: bool) -> str: +def _get_image_name_dockerfile_target(type: str, gpu: bool, arch: str) -> str: look_up = { - ("job-local", False): (ML_JOB_IMAGE, "Dockerfile.job", None), - ("job-local", True): (ML_JOB_GPU_IMAGE, "Dockerfile.job.gpu", None), - ("ads-ops-base", False): (OPS_IMAGE_BASE, "Dockerfile", "base"), - ("ads-ops-base", True): (OPS_IMAGE_GPU_BASE, "Dockerfile.gpu", "base"), + ("job-local", False, "arm"): (ML_JOB_IMAGE, "Dockerfile_arm.job", None), + ("job-local", False, "other"): (ML_JOB_IMAGE, "Dockerfile.job", None), + ("job-local", True, "other"): (ML_JOB_GPU_IMAGE, "Dockerfile.job.gpu", None), + ("ads-ops-base", False, "other"): (OPS_IMAGE_BASE, "Dockerfile", "base"), + ("ads-ops-base", True, "other"): (OPS_IMAGE_GPU_BASE, "Dockerfile.gpu", "base"), } - return look_up[(type, gpu)] + return look_up[(type, gpu, arch)] @runtime_dependency(module="docker", install_from=OptionalDependency.OPCTL) diff --git a/pyproject.toml b/pyproject.toml index 0f337f229..e3a7443e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,6 +123,7 @@ opctl = [ "nbconvert", "nbformat", "oci-cli", + "py-cpuinfo", ] optuna = [ "optuna==2.9.0", From 8535ef1c6f3f77d86864f33b58d49b1bd3d709e0 Mon Sep 17 00:00:00 2001 From: Ziqun Ye Date: Fri, 27 Oct 2023 15:16:20 -0700 Subject: [PATCH 4/7] revert back the path for docker build --- ads/opctl/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ads/opctl/utils.py b/ads/opctl/utils.py index fa1123fe4..ee26c9f19 100644 --- a/ads/opctl/utils.py +++ b/ads/opctl/utils.py @@ -188,7 +188,7 @@ def build_image( command += ["--build-arg", f"https_proxy={os.environ['https_proxy']}"] if os.environ.get(CONTAINER_NETWORK): command += ["--network", os.environ[CONTAINER_NETWORK]] - command += [os.path.join(os.path.abspath(curr_dir), "docker")] + command += [os.path.abspath(curr_dir)] logger.info("Build image with command %s", command) proc = run_command(command) if proc.returncode != 0: From e16021d813800aac48e68889b831b121ef447ea2 Mon Sep 17 00:00:00 2001 From: Ziqun Ye Date: Thu, 2 Nov 2023 14:18:31 -0700 Subject: [PATCH 5/7] update the code based on comments --- ads/opctl/conda/cmds.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ads/opctl/conda/cmds.py b/ads/opctl/conda/cmds.py index 0b0f7bd29..ec63e1a30 100644 --- a/ads/opctl/conda/cmds.py +++ b/ads/opctl/conda/cmds.py @@ -638,8 +638,10 @@ def _publish( ) except Exception: raise RuntimeError(f"Could not pack environment {conda_slug}.") - if "/" in conda_slug: - raise ValueError("Invalid conda_slug. found `/` in slug name. Please use a different slug name.") + NOT_ALLOWED_CHARS = "@#$%^&*/" + + if any(chr in conda_slug for chr in NOT_ALLOWED_CHARS): + raise ValueError(f"Invalid conda_slug. Found {NOT_ALLOWED_CHARS} in slug name. Please use a different slug name.") pack_file = os.path.join(pack_folder_path, f"{conda_slug}.tar.gz") if not os.path.exists(pack_file): raise RuntimeError(f"Pack {pack_file} was not created.") From 5bc1dc9e49c4b94de04ae69e11af45161d605ba2 Mon Sep 17 00:00:00 2001 From: Ziqun Ye Date: Mon, 13 Nov 2023 10:03:22 -0800 Subject: [PATCH 6/7] fix the unit test --- tests/unitary/with_extras/opctl/test_opctl_conda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unitary/with_extras/opctl/test_opctl_conda.py b/tests/unitary/with_extras/opctl/test_opctl_conda.py index b5472a74c..6b1a30a17 100644 --- a/tests/unitary/with_extras/opctl/test_opctl_conda.py +++ b/tests/unitary/with_extras/opctl/test_opctl_conda.py @@ -161,7 +161,7 @@ def test_conda_publish( ): {"bind": "/home/datascience/pack.py"}, }, env_vars={}, - command="python /home/datascience/pack.py /home/datascience/test", + command="python /home/datascience/pack.py --conda-path /home/datascience/test", ) mock_uploader.assert_called_with( From 8a57a5dae4ef3349b7a1e4bb39e1c458f041647f Mon Sep 17 00:00:00 2001 From: Ziqun Ye Date: Tue, 14 Nov 2023 11:48:43 -0800 Subject: [PATCH 7/7] adding license info for py-cpuinfo --- THIRD_PARTY_LICENSES.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/THIRD_PARTY_LICENSES.txt b/THIRD_PARTY_LICENSES.txt index a6b0aab38..b11d0b777 100644 --- a/THIRD_PARTY_LICENSES.txt +++ b/THIRD_PARTY_LICENSES.txt @@ -241,6 +241,12 @@ psutil * Source code: https://github.com/giampaolo/psutil * Project home: https://github.com/giampaolo/psutil +py-cpuinfo +* Copyright (c) 2014-2022 Matthew Brennan Jones +* License: The MIT License (MIT) +* Source code: https://github.com/workhorsy/py-cpuinfo +* Project home: https://github.com/workhorsy/py-cpuinfo + pyspark * Copyright 2014 and onwards The Apache Software Foundation. * License: Apache-2.0 LICENSE