diff --git a/THIRD_PARTY_LICENSES.txt b/THIRD_PARTY_LICENSES.txt index a6b0aab38..b11d0b777 100644 --- a/THIRD_PARTY_LICENSES.txt +++ b/THIRD_PARTY_LICENSES.txt @@ -241,6 +241,12 @@ psutil * Source code: https://github.com/giampaolo/psutil * Project home: https://github.com/giampaolo/psutil +py-cpuinfo +* Copyright (c) 2014-2022 Matthew Brennan Jones +* License: The MIT License (MIT) +* Source code: https://github.com/workhorsy/py-cpuinfo +* Project home: https://github.com/workhorsy/py-cpuinfo + pyspark * Copyright 2014 and onwards The Apache Software Foundation. * License: Apache-2.0 LICENSE diff --git a/ads/opctl/conda/cmds.py b/ads/opctl/conda/cmds.py index cacc764b0..ec63e1a30 100644 --- a/ads/opctl/conda/cmds.py +++ b/ads/opctl/conda/cmds.py @@ -638,7 +638,10 @@ def _publish( ) except Exception: raise RuntimeError(f"Could not pack environment {conda_slug}.") + NOT_ALLOWED_CHARS = "@#$%^&*/" + if any(chr in conda_slug for chr in NOT_ALLOWED_CHARS): + raise ValueError(f"Invalid conda_slug. Found {NOT_ALLOWED_CHARS} in slug name. Please use a different slug name.") pack_file = os.path.join(pack_folder_path, f"{conda_slug}.tar.gz") if not os.path.exists(pack_file): raise RuntimeError(f"Pack {pack_file} was not created.") diff --git a/ads/opctl/conda/pack.py b/ads/opctl/conda/pack.py index bcaeeff93..805ed1e96 100644 --- a/ads/opctl/conda/pack.py +++ b/ads/opctl/conda/pack.py @@ -26,7 +26,7 @@ def main(pack_folder_path, manifest_file=None): ) with open(manifest_path) as f: env = yaml.safe_load(f.read()) - + with tempfile.TemporaryDirectory() as td: process = subprocess.Popen( ["conda", "env", "export", "--prefix", pack_folder_path], @@ -34,7 +34,8 @@ def main(pack_folder_path, manifest_file=None): stderr=subprocess.PIPE, ) stdout, stderr = process.communicate() - if stderr: + + if process.returncode and stderr: print(stderr) raise Exception( f"Error export environment information from {pack_folder_path}" diff --git a/ads/opctl/docker/Dockerfile.job b/ads/opctl/docker/Dockerfile.job index 3bc1f2482..55eb60baf 100644 --- a/ads/opctl/docker/Dockerfile.job +++ b/ads/opctl/docker/Dockerfile.job @@ -100,4 +100,4 @@ RUN rm -rf /tmp/* RUN mkdir -p /etc/datascience/operators -USER datascience +USER datascience \ No newline at end of file diff --git a/ads/opctl/utils.py b/ads/opctl/utils.py index 1ae64a6e1..54ffd9d93 100644 --- a/ads/opctl/utils.py +++ b/ads/opctl/utils.py @@ -17,6 +17,7 @@ from subprocess import Popen, PIPE, STDOUT from typing import Union, List, Tuple, Dict import yaml +import re import ads from ads.common.oci_client import OCIClientFactory @@ -160,7 +161,14 @@ def build_image( ) proc = _build_custom_operator_image(gpu, source_folder, dst_image) else: - image, dockerfile, target = _get_image_name_dockerfile_target(image_type, gpu) + # https://stackoverflow.com/questions/66842004/get-the-processor-type-using-python-for-apple-m1-processor-gives-me-an-intel-pro + import cpuinfo + # Just get the manufacturer of the processors + manufacturer = cpuinfo.get_cpu_info().get('brand_raw') + arch = 'arm' if re.search("apple m\d ", manufacturer, re.IGNORECASE) else 'other' + print(f"The local machine's platform is {arch}.") + image, dockerfile, target = _get_image_name_dockerfile_target(image_type, gpu, arch) + print(f"dockerfile used is {dockerfile}") command = [ "docker", "build", @@ -186,14 +194,15 @@ def build_image( raise RuntimeError("Docker build failed.") -def _get_image_name_dockerfile_target(type: str, gpu: bool) -> str: +def _get_image_name_dockerfile_target(type: str, gpu: bool, arch: str) -> str: look_up = { - ("job-local", False): (ML_JOB_IMAGE, "Dockerfile.job", None), - ("job-local", True): (ML_JOB_GPU_IMAGE, "Dockerfile.job.gpu", None), - ("ads-ops-base", False): (OPS_IMAGE_BASE, "Dockerfile", "base"), - ("ads-ops-base", True): (OPS_IMAGE_GPU_BASE, "Dockerfile.gpu", "base"), + ("job-local", False, "arm"): (ML_JOB_IMAGE, "Dockerfile.job.arm", None), + ("job-local", False, "other"): (ML_JOB_IMAGE, "Dockerfile.job", None), + ("job-local", True, "other"): (ML_JOB_GPU_IMAGE, "Dockerfile.job.gpu", None), + ("ads-ops-base", False, "other"): (OPS_IMAGE_BASE, "Dockerfile", "base"), + ("ads-ops-base", True, "other"): (OPS_IMAGE_GPU_BASE, "Dockerfile.gpu", "base"), } - return look_up[(type, gpu)] + return look_up[(type, gpu, arch)] @runtime_dependency(module="docker", install_from=OptionalDependency.OPCTL) diff --git a/pyproject.toml b/pyproject.toml index 175953c4f..a21bd4bee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,6 +123,7 @@ opctl = [ "nbconvert", "nbformat", "oci-cli", + "py-cpuinfo", ] optuna = [ "optuna==2.9.0",