diff --git a/CHANGELOG.md b/CHANGELOG.md index 95e4a7b9cf..b8b3155231 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,33 @@ # Changelog +## v2.119.0 (2022-12-03) + +### Features + + * Add Code Owners file + * Added transform with monitoring pipeline step in transformer + * Update TF 2.9 and TF 2.10 inference DLCs + * make estimator accept json file as modelparallel config + * SageMaker Training Compiler does not support p4de instances + * Add support for SparkML v3.3 + +### Bug Fixes and Other Changes + + * Fix bug forcing uploaded tar to be named sourcedir + * Update local_requirements.txt PyYAML version + * refactoring : using with statement + * Allow Py 3.7 for MMS Test Docker env + * fix PySparkProcessor __init__ params type + * type hint of PySparkProcessor __init__ + * Return ARM XGB/SKLearn tags if `image_scope` is `inference_graviton` + * Update scipy to 1.7.3 to support M1 development envs + * Fixing type hints for Spark processor that has instance type/count params in reverse order + * Add DeepAR ap-northeast-3 repository. + * Fix AsyncInferenceConfig documentation typo + * fix ml_inf to ml_inf1 in Neo multi-version support + * Fix type annotations + * add neo mvp region accounts + ## v2.118.0 (2022-12-01) ### Features diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000000..7f7ac28644 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @aws/sagemaker-ml-frameworks diff --git a/VERSION b/VERSION index 34d47b7f52..dda4128cf2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.118.1.dev0 +2.119.1.dev0 diff --git a/src/sagemaker/git_utils.py b/src/sagemaker/git_utils.py index 80bd62d5be..c424753286 100644 --- a/src/sagemaker/git_utils.py +++ b/src/sagemaker/git_utils.py @@ -279,9 +279,8 @@ def _run_clone_command(repo_url, dest_dir): subprocess.check_call(["git", "clone", repo_url, dest_dir], env=my_env) elif repo_url.startswith("git@"): with tempfile.NamedTemporaryFile() as sshnoprompt: - write_pipe = open(sshnoprompt.name, "w") - write_pipe.write("ssh -oBatchMode=yes $@") - write_pipe.close() + with open(sshnoprompt.name, "w") as write_pipe: + write_pipe.write("ssh -oBatchMode=yes $@") os.chmod(sshnoprompt.name, 0o511) my_env["GIT_SSH"] = sshnoprompt.name subprocess.check_call(["git", "clone", repo_url, dest_dir], env=my_env) diff --git a/src/sagemaker/image_uri_config/autogluon.json b/src/sagemaker/image_uri_config/autogluon.json index 3cc488c55d..3a9f02142c 100644 --- a/src/sagemaker/image_uri_config/autogluon.json +++ b/src/sagemaker/image_uri_config/autogluon.json @@ -26,6 +26,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -56,6 +57,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -86,6 +88,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -116,6 +119,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -146,6 +150,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -176,6 +181,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -204,6 +210,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -211,12 +218,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -237,6 +247,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -244,12 +255,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -270,6 +284,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -277,12 +292,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -303,6 +321,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -310,12 +329,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -336,6 +358,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -343,12 +366,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -369,6 +395,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -376,12 +403,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", diff --git a/src/sagemaker/image_uri_config/huggingface-neuron.json b/src/sagemaker/image_uri_config/huggingface-neuron.json index 1e2246cb11..47d6dbd1dc 100644 --- a/src/sagemaker/image_uri_config/huggingface-neuron.json +++ b/src/sagemaker/image_uri_config/huggingface-neuron.json @@ -15,17 +15,20 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ca-central-1": "763104351884", "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", "sa-east-1": "763104351884", "us-east-1": "763104351884", diff --git a/src/sagemaker/image_uri_config/huggingface-training-compiler.json b/src/sagemaker/image_uri_config/huggingface-training-compiler.json index e771e2a548..482264b773 100644 --- a/src/sagemaker/image_uri_config/huggingface-training-compiler.json +++ b/src/sagemaker/image_uri_config/huggingface-training-compiler.json @@ -60,6 +60,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -89,6 +90,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -123,6 +125,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", diff --git a/src/sagemaker/image_uri_config/huggingface.json b/src/sagemaker/image_uri_config/huggingface.json index 317c17030a..5b98fc0d02 100644 --- a/src/sagemaker/image_uri_config/huggingface.json +++ b/src/sagemaker/image_uri_config/huggingface.json @@ -38,6 +38,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -70,6 +71,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -108,6 +110,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -140,6 +143,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -180,6 +184,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -213,6 +218,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -246,6 +252,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -279,6 +286,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -320,6 +328,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -353,6 +362,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -386,6 +396,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -419,6 +430,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -458,6 +470,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -491,6 +504,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -530,6 +544,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -563,6 +578,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -602,6 +618,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -635,6 +652,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -674,6 +692,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -681,12 +700,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -707,6 +729,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -714,12 +737,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -740,6 +766,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -747,12 +774,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -781,6 +811,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -788,12 +819,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -814,6 +848,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -821,12 +856,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -847,6 +885,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -854,12 +893,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -880,6 +922,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -887,12 +930,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -919,6 +965,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -926,12 +973,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -952,6 +1002,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -959,12 +1010,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -991,6 +1045,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -998,12 +1053,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1024,6 +1082,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -1031,12 +1090,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1063,6 +1125,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -1070,12 +1133,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1096,6 +1162,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -1103,12 +1170,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", diff --git a/src/sagemaker/image_uri_config/mxnet.json b/src/sagemaker/image_uri_config/mxnet.json index 12bc40fccf..8d8733e480 100644 --- a/src/sagemaker/image_uri_config/mxnet.json +++ b/src/sagemaker/image_uri_config/mxnet.json @@ -245,6 +245,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -277,6 +278,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -309,6 +311,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -341,6 +344,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -373,6 +377,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -619,6 +624,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -626,12 +632,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -651,6 +660,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -658,12 +668,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -683,6 +696,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -690,12 +704,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -715,6 +732,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -722,12 +740,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -747,6 +768,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -754,12 +776,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -852,6 +877,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -859,12 +885,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -884,6 +913,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -891,12 +921,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -916,6 +949,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -923,12 +957,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", diff --git a/src/sagemaker/image_uri_config/pytorch.json b/src/sagemaker/image_uri_config/pytorch.json index 3bf8016ba8..18a382e591 100644 --- a/src/sagemaker/image_uri_config/pytorch.json +++ b/src/sagemaker/image_uri_config/pytorch.json @@ -17,6 +17,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -25,7 +26,9 @@ "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", "eu-north-1": "763104351884", + "eu-central-2": "380420809688", "eu-west-1": "763104351884", + "eu-south-2": "503227376785", "us-east-1": "763104351884", "us-east-2": "763104351884", "us-west-2": "763104351884" @@ -39,8 +42,11 @@ "registries": { "ap-northeast-1": "763104351884", "ap-northeast-2": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-3": "907027046896", + "eu-central-2": "380420809688", "eu-west-1": "763104351884", + "eu-south-2": "503227376785", "us-east-1": "763104351884", "us-east-2": "763104351884", "us-west-2": "763104351884" @@ -182,6 +188,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -189,12 +196,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -217,6 +227,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -224,12 +235,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -251,6 +265,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -258,12 +273,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -285,6 +303,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -292,12 +311,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -320,6 +342,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -327,12 +350,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -355,6 +381,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -362,12 +389,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -390,6 +420,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -397,12 +428,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -425,6 +459,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -432,12 +467,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -459,6 +497,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -466,12 +505,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -493,6 +535,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -500,12 +543,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -527,6 +573,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -534,12 +581,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -561,6 +611,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -568,12 +619,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -595,6 +649,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -602,12 +657,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -629,6 +687,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -636,12 +695,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -663,6 +725,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -670,11 +733,13 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", "sa-east-1": "763104351884", "us-east-1": "763104351884", @@ -707,6 +772,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -714,11 +780,13 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", "sa-east-1": "763104351884", "us-east-1": "763104351884", @@ -879,6 +947,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -914,6 +983,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -949,6 +1019,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -983,6 +1054,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1018,6 +1090,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1053,6 +1126,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1088,6 +1162,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1123,6 +1198,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1157,6 +1233,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1191,6 +1268,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1225,6 +1303,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1259,6 +1338,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1293,6 +1373,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1327,6 +1408,7 @@ "eu-west-3": "763104351884", "eu-south-1": "692866216735", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", diff --git a/src/sagemaker/image_uri_config/tensorflow.json b/src/sagemaker/image_uri_config/tensorflow.json index 6a01c3e3e6..a0f2bba014 100644 --- a/src/sagemaker/image_uri_config/tensorflow.json +++ b/src/sagemaker/image_uri_config/tensorflow.json @@ -141,6 +141,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -148,12 +149,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "eu-south-2": "503227376785", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -172,6 +176,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -179,12 +184,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -203,6 +211,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -210,12 +219,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -234,6 +246,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -241,12 +254,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -285,7 +301,9 @@ "2.5": "2.5.1", "2.6": "2.6.3", "2.7": "2.7.0", - "2.8": "2.8.0" + "2.8": "2.8.0", + "2.9": "2.9.2", + "2.10": "2.10.0" }, "versions": { "1.10.0": { @@ -386,6 +404,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -393,12 +412,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -417,6 +439,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -424,12 +447,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -448,6 +474,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -455,12 +482,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -479,6 +509,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -486,12 +517,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -510,6 +544,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -517,12 +552,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -541,6 +579,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -548,12 +587,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -572,6 +614,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -579,12 +622,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -795,6 +841,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -802,12 +849,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -826,6 +876,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -833,12 +884,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -857,6 +911,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -864,12 +919,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -888,6 +946,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -895,12 +954,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -919,6 +981,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -926,12 +989,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -950,6 +1016,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -957,12 +1024,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -981,6 +1051,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -988,12 +1059,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1012,6 +1086,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -1019,12 +1094,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1043,6 +1121,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -1050,12 +1129,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1074,6 +1156,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -1081,12 +1164,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1105,6 +1191,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -1112,12 +1199,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1136,6 +1226,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -1143,12 +1234,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1167,6 +1261,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -1174,12 +1269,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1198,6 +1296,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -1205,12 +1304,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1229,6 +1331,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -1236,12 +1339,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1260,6 +1366,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -1267,12 +1374,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1291,6 +1401,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -1298,12 +1409,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1322,6 +1436,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -1329,12 +1444,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1353,6 +1471,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -1360,12 +1479,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1384,6 +1506,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -1391,12 +1514,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1415,6 +1541,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -1422,12 +1549,15 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1439,6 +1569,72 @@ "repository": "tensorflow-inference" }, "2.8.0": { + "registries": { + "af-south-1": "626614931356", + "ap-east-1": "871362719292", + "ap-northeast-1": "763104351884", + "ap-northeast-2": "763104351884", + "ap-northeast-3": "364406365360", + "ap-south-1": "763104351884", + "ap-south-2": "772153158452", + "ap-southeast-1": "763104351884", + "ap-southeast-2": "763104351884", + "ap-southeast-3": "907027046896", + "ca-central-1": "763104351884", + "cn-north-1": "727897471807", + "cn-northwest-1": "727897471807", + "eu-central-1": "763104351884", + "eu-central-2": "380420809688", + "eu-north-1": "763104351884", + "eu-south-1": "692866216735", + "eu-south-2": "503227376785", + "eu-west-1": "763104351884", + "eu-west-2": "763104351884", + "eu-west-3": "763104351884", + "me-south-1": "217643126080", + "me-central-1": "914824155844", + "sa-east-1": "763104351884", + "us-east-1": "763104351884", + "us-east-2": "763104351884", + "us-gov-west-1": "442386744353", + "us-iso-east-1": "886529160074", + "us-west-1": "763104351884", + "us-west-2": "763104351884" + }, + "repository": "tensorflow-inference" + }, + "2.9.2": { + "registries": { + "af-south-1": "626614931356", + "ap-east-1": "871362719292", + "ap-northeast-1": "763104351884", + "ap-northeast-2": "763104351884", + "ap-northeast-3": "364406365360", + "ap-south-1": "763104351884", + "ap-southeast-1": "763104351884", + "ap-southeast-2": "763104351884", + "ap-southeast-3": "907027046896", + "ca-central-1": "763104351884", + "cn-north-1": "727897471807", + "cn-northwest-1": "727897471807", + "eu-central-1": "763104351884", + "eu-north-1": "763104351884", + "eu-south-1": "692866216735", + "eu-west-1": "763104351884", + "eu-west-2": "763104351884", + "eu-west-3": "763104351884", + "me-south-1": "217643126080", + "sa-east-1": "763104351884", + "us-east-1": "763104351884", + "us-east-2": "763104351884", + "us-gov-west-1": "442386744353", + "us-iso-east-1": "886529160074", + "us-west-1": "763104351884", + "us-west-2": "763104351884" + }, + "repository": "tensorflow-inference" + }, + "2.10.0": { "registries": { "af-south-1": "626614931356", "ap-east-1": "871362719292", @@ -1490,6 +1686,7 @@ "ap-northeast-2": "763104351884", "ap-northeast-3": "364406365360", "ap-south-1": "763104351884", + "ap-south-2": "772153158452", "ap-southeast-1": "763104351884", "ap-southeast-2": "763104351884", "ap-southeast-3": "907027046896", @@ -1497,11 +1694,13 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", "eu-central-1": "763104351884", + "eu-central-2": "380420809688", "eu-north-1": "763104351884", "eu-west-1": "763104351884", "eu-west-2": "763104351884", "eu-west-3": "763104351884", "eu-south-1": "692866216735", + "eu-south-2": "503227376785", "me-south-1": "217643126080", "sa-east-1": "763104351884", "us-east-1": "763104351884", @@ -1696,6 +1895,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1732,6 +1932,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1767,6 +1968,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1803,6 +2005,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1839,6 +2042,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1875,6 +2079,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -1911,6 +2116,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2138,6 +2344,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2173,6 +2380,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2208,6 +2416,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2242,6 +2451,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2276,6 +2486,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2311,6 +2522,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2346,6 +2558,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2380,6 +2593,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2414,6 +2628,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2448,6 +2663,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2482,6 +2698,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2516,6 +2733,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2550,6 +2768,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2584,6 +2803,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2618,6 +2838,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2652,6 +2873,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2686,6 +2908,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2720,6 +2943,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2754,6 +2978,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2788,6 +3013,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2822,6 +3048,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2856,6 +3083,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2890,6 +3118,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2924,6 +3153,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", @@ -2958,6 +3188,7 @@ "eu-west-2": "763104351884", "eu-west-3": "763104351884", "me-south-1": "217643126080", + "me-central-1": "914824155844", "sa-east-1": "763104351884", "us-east-1": "763104351884", "us-east-2": "763104351884", diff --git a/src/sagemaker/processing.py b/src/sagemaker/processing.py index db6ce2badd..81e3d34b1d 100644 --- a/src/sagemaker/processing.py +++ b/src/sagemaker/processing.py @@ -23,6 +23,7 @@ import logging from textwrap import dedent from typing import Dict, List, Optional, Union +from copy import copy import attr @@ -1587,13 +1588,13 @@ def run( # type: ignore[override] framework script to run.Path (absolute or relative) to the local Python source file which should be executed as the entry point to training. When `code` is an S3 URI, ignore `source_dir`, - `dependencies, and `git_config`. If ``source_dir`` is specified, + `dependencies`, and `git_config`. If ``source_dir`` is specified, then ``code`` must point to a file located at the root of ``source_dir``. source_dir (str): Path (absolute, relative or an S3 URI) to a directory with any other processing source code dependencies aside from the entry point file (default: None). If ``source_dir`` is an S3 URI, it must - point to a tar.gz file. Structure within this directory are preserved - when processing on Amazon SageMaker (default: None). + point to a file named `sourcedir.tar.gz`. Structure within this directory + are preserved when processing on Amazon SageMaker (default: None). dependencies (list[str]): A list of paths to directories (absolute or relative) with any additional libraries that will be exported to the container (default: []). The library folders will be @@ -1730,12 +1731,15 @@ def _pack_and_upload_code( "sagemaker_session unspecified when creating your Processor to have one set up " "automatically." ) + if "/sourcedir.tar.gz" in estimator.uploaded_code.s3_prefix: + # Upload the bootstrapping code as s3://.../jobname/source/runproc.sh. + entrypoint_s3_uri = estimator.uploaded_code.s3_prefix.replace( + "sourcedir.tar.gz", + "runproc.sh", + ) + else: + raise RuntimeError("S3 source_dir file must be named `sourcedir.tar.gz.`") - # Upload the bootstrapping code as s3://.../jobname/source/runproc.sh. - entrypoint_s3_uri = estimator.uploaded_code.s3_prefix.replace( - "sourcedir.tar.gz", - "runproc.sh", - ) script = estimator.uploaded_code.script_name s3_runproc_sh = S3Uploader.upload_string_as_file_body( self._generate_framework_script(script), @@ -1827,14 +1831,17 @@ def _patch_inputs_with_payload(self, inputs, s3_payload) -> List[ProcessingInput # a7399455f5386d83ddc5cb15c0db00c04bd518ec/src/sagemaker/processing.py#L425-L426 if inputs is None: inputs = [] - inputs.append( + + # make a shallow copy of user inputs + patched_inputs = copy(inputs) + patched_inputs.append( ProcessingInput( input_name="code", source=s3_payload, destination="/opt/ml/processing/input/code/", ) ) - return inputs + return patched_inputs def _set_entrypoint(self, command, user_script_name): """Framework processor override for setting processing job entrypoint. diff --git a/src/sagemaker/spark/processing.py b/src/sagemaker/spark/processing.py index dc3d26a355..912bc90d80 100644 --- a/src/sagemaker/spark/processing.py +++ b/src/sagemaker/spark/processing.py @@ -30,6 +30,7 @@ from enum import Enum from io import BytesIO from urllib.parse import urlparse +from copy import copy from typing import Union, List, Dict, Optional @@ -279,6 +280,10 @@ def run( def _extend_processing_args(self, inputs, outputs, **kwargs): """Extends processing job args such as inputs.""" + # make a shallow copy of user outputs + outputs = outputs or [] + extended_outputs = copy(outputs) + if kwargs.get("spark_event_logs_s3_uri"): spark_event_logs_s3_uri = kwargs.get("spark_event_logs_s3_uri") self._validate_s3_uri(spark_event_logs_s3_uri) @@ -297,16 +302,21 @@ def _extend_processing_args(self, inputs, outputs, **kwargs): s3_upload_mode="Continuous", ) - outputs = outputs or [] - outputs.append(output) + extended_outputs.append(output) + + # make a shallow copy of user inputs + inputs = inputs or [] + extended_inputs = copy(inputs) if kwargs.get("configuration"): configuration = kwargs.get("configuration") self._validate_configuration(configuration) - inputs = inputs or [] - inputs.append(self._stage_configuration(configuration)) + extended_inputs.append(self._stage_configuration(configuration)) - return inputs, outputs + return ( + extended_inputs if extended_inputs else None, + extended_outputs if extended_outputs else None, + ) def start_history_server(self, spark_event_logs_s3_uri=None): """Starts a Spark history server. @@ -940,9 +950,16 @@ def _extend_processing_args(self, inputs, outputs, **kwargs): outputs: Processing outputs. kwargs: Additional keyword arguments passed to `super()`. """ + + if inputs is None: + inputs = [] + + # make a shallow copy of user inputs + extended_inputs = copy(inputs) + self.command = [_SparkProcessorBase._default_command] extended_inputs = self._handle_script_dependencies( - inputs, kwargs.get("submit_py_files"), FileType.PYTHON + extended_inputs, kwargs.get("submit_py_files"), FileType.PYTHON ) extended_inputs = self._handle_script_dependencies( extended_inputs, kwargs.get("submit_jars"), FileType.JAR @@ -1199,8 +1216,14 @@ def _extend_processing_args(self, inputs, outputs, **kwargs): else: raise ValueError("submit_class is required") + if inputs is None: + inputs = [] + + # make a shallow copy of user inputs + extended_inputs = copy(inputs) + extended_inputs = self._handle_script_dependencies( - inputs, kwargs.get("submit_jars"), FileType.JAR + extended_inputs, kwargs.get("submit_jars"), FileType.JAR ) extended_inputs = self._handle_script_dependencies( extended_inputs, kwargs.get("submit_files"), FileType.FILE diff --git a/src/sagemaker/transformer.py b/src/sagemaker/transformer.py index cfcc637b99..97278abdd0 100644 --- a/src/sagemaker/transformer.py +++ b/src/sagemaker/transformer.py @@ -14,14 +14,17 @@ from __future__ import absolute_import from typing import Union, Optional, List, Dict -from botocore import exceptions +import logging +import copy +import time +from botocore import exceptions from sagemaker.job import _Job -from sagemaker.session import Session +from sagemaker.session import Session, get_execution_role from sagemaker.inputs import BatchDataCaptureConfig from sagemaker.workflow.entities import PipelineVariable from sagemaker.workflow.functions import Join -from sagemaker.workflow.pipeline_context import runnable_by_pipeline +from sagemaker.workflow.pipeline_context import runnable_by_pipeline, PipelineSession from sagemaker.workflow import is_pipeline_variable from sagemaker.workflow.execution_variables import ExecutionVariables from sagemaker.utils import base_name_from_image, name_from_base @@ -266,6 +269,155 @@ def transform( if wait: self.latest_transform_job.wait(logs=logs) + def transform_with_monitoring( + self, + monitoring_config, + monitoring_resource_config, + data: str, + data_type: str = "S3Prefix", + content_type: str = None, + compression_type: str = None, + split_type: str = None, + input_filter: str = None, + output_filter: str = None, + join_source: str = None, + model_client_config: Dict[str, str] = None, + batch_data_capture_config: BatchDataCaptureConfig = None, + monitor_before_transform: bool = False, + supplied_baseline_statistics: str = None, + supplied_baseline_constraints: str = None, + wait: bool = True, + pipeline_name: str = None, + role: str = None, + ): + """Runs a transform job with monitoring job. + + Note that this function will not start a transform job immediately, + instead, it will create a SageMaker Pipeline and execute it. + If you provide an existing pipeline_name, no new pipeline will be created, otherwise, + each transform_with_monitoring call will create a new pipeline and execute. + + Args: + monitoring_config (Union[ + `sagemaker.workflow.quality_check_step.QualityCheckConfig`, + `sagemaker.workflow.quality_check_step.ClarifyCheckConfig` + ]): the monitoring configuration used for run model monitoring. + monitoring_resource_config (`sagemaker.workflow.check_job_config.CheckJobConfig`): + the check job (processing job) cluster resource configuration. + transform_step_args (_JobStepArguments): the transform step transform arguments. + data (str): Input data location in S3 for the transform job + data_type (str): What the S3 location defines (default: 'S3Prefix'). + Valid values: + * 'S3Prefix' - the S3 URI defines a key name prefix. All objects with this prefix + will be used as inputs for the transform job. + * 'ManifestFile' - the S3 URI points to a single manifest file listing each S3 + object to use as an input for the transform job. + content_type (str): MIME type of the input data (default: None). + compression_type (str): Compression type of the input data, if + compressed (default: None). Valid values: 'Gzip', None. + split_type (str): The record delimiter for the input object + (default: 'None'). Valid values: 'None', 'Line', 'RecordIO', and + 'TFRecord'. + input_filter (str): A JSONPath to select a portion of the input to + pass to the algorithm container for inference. If you omit the + field, it gets the value '$', representing the entire input. + For CSV data, each row is taken as a JSON array, + so only index-based JSONPaths can be applied, e.g. $[0], $[1:]. + CSV data should follow the `RFC format `_. + See `Supported JSONPath Operators + `_ + for a table of supported JSONPath operators. + For more information, see the SageMaker API documentation for + `CreateTransformJob + `_. + Some examples: "$[1:]", "$.features" (default: None). + output_filter (str): A JSONPath to select a portion of the + joined/original output to return as the output. + For more information, see the SageMaker API documentation for + `CreateTransformJob + `_. + Some examples: "$[1:]", "$.prediction" (default: None). + join_source (str): The source of data to be joined to the transform + output. It can be set to 'Input' meaning the entire input record + will be joined to the inference result. You can use OutputFilter + to select the useful portion before uploading to S3. (default: + None). Valid values: Input, None. + model_client_config (dict[str, str]): Model configuration. + Dictionary contains two optional keys, + 'InvocationsTimeoutInSeconds', and 'InvocationsMaxRetries'. + (default: ``None``). + batch_data_capture_config (BatchDataCaptureConfig): Configuration object which + specifies the configurations related to the batch data capture for the transform job + (default: ``None``). + monitor_before_transform (bgool): If to run data quality + or model explainability monitoring type, + a true value of this flag indicates running the check step before the transform job. + fail_on_violation (Union[bool, PipelineVariable]): A opt-out flag to not to fail the + check step when a violation is detected. + supplied_baseline_statistics (Union[str, PipelineVariable]): The S3 path + to the supplied statistics object representing the statistics JSON file + which will be used for drift to check (default: None). + supplied_baseline_constraints (Union[str, PipelineVariable]): The S3 path + to the supplied constraints object representing the constraints JSON file + which will be used for drift to check (default: None). + wait (bool): To determine if needed to wait for the pipeline execution to complete + pipeline_name (str): The name of the Pipeline for the monitoring and transfrom step + role (str): Execution role + """ + + transformer = self + if not isinstance(self.sagemaker_session, PipelineSession): + sagemaker_session = self.sagemaker_session + self.sagemaker_session = None + transformer = copy.deepcopy(self) + transformer.sagemaker_session = PipelineSession() + self.sagemaker_session = sagemaker_session + + transform_step_args = transformer.transform( + data=data, + data_type=data_type, + content_type=content_type, + compression_type=compression_type, + split_type=split_type, + input_filter=input_filter, + output_filter=output_filter, + batch_data_capture_config=batch_data_capture_config, + join_source=join_source, + model_client_config=model_client_config, + ) + + from sagemaker.workflow.monitor_batch_transform_step import MonitorBatchTransformStep + + monitoring_batch_step = MonitorBatchTransformStep( + name="MonitorBatchTransformStep", + display_name="MonitorBatchTransformStep", + description="", + transform_step_args=transform_step_args, + monitor_configuration=monitoring_config, + check_job_configuration=monitoring_resource_config, + monitor_before_transform=monitor_before_transform, + supplied_baseline_constraints=supplied_baseline_constraints, + supplied_baseline_statistics=supplied_baseline_statistics, + ) + + pipeline_name = ( + pipeline_name if pipeline_name else f"TransformWithMonitoring{int(time.time())}" + ) + # if pipeline exists, just start the execution + from sagemaker.workflow.pipeline import Pipeline + + pipeline = Pipeline( + name=pipeline_name, + steps=[monitoring_batch_step], + sagemaker_session=transformer.sagemaker_session, + ) + pipeline.upsert(role_arn=role if role else get_execution_role()) + execution = pipeline.start() + if wait: + logging.info("Waiting for transform with monitoring to execute ...") + execution.wait() + return execution + def delete_model(self): """Delete the corresponding SageMaker model for this Transformer.""" self.sagemaker_session.delete_model(self.model_name) diff --git a/src/sagemaker/workflow/airflow.py b/src/sagemaker/workflow/airflow.py index a3565ba9c1..a3c9418e9f 100644 --- a/src/sagemaker/workflow/airflow.py +++ b/src/sagemaker/workflow/airflow.py @@ -37,12 +37,12 @@ def prepare_framework(estimator, s3_operations): """ if estimator.code_location is not None: bucket, key = s3.parse_s3_url(estimator.code_location) - key = os.path.join(key, estimator._current_job_name, "source", "sourcedir.tar.gz") + key = "/".join([key, estimator._current_job_name, "source", "sourcedir.tar.gz"]) elif estimator.uploaded_code is not None: bucket, key = s3.parse_s3_url(estimator.uploaded_code.s3_prefix) else: bucket = estimator.sagemaker_session._default_bucket - key = os.path.join(estimator._current_job_name, "source", "sourcedir.tar.gz") + key = "/".join([estimator._current_job_name, "source", "sourcedir.tar.gz"]) script = os.path.basename(estimator.entry_point) diff --git a/src/sagemaker/workflow/utilities.py b/src/sagemaker/workflow/utilities.py index 89d7c5dfd9..08c170d424 100644 --- a/src/sagemaker/workflow/utilities.py +++ b/src/sagemaker/workflow/utilities.py @@ -114,11 +114,12 @@ def get_code_hash(step: Entity) -> str: if isinstance(step, ProcessingStep) and step.step_args: kwargs = step.step_args.func_kwargs source_dir = kwargs.get("source_dir") + submit_class = kwargs.get("submit_class") dependencies = get_processing_dependencies( [ kwargs.get("dependencies"), kwargs.get("submit_py_files"), - kwargs.get("submit_class"), + [submit_class] if submit_class else None, kwargs.get("submit_jars"), kwargs.get("submit_files"), ] @@ -168,7 +169,7 @@ def get_processing_code_hash(code: str, source_dir: str, dependencies: List[str] str: A hash string representing the unique code artifact(s) for the step """ - # FrameworkProcessor + # If FrameworkProcessor contains source_dir if source_dir: source_dir_url = urlparse(source_dir) if source_dir_url.scheme == "" or source_dir_url.scheme == "file": @@ -400,5 +401,5 @@ def execute_job_functions(step_args: _StepArguments): """ chained_args = step_args.func(*step_args.func_args, **step_args.func_kwargs) - if chained_args: + if isinstance(chained_args, _StepArguments): execute_job_functions(chained_args) diff --git a/tests/data/multimodel/container/Dockerfile b/tests/data/multimodel/container/Dockerfile index 4792a429c1..71c38a6605 100644 --- a/tests/data/multimodel/container/Dockerfile +++ b/tests/data/multimodel/container/Dockerfile @@ -1,4 +1,5 @@ -FROM public.ecr.aws/ubuntu/ubuntu:18.04 +# added latest image from https://gallery.ecr.aws/lts/ubuntu +FROM public.ecr.aws/ubuntu/ubuntu:22.04 # Set a docker label to advertise multi-model support on the container LABEL com.amazonaws.sagemaker.capabilities.multi-models=true @@ -15,7 +16,7 @@ RUN apt-get update && \ curl \ vim \ && rm -rf /var/lib/apt/lists/* \ - && curl -O https://bootstrap.pypa.io/pip/3.6/get-pip.py \ + && curl -O https://bootstrap.pypa.io/pip/get-pip.py \ && python3 get-pip.py RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 diff --git a/tests/data/spark/code/java/TestJarFile.jar b/tests/data/spark/code/java/TestJarFile.jar new file mode 100644 index 0000000000..d528331d55 Binary files /dev/null and b/tests/data/spark/code/java/TestJarFile.jar differ diff --git a/tests/data/spark/code/java/hello-java-spark/HelloJavaSparkApp.jar b/tests/data/spark/code/java/hello-java-spark/HelloJavaSparkApp.jar new file mode 100644 index 0000000000..056675146d Binary files /dev/null and b/tests/data/spark/code/java/hello-java-spark/HelloJavaSparkApp.jar differ diff --git a/tests/integ/test_transformer.py b/tests/integ/test_transformer.py index a0e37ffc77..1de333b987 100644 --- a/tests/integ/test_transformer.py +++ b/tests/integ/test_transformer.py @@ -25,6 +25,7 @@ from sagemaker.transformer import Transformer from sagemaker.estimator import Estimator from sagemaker.inputs import BatchDataCaptureConfig +from sagemaker.xgboost import XGBoostModel from sagemaker.utils import unique_name_from_base from tests.integ import ( datasets, @@ -36,7 +37,7 @@ from tests.integ.timeout import timeout, timeout_and_delete_model_with_transformer from tests.integ.vpc_test_utils import get_or_create_vpc_resources -from sagemaker.model_monitor import DatasetFormat, Statistics +from sagemaker.model_monitor import DatasetFormat, Statistics, Constraints from sagemaker.workflow.check_job_config import CheckJobConfig from sagemaker.workflow.quality_check_step import ( @@ -645,3 +646,66 @@ def _create_transformer_and_transform_job( job_name=unique_name_from_base("test-transform"), ) return transformer + + +def test_transformer_and_monitoring_job( + pipeline_session, + sagemaker_session, + role, + pipeline_name, + check_job_config, + data_bias_check_config, +): + xgb_model_data_s3 = pipeline_session.upload_data( + path=os.path.join(os.path.join(DATA_DIR, "xgboost_abalone"), "xgb_model.tar.gz"), + key_prefix="integ-test-data/xgboost/model", + ) + data_bias_supplied_baseline_constraints = Constraints.from_file_path( + constraints_file_path=os.path.join( + DATA_DIR, "pipeline/clarify_check_step/data_bias/good_cases/analysis.json" + ), + sagemaker_session=sagemaker_session, + ).file_s3_uri + + xgb_model = XGBoostModel( + model_data=xgb_model_data_s3, + framework_version="1.3-1", + role=role, + sagemaker_session=sagemaker_session, + entry_point=os.path.join(os.path.join(DATA_DIR, "xgboost_abalone"), "inference.py"), + enable_network_isolation=True, + ) + + xgb_model.deploy(_INSTANCE_COUNT, _INSTANCE_TYPE) + + transform_output = f"s3://{sagemaker_session.default_bucket()}/{pipeline_name}Transform" + transformer = Transformer( + model_name=xgb_model.name, + strategy="SingleRecord", + instance_type="ml.m5.xlarge", + instance_count=1, + output_path=transform_output, + sagemaker_session=pipeline_session, + ) + + transform_input = pipeline_session.upload_data( + path=os.path.join(DATA_DIR, "xgboost_abalone", "abalone"), + key_prefix="integ-test-data/xgboost_abalone/abalone", + ) + + execution = transformer.transform_with_monitoring( + monitoring_config=data_bias_check_config, + monitoring_resource_config=check_job_config, + data=transform_input, + content_type="text/libsvm", + supplied_baseline_constraints=data_bias_supplied_baseline_constraints, + role=role, + ) + + execution_steps = execution.list_steps() + assert len(execution_steps) == 2 + + for execution_step in execution_steps: + assert execution_step["StepStatus"] == "Succeeded" + + xgb_model.delete_model() diff --git a/tests/integ/test_xgboost.py b/tests/integ/test_xgboost.py index 733ab4665a..df06a8863a 100644 --- a/tests/integ/test_xgboost.py +++ b/tests/integ/test_xgboost.py @@ -40,6 +40,26 @@ def xgboost_training_job( ) +def test_sourcedir_naming( + sagemaker_session, + xgboost_latest_version, + xgboost_latest_py_version, + cpu_instance_type, +): + with pytest.raises(RuntimeError): + processor = XGBoostProcessor( + framework_version=xgboost_latest_version, + role=ROLE, + instance_count=1, + instance_type=cpu_instance_type, + sagemaker_session=sagemaker_session, + ) + processor.run( + source_dir="s3://bucket/deps.tar.gz", + code="main_script.py", + ) + + @pytest.mark.release def test_framework_processing_job_with_deps( sagemaker_session, diff --git a/tests/unit/sagemaker/workflow/test_pipeline.py b/tests/unit/sagemaker/workflow/test_pipeline.py index 327443aee7..f0cb2e5234 100644 --- a/tests/unit/sagemaker/workflow/test_pipeline.py +++ b/tests/unit/sagemaker/workflow/test_pipeline.py @@ -17,7 +17,7 @@ import pytest -from mock import Mock +from mock import Mock, patch from sagemaker import s3 from sagemaker.workflow.condition_step import ConditionStep @@ -78,6 +78,7 @@ def test_pipeline_create_with_parallelism_config(sagemaker_session_mock, role_ar ) +@patch("sagemaker.s3.S3Uploader.upload_string_as_file_body") def test_large_pipeline_create(sagemaker_session_mock, role_arn): parameter = ParameterString("MyStr") pipeline = Pipeline( @@ -87,8 +88,6 @@ def test_large_pipeline_create(sagemaker_session_mock, role_arn): sagemaker_session=sagemaker_session_mock, ) - s3.S3Uploader.upload_string_as_file_body = Mock() - pipeline.create(role_arn=role_arn) assert s3.S3Uploader.upload_string_as_file_body.called_with( @@ -151,6 +150,7 @@ def test_pipeline_update_with_parallelism_config(sagemaker_session_mock, role_ar ) +@patch("sagemaker.s3.S3Uploader.upload_string_as_file_body") def test_large_pipeline_update(sagemaker_session_mock, role_arn): parameter = ParameterString("MyStr") pipeline = Pipeline( @@ -160,8 +160,6 @@ def test_large_pipeline_update(sagemaker_session_mock, role_arn): sagemaker_session=sagemaker_session_mock, ) - s3.S3Uploader.upload_string_as_file_body = Mock() - pipeline.create(role_arn=role_arn) assert s3.S3Uploader.upload_string_as_file_body.called_with( diff --git a/tests/unit/sagemaker/workflow/test_processing_step.py b/tests/unit/sagemaker/workflow/test_processing_step.py index fd32fd7c73..9ba242b9b2 100644 --- a/tests/unit/sagemaker/workflow/test_processing_step.py +++ b/tests/unit/sagemaker/workflow/test_processing_step.py @@ -13,7 +13,8 @@ from __future__ import absolute_import import json -from mock import Mock, PropertyMock +import os +from mock import Mock, PropertyMock, patch import pytest import warnings @@ -45,9 +46,11 @@ from sagemaker.workflow.steps import CacheConfig, ProcessingStep from sagemaker.workflow.pipeline import Pipeline, PipelineGraph +from sagemaker.workflow.pipeline_context import _PipelineConfig from sagemaker.workflow.properties import PropertyFile from sagemaker.workflow.parameters import ParameterString from sagemaker.workflow.functions import Join +from sagemaker.workflow.utilities import hash_files_or_dirs from sagemaker.workflow import is_pipeline_variable from sagemaker.network import NetworkConfig @@ -63,6 +66,7 @@ SHAPConfig, ) from tests.unit.sagemaker.workflow.helpers import CustomStep, ordered, get_step_args_helper +from tests.unit import DATA_DIR REGION = "us-west-2" BUCKET = "my-bucket" @@ -70,7 +74,20 @@ IMAGE_URI = "fakeimage" MODEL_NAME = "gisele" DUMMY_S3_SCRIPT_PATH = "s3://dummy-s3/dummy_script.py" +LOCAL_SCRIPT_PATH = os.path.join(DATA_DIR, "workflow/abalone/preprocessing.py") +SPARK_APP_JAR_PATH = os.path.join( + DATA_DIR, "spark/code/java/hello-java-spark/HelloJavaSparkApp.jar" +) +SPARK_DEP_JAR = os.path.join(DATA_DIR, "spark/code/java/TestJarFile.jar") +SPARK_APP_PY_PATH = os.path.join(DATA_DIR, "spark/code/python/hello_py_spark/hello_py_spark_app.py") +SPARK_PY_FILE1 = os.path.join(DATA_DIR, "spark/code/python/hello_py_spark/__init__.py") +SPARK_PY_FILE2 = os.path.join(DATA_DIR, "spark/code/python/hello_py_spark/hello_py_spark_udfs.py") +SPARK_SUBMIT_FILE1 = os.path.join(DATA_DIR, "spark/files/data.jsonl") +SPARK_SUBMIT_FILE2 = os.path.join(DATA_DIR, "spark/files/sample_spark_event_logs") INSTANCE_TYPE = "ml.m4.xlarge" +MOCKED_PIPELINE_CONFIG = _PipelineConfig( + "MyPipeline", "MyProcessingStep", hash_files_or_dirs([LOCAL_SCRIPT_PATH]), "config-hash-abcdefg" +) FRAMEWORK_PROCESSOR = [ ( @@ -154,6 +171,19 @@ ), ] +FRAMEWORK_PROCESSOR_LOCAL_CODE = [ + ( + FrameworkProcessor( + framework_version="1.8", + instance_type=INSTANCE_TYPE, + instance_count=1, + role=ROLE, + estimator_cls=PyTorch, + ), + {"code": LOCAL_SCRIPT_PATH}, + ), +] + PROCESSING_INPUT = [ ProcessingInput(source="s3://my-bucket/processing_manifest", destination="processing_manifest"), ProcessingInput( @@ -318,7 +348,8 @@ def test_processing_step_with_processor( else: expected_step_arguments["ExperimentConfig"] = expected_experiment_config - assert json.loads(pipeline.definition())["Steps"][0] == { + step_def = json.loads(pipeline.definition())["Steps"][0] + assert step_def == { "Name": "MyProcessingStep", "Description": "ProcessingStep description", "DisplayName": "MyProcessingStep", @@ -346,6 +377,10 @@ def test_processing_step_with_processor( } ) + # test idempotency + step_def2 = json.loads(pipeline.definition())["Steps"][0] + assert step_def == step_def2 + @pytest.mark.parametrize( "image_uri", @@ -387,7 +422,11 @@ def test_processing_step_with_processor_and_step_args( assert isinstance(e, ValueError) -def test_processing_step_with_script_processor(pipeline_session, processing_input, network_config): +@patch("sagemaker.workflow.utilities._pipeline_config", MOCKED_PIPELINE_CONFIG) +@pytest.mark.parametrize("code_artifact", [DUMMY_S3_SCRIPT_PATH, LOCAL_SCRIPT_PATH]) +def test_processing_step_with_script_processor( + pipeline_session, processing_input, network_config, code_artifact +): processor = ScriptProcessor( role=ROLE, image_uri=IMAGE_URI, @@ -406,7 +445,7 @@ def test_processing_step_with_script_processor(pipeline_session, processing_inpu ) step_args = processor.run( - inputs=processing_input, code=DUMMY_S3_SCRIPT_PATH, job_name="my-processing-job" + inputs=processing_input, code=code_artifact, job_name="my-processing-job" ) step = ProcessingStep( @@ -420,11 +459,13 @@ def test_processing_step_with_script_processor(pipeline_session, processing_inpu sagemaker_session=pipeline_session, ) - assert json.loads(pipeline.definition())["Steps"][0] == { - "Name": "MyProcessingStep", - "Type": "Processing", - "Arguments": get_step_args_helper(step_args, "Processing"), - } + step_args = get_step_args_helper(step_args, "Processing") + step_def = json.loads(pipeline.definition())["Steps"][0] + assert step_def == {"Name": "MyProcessingStep", "Type": "Processing", "Arguments": step_args} + + # test idempotency + step_def2 = json.loads(pipeline.definition())["Steps"][0] + assert step_def == step_def2 @pytest.mark.parametrize("framework_processor", FRAMEWORK_PROCESSOR) @@ -477,6 +518,66 @@ def test_processing_step_with_framework_processor( "Arguments": step_args, } + # test idempotency + step_def2 = json.loads(pipeline.definition())["Steps"][0] + del step_def2["Arguments"]["ProcessingInputs"][0]["S3Input"]["S3Uri"] + del step_def2["Arguments"]["ProcessingOutputConfig"]["Outputs"][0]["S3Output"]["S3Uri"] + assert step_def == step_def2 + + +@patch("sagemaker.workflow.utilities._pipeline_config", MOCKED_PIPELINE_CONFIG) +@pytest.mark.parametrize("framework_processor", FRAMEWORK_PROCESSOR_LOCAL_CODE) +def test_processing_step_with_framework_processor_local_code( + framework_processor, pipeline_session, network_config +): + processor, run_inputs = framework_processor + processor.sagemaker_session = pipeline_session + processor.role = ROLE + + processor.volume_kms_key = "volume-kms-key" + processor.network_config = network_config + + processing_input = ProcessingInput( + source="s3://my-bucket/processing_manifest", + destination="processing_manifest", + input_name="manifest", + ) + processing_output = ProcessingOutput( + output_name="framework_output", source="/opt/ml/processing/framework_output" + ) + + run_inputs["inputs"] = [processing_input] + run_inputs["outputs"] = [processing_output] + + step_args = processor.run(**run_inputs) + + step = ProcessingStep( + name="MyProcessingStep", + step_args=step_args, + ) + pipeline = Pipeline( + name="MyPipeline", + steps=[step], + sagemaker_session=pipeline_session, + ) + + step_args = get_step_args_helper(step_args, "Processing") + step_def = json.loads(pipeline.definition())["Steps"][0] + + del step_args["ProcessingOutputConfig"]["Outputs"][0]["S3Output"]["S3Uri"] + del step_def["Arguments"]["ProcessingOutputConfig"]["Outputs"][0]["S3Output"]["S3Uri"] + + assert step_def == { + "Name": "MyProcessingStep", + "Type": "Processing", + "Arguments": step_args, + } + + # test idempotency + step_def2 = json.loads(pipeline.definition())["Steps"][0] + del step_def2["Arguments"]["ProcessingOutputConfig"]["Outputs"][0]["S3Output"]["S3Uri"] + assert step_def == step_def2 + def test_processing_step_with_clarify_processor(pipeline_session): def headers(): @@ -530,12 +631,17 @@ def verify(step_args): steps=[step], sagemaker_session=pipeline_session, ) - assert json.loads(pipeline.definition())["Steps"][0] == { + step_def = json.loads(pipeline.definition())["Steps"][0] + assert step_def == { "Name": "MyProcessingStep", "Type": "Processing", "Arguments": get_step_args_helper(step_args, "Processing"), } + # test idempotency + step_def2 = json.loads(pipeline.definition())["Steps"][0] + assert step_def == step_def2 + test_run = utils.unique_name_from_base("test_run") output_path = "s3://{}/{}/{}".format( pipeline_session.default_bucket(), "linear_learner_analysis_result", test_run @@ -852,4 +958,153 @@ def test_spark_processor(spark_processor, processing_input, pipeline_session): steps=[step], sagemaker_session=pipeline_session, ) - pipeline.definition() + + # test for idempotency + step_def = json.loads(pipeline.definition())["Steps"][0] + step_def_2 = json.loads(pipeline.definition())["Steps"][0] + assert step_def == step_def_2 + + +@pytest.mark.parametrize( + "spark_processor", + [ + ( + SparkJarProcessor( + role=ROLE, + framework_version="2.4", + instance_count=1, + instance_type=INSTANCE_TYPE, + ), + { + "submit_app": SPARK_APP_JAR_PATH, + "submit_class": "com.amazonaws.sagemaker.spark.test.HelloJavaSparkApp", + "arguments": [ + "--input", + "input-data-uri", + "--output", + ParameterString("MyArgOutput"), + ], + "submit_jars": [ + SPARK_DEP_JAR, + ], + "submit_files": [ + SPARK_SUBMIT_FILE1, + SPARK_SUBMIT_FILE2, + ], + "spark_event_logs_s3_uri": ParameterString("MySparkEventLogS3Uri"), + "configuration": { + "Classification": "core-site", + "Properties": {"hadoop.security.groups.cache.secs": "250"}, + }, + }, + ), + ( + PySparkProcessor( + role=ROLE, + framework_version="2.4", + instance_count=1, + instance_type=INSTANCE_TYPE, + ), + { + "submit_app": SPARK_APP_PY_PATH, + "arguments": [ + "--input", + "input-data-uri", + "--output", + ParameterString("MyArgOutput"), + ], + "submit_py_files": [ + SPARK_PY_FILE1, + SPARK_PY_FILE2, + ], + "submit_jars": [SPARK_DEP_JAR], + "submit_files": [SPARK_SUBMIT_FILE1, SPARK_SUBMIT_FILE2], + "spark_event_logs_s3_uri": ParameterString("MySparkEventLogS3Uri"), + "configuration": { + "Classification": "core-site", + "Properties": {"hadoop.security.groups.cache.secs": "250"}, + }, + }, + ), + ], +) +def test_spark_processor_local_code(spark_processor, processing_input, pipeline_session): + processor, run_inputs = spark_processor + processor.sagemaker_session = pipeline_session + processor.role = ROLE + + run_inputs["inputs"] = processing_input + + step_args = processor.run(**run_inputs) + step = ProcessingStep( + name="MyProcessingStep", + step_args=step_args, + ) + + step_args = get_step_args_helper(step_args, "Processing") + + assert step_args["AppSpecification"]["ContainerArguments"] == run_inputs["arguments"] + + entry_points = step_args["AppSpecification"]["ContainerEntrypoint"] + entry_points_expr = [] + for entry_point in entry_points: + if is_pipeline_variable(entry_point): + entry_points_expr.append(entry_point.expr) + else: + entry_points_expr.append(entry_point) + + if "submit_py_files" in run_inputs: + expected = [ + "smspark-submit", + "--py-files", + "/opt/ml/processing/input/py-files", + "--jars", + "/opt/ml/processing/input/jars", + "--files", + "/opt/ml/processing/input/files", + "--local-spark-event-logs-dir", + "/opt/ml/processing/spark-events/", + "/opt/ml/processing/input/code/hello_py_spark_app.py", + ] + # py spark + else: + expected = [ + "smspark-submit", + "--class", + "com.amazonaws.sagemaker.spark.test.HelloJavaSparkApp", + "--jars", + "/opt/ml/processing/input/jars", + "--files", + "/opt/ml/processing/input/files", + "--local-spark-event-logs-dir", + "/opt/ml/processing/spark-events/", + "/opt/ml/processing/input/code/HelloJavaSparkApp.jar", + ] + + assert entry_points_expr == expected + for output in step_args["ProcessingOutputConfig"]["Outputs"]: + if is_pipeline_variable(output["S3Output"]["S3Uri"]): + output["S3Output"]["S3Uri"] = output["S3Output"]["S3Uri"].expr + + assert step_args["ProcessingOutputConfig"]["Outputs"] == [ + { + "OutputName": "output-1", + "AppManaged": False, + "S3Output": { + "S3Uri": {"Get": "Parameters.MySparkEventLogS3Uri"}, + "LocalPath": "/opt/ml/processing/spark-events/", + "S3UploadMode": "Continuous", + }, + } + ] + + pipeline = Pipeline( + name="MyPipeline", + steps=[step], + sagemaker_session=pipeline_session, + ) + + # test for idempotency + step_def = json.loads(pipeline.definition())["Steps"][0] + step_def2 = json.loads(pipeline.definition())["Steps"][0] + assert step_def == step_def2 diff --git a/tests/unit/sagemaker/workflow/test_training_step.py b/tests/unit/sagemaker/workflow/test_training_step.py index 4133343c93..3e8b57b069 100644 --- a/tests/unit/sagemaker/workflow/test_training_step.py +++ b/tests/unit/sagemaker/workflow/test_training_step.py @@ -14,7 +14,7 @@ import os import json -from mock import Mock, PropertyMock +from mock import Mock, PropertyMock, patch import pytest import warnings @@ -25,11 +25,12 @@ from sagemaker.parameter import IntegerParameter from sagemaker.transformer import Transformer from sagemaker.tuner import HyperparameterTuner -from sagemaker.workflow.pipeline_context import PipelineSession +from sagemaker.workflow.pipeline_context import PipelineSession, _PipelineConfig from sagemaker.workflow.parameters import ParameterString, ParameterBoolean from sagemaker.workflow.steps import TrainingStep from sagemaker.workflow.pipeline import Pipeline, PipelineGraph +from sagemaker.workflow.utilities import hash_files_or_dirs from sagemaker.workflow.functions import Join from sagemaker.estimator import Estimator @@ -66,9 +67,19 @@ ROLE = "DummyRole" IMAGE_URI = "fakeimage" MODEL_NAME = "gisele" -DUMMY_LOCAL_SCRIPT_PATH = os.path.join(DATA_DIR, "dummy_script.py") +LOCAL_ENTRY_POINT = os.path.join(DATA_DIR, "tfs/tfs-test-entrypoint-with-handler/training.py") +LOCAL_SOURCE_DIR = os.path.join(DATA_DIR, "tfs/tfs-test-entrypoint-with-handler") +LOCAL_DEPS = [ + os.path.join(DATA_DIR, "tfs/tfs-test-entrypoint-and-dependencies"), +] DUMMY_S3_SCRIPT_PATH = "s3://dummy-s3/dummy_script.py" INSTANCE_TYPE = "ml.m4.xlarge" +MOCKED_PIPELINE_CONFIG = _PipelineConfig( + "MyPipeline", + "MyTrainingStep", + hash_files_or_dirs([LOCAL_SOURCE_DIR] + LOCAL_DEPS), + "config-hash-abcdefg", +) ESTIMATOR_LISTS = [ SKLearn( @@ -152,6 +163,88 @@ ), ] +ESTIMATOR_LISTS_LOCAL_CODE = [ + SKLearn( + framework_version="0.23-1", + py_version="py3", + instance_type=INSTANCE_TYPE, + instance_count=1, + role=ROLE, + entry_point=LOCAL_ENTRY_POINT, + source_dir=LOCAL_SOURCE_DIR, + ), + PyTorch( + role=ROLE, + instance_type=INSTANCE_TYPE, + instance_count=1, + framework_version="1.8.0", + py_version="py36", + entry_point=LOCAL_ENTRY_POINT, + source_dir=LOCAL_SOURCE_DIR, + ), + TensorFlow( + role=ROLE, + entry_point=LOCAL_ENTRY_POINT, + source_dir=LOCAL_SOURCE_DIR, + instance_type=INSTANCE_TYPE, + instance_count=1, + framework_version="2.0", + py_version="py3", + ), + HuggingFace( + transformers_version="4.6", + pytorch_version="1.7", + role=ROLE, + instance_type="ml.p3.2xlarge", + instance_count=1, + py_version="py36", + entry_point=LOCAL_ENTRY_POINT, + source_dir=LOCAL_SOURCE_DIR, + ), + XGBoost( + framework_version="1.3-1", + py_version="py3", + role=ROLE, + instance_type=INSTANCE_TYPE, + instance_count=1, + entry_point=LOCAL_ENTRY_POINT, + source_dir=LOCAL_SOURCE_DIR, + ), + MXNet( + framework_version="1.4.1", + py_version="py3", + role=ROLE, + instance_type=INSTANCE_TYPE, + instance_count=1, + entry_point=LOCAL_ENTRY_POINT, + source_dir=LOCAL_SOURCE_DIR, + toolkit=RLToolkit.RAY, + framework=RLFramework.TENSORFLOW, + toolkit_version="0.8.5", + ), + RLEstimator( + entry_point=LOCAL_ENTRY_POINT, + source_dir=LOCAL_SOURCE_DIR, + toolkit=RLToolkit.RAY, + framework=RLFramework.TENSORFLOW, + toolkit_version="0.8.5", + role=ROLE, + instance_type=INSTANCE_TYPE, + instance_count=1, + ), + Chainer( + role=ROLE, + entry_point=LOCAL_ENTRY_POINT, + source_dir=LOCAL_SOURCE_DIR, + use_mpi=True, + num_processes=4, + framework_version="5.0.0", + instance_type=INSTANCE_TYPE, + instance_count=1, + py_version="py3", + ), +] + INPUT_PARAM_LISTS = [ "s3://my-bucket/my-training-input", @@ -209,6 +302,7 @@ def hyperparameters(): return {"test-key": "test-val"} +@patch("sagemaker.workflow.utilities._pipeline_config", MOCKED_PIPELINE_CONFIG) @pytest.mark.parametrize( "experiment_config, expected_experiment_config", [ @@ -250,6 +344,9 @@ def test_training_step_with_estimator( hyperparameters=hyperparameters, enable_network_isolation=enable_network_isolation, encrypt_inter_container_traffic=encrypt_container_traffic, + entry_point=LOCAL_ENTRY_POINT, + source_dir=LOCAL_SOURCE_DIR, + dependencies=LOCAL_DEPS, ) with warnings.catch_warnings(record=True) as w: @@ -289,18 +386,15 @@ def test_training_step_with_estimator( sagemaker_session=pipeline_session, ) step_args = get_step_args_helper(step_args, "Training") - expected_step_arguments = deepcopy(step_args) - expected_step_arguments["EnableInterContainerTrafficEncryption"] = { + step_args["EnableInterContainerTrafficEncryption"] = { "Get": "Parameters.encrypt_container_traffic" } - expected_step_arguments["EnableNetworkIsolation"] = { - "Get": "Parameters.enable_network_isolation" - } + step_args["EnableNetworkIsolation"] = {"Get": "Parameters.enable_network_isolation"} if expected_experiment_config is None: - expected_step_arguments.pop("ExperimentConfig", None) + step_args.pop("ExperimentConfig", None) else: - expected_step_arguments["ExperimentConfig"] = expected_experiment_config + step_args["ExperimentConfig"] = expected_experiment_config assert step_condition.conditions[0].left.expr == { "Get": "Steps.MyTrainingStep.FinalMetricDataList['val:acc'].Value" @@ -309,7 +403,7 @@ def test_training_step_with_estimator( # delete profiler rule configurations because of timestamp collision del step_definition["Arguments"]["ProfilerRuleConfigurations"] - del expected_step_arguments["ProfilerRuleConfigurations"] + del step_args["ProfilerRuleConfigurations"] assert step_definition == { "Name": "MyTrainingStep", @@ -317,7 +411,7 @@ def test_training_step_with_estimator( "DisplayName": "MyTrainingStep", "Type": "Training", "DependsOn": ["TestStep"], - "Arguments": expected_step_arguments, + "Arguments": step_args, } assert step_train.properties.TrainingJobName.expr == { "Get": "Steps.MyTrainingStep.TrainingJobName" @@ -332,6 +426,11 @@ def test_training_step_with_estimator( } ) + # test idempotency + step_def2 = json.loads(pipeline.definition())["Steps"][0] + del step_def2["Arguments"]["ProfilerRuleConfigurations"] + assert step_definition == step_def2 + def test_training_step_estimator_with_param_code_input( pipeline_session, training_input, hyperparameters @@ -374,7 +473,8 @@ def test_training_step_estimator_with_param_code_input( step_args = get_step_args_helper(step_args, "Training") step_args["HyperParameters"]["sagemaker_program"] = {"Get": "Parameters.EntryPoint"} step_args["HyperParameters"]["sagemaker_submit_directory"] = {"Get": "Parameters.SourceDir"} - assert json.loads(pipeline.definition())["Steps"][0] == { + step_def = json.loads(pipeline.definition())["Steps"][0] + assert step_def == { "Name": "MyTrainingStep", "Description": "TrainingStep description", "DisplayName": "MyTrainingStep", @@ -382,6 +482,10 @@ def test_training_step_estimator_with_param_code_input( "Arguments": step_args, } + # test idempotency + step_def2 = json.loads(pipeline.definition())["Steps"][0] + assert step_def == step_def2 + @pytest.mark.parametrize("estimator", ESTIMATOR_LISTS) @pytest.mark.parametrize("training_input", INPUT_PARAM_LISTS) @@ -414,41 +518,119 @@ def test_training_step_with_framework_estimator( ) step_args = get_step_args_helper(step_args, "Training") + expected_step_args = deepcopy(step_args) step_def = json.loads(pipeline.definition())["Steps"][0] - assert step_args["InputDataConfig"][0]["DataSource"]["S3DataSource"]["S3Uri"] == training_input - assert step_args["OutputDataConfig"]["S3OutputPath"] == output_path - step_args["HyperParameters"]["sagemaker_program"] = {"Get": "Parameters.EntryPoint"} - step_args["HyperParameters"]["sagemaker_submit_directory"] = {"Get": "Parameters.SourceDir"} + assert ( + expected_step_args["InputDataConfig"][0]["DataSource"]["S3DataSource"]["S3Uri"] + == training_input + ) + assert expected_step_args["OutputDataConfig"]["S3OutputPath"] == output_path + expected_step_args["HyperParameters"]["sagemaker_program"] = {"Get": "Parameters.EntryPoint"} + expected_step_args["HyperParameters"]["sagemaker_submit_directory"] = { + "Get": "Parameters.SourceDir" + } - del step_args["InputDataConfig"][0]["DataSource"]["S3DataSource"]["S3Uri"] + del expected_step_args["InputDataConfig"][0]["DataSource"]["S3DataSource"]["S3Uri"] del step_def["Arguments"]["InputDataConfig"][0]["DataSource"]["S3DataSource"]["S3Uri"] - del step_args["OutputDataConfig"]["S3OutputPath"] + del expected_step_args["OutputDataConfig"]["S3OutputPath"] del step_def["Arguments"]["OutputDataConfig"]["S3OutputPath"] - # trim timestamp so RuleConfigurationName will match - rule_config_name_step_args = step_args["ProfilerRuleConfigurations"][0]["RuleConfigurationName"] - step_args["ProfilerRuleConfigurations"][0][ - "RuleConfigurationName" - ] = rule_config_name_step_args[:-11] - rule_config_name_step_def = step_def["Arguments"]["ProfilerRuleConfigurations"][0][ - "RuleConfigurationName" - ] - step_def["Arguments"]["ProfilerRuleConfigurations"][0][ - "RuleConfigurationName" - ] = rule_config_name_step_def[:-11] + # delete profiler rule configurations because of timestamp collision + del step_def["Arguments"]["ProfilerRuleConfigurations"] + del expected_step_args["ProfilerRuleConfigurations"] if "sagemaker_s3_output" in step_args["HyperParameters"]: - del step_args["HyperParameters"]["sagemaker_s3_output"] + del expected_step_args["HyperParameters"]["sagemaker_s3_output"] del step_def["Arguments"]["HyperParameters"]["sagemaker_s3_output"] assert step_def == { "Name": "MyTrainingStep", "Type": "Training", - "Arguments": step_args, + "Arguments": expected_step_args, } + # test idempotency + step_def2 = json.loads(pipeline.definition())["Steps"][0] + del step_def2["Arguments"]["InputDataConfig"][0]["DataSource"]["S3DataSource"]["S3Uri"] + del step_def2["Arguments"]["OutputDataConfig"]["S3OutputPath"] + del step_def2["Arguments"]["ProfilerRuleConfigurations"] + if "sagemaker_s3_output" in step_def2["Arguments"]["HyperParameters"]: + del step_def2["Arguments"]["HyperParameters"]["sagemaker_s3_output"] + assert step_def == step_def2 + + +@patch("sagemaker.workflow.utilities._pipeline_config", MOCKED_PIPELINE_CONFIG) +@pytest.mark.parametrize("estimator", ESTIMATOR_LISTS_LOCAL_CODE) +@pytest.mark.parametrize("training_input", INPUT_PARAM_LISTS) +@pytest.mark.parametrize( + "output_path", ["s3://my-bucket/my-output-path", ParameterString(name="OutputPath")] +) +def test_training_step_with_framework_estimator_local_code( + estimator, pipeline_session, training_input, output_path, hyperparameters +): + estimator.set_hyperparameters(**hyperparameters) + estimator.volume_kms_key = "volume-kms-key" + estimator.output_kms_key = "output-kms-key" + estimator.dependencies = LOCAL_DEPS + estimator.output_path = output_path + # TODO: remove job_name once we merge + # https://github.com/aws/sagemaker-python-sdk/pull/3158/files + estimator.base_job_name = "TestJob" + + estimator.sagemaker_session = pipeline_session + step_args = estimator.fit(inputs=TrainingInput(s3_data=training_input)) + + step = TrainingStep( + name="MyTrainingStep", + step_args=step_args, + ) + pipeline = Pipeline( + name="MyPipeline", + steps=[step], + sagemaker_session=pipeline_session, + ) + + step_args = get_step_args_helper(step_args, "Training") + expected_step_args = deepcopy(step_args) + step_def = json.loads(pipeline.definition())["Steps"][0] + + assert ( + expected_step_args["InputDataConfig"][0]["DataSource"]["S3DataSource"]["S3Uri"] + == training_input + ) + assert expected_step_args["OutputDataConfig"]["S3OutputPath"] == output_path + + del expected_step_args["InputDataConfig"][0]["DataSource"]["S3DataSource"]["S3Uri"] + del step_def["Arguments"]["InputDataConfig"][0]["DataSource"]["S3DataSource"]["S3Uri"] + + del expected_step_args["OutputDataConfig"]["S3OutputPath"] + del step_def["Arguments"]["OutputDataConfig"]["S3OutputPath"] + + # delete profiler rule configurations because of timestamp collision + del step_def["Arguments"]["ProfilerRuleConfigurations"] + del expected_step_args["ProfilerRuleConfigurations"] + + if "sagemaker_s3_output" in step_args["HyperParameters"]: + del expected_step_args["HyperParameters"]["sagemaker_s3_output"] + del step_def["Arguments"]["HyperParameters"]["sagemaker_s3_output"] + + assert step_def == { + "Name": "MyTrainingStep", + "Type": "Training", + "Arguments": expected_step_args, + } + + # test idempotency + step_def2 = json.loads(pipeline.definition())["Steps"][0] + del step_def2["Arguments"]["InputDataConfig"][0]["DataSource"]["S3DataSource"]["S3Uri"] + del step_def2["Arguments"]["OutputDataConfig"]["S3OutputPath"] + del step_def2["Arguments"]["ProfilerRuleConfigurations"] + if "sagemaker_s3_output" in step_def2["Arguments"]["HyperParameters"]: + del step_def2["Arguments"]["HyperParameters"]["sagemaker_s3_output"] + assert step_def == step_def2 + @pytest.mark.parametrize( "algo_estimator", @@ -519,17 +701,9 @@ def test_training_step_with_algorithm_base(algo_estimator, training_input, pipel del step_args["InputDataConfig"][0]["DataSource"]["S3DataSource"]["S3Uri"] del step_def["Arguments"]["InputDataConfig"][0]["DataSource"]["S3DataSource"]["S3Uri"] - # trim timestamp so RuleConfigurationName will match - rule_config_name_step_args = step_args["ProfilerRuleConfigurations"][0]["RuleConfigurationName"] - step_args["ProfilerRuleConfigurations"][0][ - "RuleConfigurationName" - ] = rule_config_name_step_args[:-11] - rule_config_name_step_def = step_def["Arguments"]["ProfilerRuleConfigurations"][0][ - "RuleConfigurationName" - ] - step_def["Arguments"]["ProfilerRuleConfigurations"][0][ - "RuleConfigurationName" - ] = rule_config_name_step_def[:-11] + # delete profiler rule configurations because of timestamp collision + del step_def["Arguments"]["ProfilerRuleConfigurations"] + del step_args["ProfilerRuleConfigurations"] assert step_def == { "Name": "MyTrainingStep", @@ -537,6 +711,100 @@ def test_training_step_with_algorithm_base(algo_estimator, training_input, pipel "Arguments": step_args, } + # test idempotency + step_def2 = json.loads(pipeline.definition())["Steps"][0] + del step_def2["Arguments"]["InputDataConfig"][0]["DataSource"]["S3DataSource"]["S3Uri"] + del step_def2["Arguments"]["ProfilerRuleConfigurations"] + assert step_def == step_def2 + + +@patch("sagemaker.workflow.utilities._pipeline_config", MOCKED_PIPELINE_CONFIG) +@pytest.mark.parametrize( + "algo_estimator", + [ + KNN, + KMeans, + LinearLearner, + RandomCutForest, + LDA, + Object2Vec, + NTM, + PCA, + FactorizationMachines, + IPInsights, + ], +) +@pytest.mark.parametrize( + "training_input", + INPUT_PARAM_LISTS, +) +def test_training_step_with_algorithm_base_local_code( + algo_estimator, training_input, pipeline_session +): + estimator = algo_estimator( + role=ROLE, + instance_type=INSTANCE_TYPE, + instance_count=1, + sagemaker_session=pipeline_session, + entry_point=LOCAL_ENTRY_POINT, + source_dir=LOCAL_SOURCE_DIR, + dependencies=LOCAL_DEPS, + # TODO: remove job_name once we merge + # https://github.com/aws/sagemaker-python-sdk/pull/3158/files + base_job_name="TestJob", + ) + data = RecordSet( + s3_data=training_input, + num_records=1000, + feature_dim=128, + channel="train", + ) + + with warnings.catch_warnings(record=True) as w: + step_args = estimator.fit( + records=data, + mini_batch_size=1000, + ) + assert len(w) == 1 + assert issubclass(w[-1].category, UserWarning) + assert "Running within a PipelineSession" in str(w[-1].message) + + with warnings.catch_warnings(record=True) as w: + step = TrainingStep( + name="MyTrainingStep", + step_args=step_args, + ) + assert len(w) == 0 + + pipeline = Pipeline( + name="MyPipeline", + steps=[step], + sagemaker_session=pipeline_session, + ) + + step_args = get_step_args_helper(step_args, "Training") + + step_def = json.loads(pipeline.definition())["Steps"][0] + assert step_args["InputDataConfig"][0]["DataSource"]["S3DataSource"]["S3Uri"] == training_input + del step_args["InputDataConfig"][0]["DataSource"]["S3DataSource"]["S3Uri"] + del step_def["Arguments"]["InputDataConfig"][0]["DataSource"]["S3DataSource"]["S3Uri"] + + # delete profiler rule configurations because of timestamp collision + del step_def["Arguments"]["ProfilerRuleConfigurations"] + del step_args["ProfilerRuleConfigurations"] + + assert step_def == { + "Name": "MyTrainingStep", + "Type": "Training", + "Arguments": step_args, + } + + # test idempotency + step_def2 = json.loads(pipeline.definition())["Steps"][0] + del step_def2["Arguments"]["InputDataConfig"][0]["DataSource"]["S3DataSource"]["S3Uri"] + del step_def2["Arguments"]["ProfilerRuleConfigurations"] + assert step_def == step_def2 + @pytest.mark.parametrize( "inputs", diff --git a/tests/unit/sagemaker/workflow/test_transform_step.py b/tests/unit/sagemaker/workflow/test_transform_step.py index 5699f13538..ffc901bf5c 100644 --- a/tests/unit/sagemaker/workflow/test_transform_step.py +++ b/tests/unit/sagemaker/workflow/test_transform_step.py @@ -176,6 +176,10 @@ def test_transform_step_with_transformer(model_name, data, output_path, pipeline "Arguments": expected_step_arguments, } + # test idempotency + step_def2 = json.loads(pipeline.definition())["Steps"][0] + assert step_def == step_def2 + @pytest.mark.parametrize( "experiment_config, expected_experiment_config", @@ -261,6 +265,10 @@ def test_transform_step_with_transformer_experiment_config( adjacency_list = PipelineGraph.from_pipeline(pipeline).adjacency_list assert adjacency_list == {"MyTransformStep": []} + # test idempotency + step_def2 = json.loads(pipeline.definition())["Steps"][0] + assert step_def == step_def2 + @pytest.mark.parametrize( "inputs", diff --git a/tests/unit/sagemaker/workflow/test_tuning_step.py b/tests/unit/sagemaker/workflow/test_tuning_step.py index 9c7b764c3b..6c022bb255 100644 --- a/tests/unit/sagemaker/workflow/test_tuning_step.py +++ b/tests/unit/sagemaker/workflow/test_tuning_step.py @@ -159,22 +159,14 @@ def test_tuning_step_with_single_algo_tuner(pipeline_session, training_input, en "S3DataSource" ]["S3Uri"] - # trim timestamp so sagemaker_job_name will still match - step_args_sm_job_name = step_args["TrainingJobDefinition"]["StaticHyperParameters"][ - "sagemaker_job_name" - ] - step_args["TrainingJobDefinition"]["StaticHyperParameters"][ - "sagemaker_job_name" - ] = step_args_sm_job_name[:-24] - step_def_sm_job_name = step_def["Arguments"]["TrainingJobDefinition"]["StaticHyperParameters"][ + # delete sagemaker_job_name b/c of timestamp collision + del step_args["TrainingJobDefinition"]["StaticHyperParameters"]["sagemaker_job_name"] + del step_def["Arguments"]["TrainingJobDefinition"]["StaticHyperParameters"][ "sagemaker_job_name" ] - step_def["Arguments"]["TrainingJobDefinition"]["StaticHyperParameters"][ - "sagemaker_job_name" - ] = step_def_sm_job_name[:-24] - # delete S3 path assertions for now because job name is included with timestamp. These will be re-enabled once - # next PRs are submitted with s3 path updates, removing the job name. + # delete S3 path assertions for now because job name is included with timestamp. These will be re-enabled after + # caching improvements phase 2. del step_args["TrainingJobDefinition"]["StaticHyperParameters"]["sagemaker_submit_directory"] del step_def["Arguments"]["TrainingJobDefinition"]["StaticHyperParameters"][ "sagemaker_submit_directory" @@ -188,6 +180,16 @@ def test_tuning_step_with_single_algo_tuner(pipeline_session, training_input, en adjacency_list = PipelineGraph.from_pipeline(pipeline).adjacency_list assert adjacency_list == {"MyTuningStep": []} + # test idempotency + step_def2 = json.loads(pipeline.definition())["Steps"][0] + del step_def2["Arguments"]["TrainingJobDefinition"]["StaticHyperParameters"][ + "sagemaker_job_name" + ] + del step_def2["Arguments"]["TrainingJobDefinition"]["StaticHyperParameters"][ + "sagemaker_submit_directory" + ] + assert step_def == step_def2 + def test_tuning_step_with_multi_algo_tuner(pipeline_session, entry_point): pytorch_estimator = PyTorch( @@ -249,20 +251,14 @@ def test_tuning_step_with_multi_algo_tuner(pipeline_session, entry_point): step_def = json.loads(pipeline.definition())["Steps"][0] for i, step in enumerate(step_args["TrainingJobDefinitions"]): - # trim timestamp so sagemaker_job_name will still match - step_args_sm_job_name = step["StaticHyperParameters"]["sagemaker_job_name"] - step_args["TrainingJobDefinitions"][i]["StaticHyperParameters"][ - "sagemaker_job_name" - ] = step_args_sm_job_name[:-24] - step_def_sm_job_name = step_def["Arguments"]["TrainingJobDefinitions"][i][ - "StaticHyperParameters" - ]["sagemaker_job_name"] - step_def["Arguments"]["TrainingJobDefinitions"][i]["StaticHyperParameters"][ + # delete sagemaker_job_name b/c of timestamp collision + del step_args["TrainingJobDefinitions"][i]["StaticHyperParameters"]["sagemaker_job_name"] + del step_def["Arguments"]["TrainingJobDefinitions"][i]["StaticHyperParameters"][ "sagemaker_job_name" - ] = step_def_sm_job_name[:-24] + ] - # delete S3 path assertions for now because job name is included with timestamp. These will be re-enabled once - # next PRs are submitted with s3 path updates, removing the job name. + # delete S3 path assertions for now because job name is included with timestamp. These will be re-enabled after + # caching improvements phase 2. del step_args["TrainingJobDefinitions"][i]["StaticHyperParameters"][ "sagemaker_submit_directory" ] @@ -278,6 +274,18 @@ def test_tuning_step_with_multi_algo_tuner(pipeline_session, entry_point): adjacency_list = PipelineGraph.from_pipeline(pipeline).adjacency_list assert adjacency_list == {"MyTuningStep": []} + # test idempotency + step_def2 = json.loads(pipeline.definition())["Steps"][0] + for i, step in enumerate(step_def2["Arguments"]["TrainingJobDefinitions"]): + del step_def2["Arguments"]["TrainingJobDefinitions"][i]["StaticHyperParameters"][ + "sagemaker_job_name" + ] + + del step_def2["Arguments"]["TrainingJobDefinitions"][i]["StaticHyperParameters"][ + "sagemaker_submit_directory" + ] + assert step_def == step_def2 + @pytest.mark.parametrize( "inputs",