From 9e61f9033cdfbef155240d3afec659658fdab927 Mon Sep 17 00:00:00 2001 From: Chuyang Deng Date: Fri, 31 Jul 2020 13:15:14 -0700 Subject: [PATCH 1/2] change: upgrade TFS version and fix py_versions KeyError --- src/sagemaker/cli/framework_upgrade.py | 12 +- .../image_uri_config/tensorflow.json | 876 ++++++++++-------- .../cli/test_framework_upgrade_tool.py | 52 ++ 3 files changed, 545 insertions(+), 395 deletions(-) diff --git a/src/sagemaker/cli/framework_upgrade.py b/src/sagemaker/cli/framework_upgrade.py index 3378f8140a..8f7dafae66 100644 --- a/src/sagemaker/cli/framework_upgrade.py +++ b/src/sagemaker/cli/framework_upgrade.py @@ -41,9 +41,9 @@ def get_latest_values(existing_content, scope=None): ) latest_version = list(existing_content["versions"].keys())[-1] - registries = existing_content["versions"][latest_version]["registries"] - py_versions = existing_content["versions"][latest_version]["py_versions"] - repository = existing_content["versions"][latest_version]["repository"] + registries = existing_content["versions"][latest_version].get("registries", None) + py_versions = existing_content["versions"][latest_version].get("py_versions", None) + repository = existing_content["versions"][latest_version].get("repository", None) return registries, py_versions, repository @@ -92,8 +92,9 @@ def add_dlc_framework_version( new_version = { "registries": registries, "repository": repository, - "py_versions": py_versions, } + if py_versions: + new_version["py_versions"] = py_versions existing_content[scope]["versions"][full_version] = new_version @@ -128,10 +129,11 @@ def add_algo_version( existing_content["scope"].append(scope) new_version = { - "py_versions": py_versions, "registries": registries, "repository": repository, } + if py_versions: + new_version["py_versions"] = py_versions if tag_prefix: new_version["tag_prefix"] = tag_prefix existing_content["versions"][full_version] = new_version diff --git a/src/sagemaker/image_uri_config/tensorflow.json b/src/sagemaker/image_uri_config/tensorflow.json index 2e5b628d06..6839483996 100644 --- a/src/sagemaker/image_uri_config/tensorflow.json +++ b/src/sagemaker/image_uri_config/tensorflow.json @@ -1,25 +1,22 @@ { - "training": { - "processors": ["cpu", "gpu"], + "eia": { + "processors": [ + "cpu" + ], "version_aliases": { - "1.4": "1.4.1", - "1.5": "1.5.0", - "1.6": "1.6.0", - "1.7": "1.7.0", - "1.8": "1.8.0", - "1.9": "1.9.0", "1.10": "1.10.0", "1.11": "1.11.0", "1.12": "1.12.0", - "1.13": "1.13.1", + "1.13": "1.13.0", "1.14": "1.14.0", - "1.15": "1.15.2", - "2.0": "2.0.1", - "2.1": "2.1.0", - "2.2": "2.2.0" + "1.15": "1.15.0", + "2.0": "2.0.0" }, "versions": { - "1.4.1": { + "1.10.0": { + "py_versions": [ + "py2" + ], "registries": { "ap-east-1": "057415533634", "ap-northeast-1": "520713654638", @@ -44,10 +41,9 @@ "us-west-1": "520713654638", "us-west-2": "520713654638" }, - "repository": "sagemaker-tensorflow", - "py_versions": ["py2"] + "repository": "sagemaker-tensorflow-eia" }, - "1.5.0": { + "1.11.0": { "registries": { "ap-east-1": "057415533634", "ap-northeast-1": "520713654638", @@ -72,10 +68,9 @@ "us-west-1": "520713654638", "us-west-2": "520713654638" }, - "repository": "sagemaker-tensorflow", - "py_versions": ["py2"] + "repository": "sagemaker-tensorflow-serving-eia" }, - "1.6.0": { + "1.12.0": { "registries": { "ap-east-1": "057415533634", "ap-northeast-1": "520713654638", @@ -100,10 +95,9 @@ "us-west-1": "520713654638", "us-west-2": "520713654638" }, - "repository": "sagemaker-tensorflow", - "py_versions": ["py2"] + "repository": "sagemaker-tensorflow-serving-eia" }, - "1.7.0": { + "1.13.0": { "registries": { "ap-east-1": "057415533634", "ap-northeast-1": "520713654638", @@ -128,66 +122,118 @@ "us-west-1": "520713654638", "us-west-2": "520713654638" }, - "repository": "sagemaker-tensorflow", - "py_versions": ["py2"] + "repository": "sagemaker-tensorflow-serving-eia" }, - "1.8.0": { + "1.14.0": { "registries": { - "ap-east-1": "057415533634", - "ap-northeast-1": "520713654638", - "ap-northeast-2": "520713654638", - "ap-south-1": "520713654638", - "ap-southeast-1": "520713654638", - "ap-southeast-2": "520713654638", - "ca-central-1": "520713654638", - "cn-north-1": "422961961927", - "cn-northwest-1": "423003514399", - "eu-central-1": "520713654638", - "eu-north-1": "520713654638", - "eu-west-1": "520713654638", - "eu-west-2": "520713654638", - "eu-west-3": "520713654638", - "me-south-1": "724002660598", - "sa-east-1": "520713654638", - "us-east-1": "520713654638", - "us-east-2": "520713654638", - "us-gov-west-1": "246785580436", - "us-iso-east-1": "744548109606", - "us-west-1": "520713654638", - "us-west-2": "520713654638" + "ap-east-1": "871362719292", + "ap-northeast-1": "763104351884", + "ap-northeast-2": "763104351884", + "ap-south-1": "763104351884", + "ap-southeast-1": "763104351884", + "ap-southeast-2": "763104351884", + "ca-central-1": "763104351884", + "cn-north-1": "727897471807", + "cn-northwest-1": "727897471807", + "eu-central-1": "763104351884", + "eu-north-1": "763104351884", + "eu-west-1": "763104351884", + "eu-west-2": "763104351884", + "eu-west-3": "763104351884", + "me-south-1": "217643126080", + "sa-east-1": "763104351884", + "us-east-1": "763104351884", + "us-east-2": "763104351884", + "us-gov-west-1": "442386744353", + "us-iso-east-1": "886529160074", + "us-west-1": "763104351884", + "us-west-2": "763104351884" }, - "repository": "sagemaker-tensorflow", - "py_versions": ["py2"] + "repository": "tensorflow-inference-eia" }, - "1.9.0": { + "1.15.0": { "registries": { - "ap-east-1": "057415533634", - "ap-northeast-1": "520713654638", - "ap-northeast-2": "520713654638", - "ap-south-1": "520713654638", - "ap-southeast-1": "520713654638", - "ap-southeast-2": "520713654638", - "ca-central-1": "520713654638", - "cn-north-1": "422961961927", - "cn-northwest-1": "423003514399", - "eu-central-1": "520713654638", - "eu-north-1": "520713654638", - "eu-west-1": "520713654638", - "eu-west-2": "520713654638", - "eu-west-3": "520713654638", - "me-south-1": "724002660598", - "sa-east-1": "520713654638", - "us-east-1": "520713654638", - "us-east-2": "520713654638", - "us-gov-west-1": "246785580436", - "us-iso-east-1": "744548109606", - "us-west-1": "520713654638", - "us-west-2": "520713654638" + "ap-east-1": "871362719292", + "ap-northeast-1": "763104351884", + "ap-northeast-2": "763104351884", + "ap-south-1": "763104351884", + "ap-southeast-1": "763104351884", + "ap-southeast-2": "763104351884", + "ca-central-1": "763104351884", + "cn-north-1": "727897471807", + "cn-northwest-1": "727897471807", + "eu-central-1": "763104351884", + "eu-north-1": "763104351884", + "eu-west-1": "763104351884", + "eu-west-2": "763104351884", + "eu-west-3": "763104351884", + "me-south-1": "217643126080", + "sa-east-1": "763104351884", + "us-east-1": "763104351884", + "us-east-2": "763104351884", + "us-gov-west-1": "442386744353", + "us-iso-east-1": "886529160074", + "us-west-1": "763104351884", + "us-west-2": "763104351884" }, - "repository": "sagemaker-tensorflow", - "py_versions": ["py2"] + "repository": "tensorflow-inference-eia" }, + "2.0.0": { + "registries": { + "ap-east-1": "871362719292", + "ap-northeast-1": "763104351884", + "ap-northeast-2": "763104351884", + "ap-south-1": "763104351884", + "ap-southeast-1": "763104351884", + "ap-southeast-2": "763104351884", + "ca-central-1": "763104351884", + "cn-north-1": "727897471807", + "cn-northwest-1": "727897471807", + "eu-central-1": "763104351884", + "eu-north-1": "763104351884", + "eu-west-1": "763104351884", + "eu-west-2": "763104351884", + "eu-west-3": "763104351884", + "me-south-1": "217643126080", + "sa-east-1": "763104351884", + "us-east-1": "763104351884", + "us-east-2": "763104351884", + "us-gov-west-1": "442386744353", + "us-iso-east-1": "886529160074", + "us-west-1": "763104351884", + "us-west-2": "763104351884" + }, + "repository": "tensorflow-inference-eia" + } + } + }, + "inference": { + "processors": [ + "cpu", + "gpu" + ], + "version_aliases": { + "1.10": "1.10.0", + "1.11": "1.11.0", + "1.12": "1.12.0", + "1.13": "1.13.0", + "1.14": "1.14.0", + "1.15": "1.15.2", + "1.4": "1.4.1", + "1.5": "1.5.0", + "1.6": "1.6.0", + "1.7": "1.7.0", + "1.8": "1.8.0", + "1.9": "1.9.0", + "2.0": "2.0.1", + "2.1": "2.1.0", + "2.2": "2.2.0" + }, + "versions": { "1.10.0": { + "py_versions": [ + "py2" + ], "registries": { "ap-east-1": "057415533634", "ap-northeast-1": "520713654638", @@ -212,8 +258,7 @@ "us-west-1": "520713654638", "us-west-2": "520713654638" }, - "repository": "sagemaker-tensorflow", - "py_versions": ["py2"] + "repository": "sagemaker-tensorflow" }, "1.11.0": { "registries": { @@ -240,8 +285,7 @@ "us-west-1": "520713654638", "us-west-2": "520713654638" }, - "repository": "sagemaker-tensorflow-scriptmode", - "py_versions": ["py2", "py3"] + "repository": "sagemaker-tensorflow-serving" }, "1.12.0": { "registries": { @@ -268,66 +312,9 @@ "us-west-1": "520713654638", "us-west-2": "520713654638" }, - "repository": "sagemaker-tensorflow-scriptmode", - "py_versions": ["py2", "py3"] - }, - "1.13.1": { - "py2": { - "registries": { - "ap-east-1": "057415533634", - "ap-northeast-1": "520713654638", - "ap-northeast-2": "520713654638", - "ap-south-1": "520713654638", - "ap-southeast-1": "520713654638", - "ap-southeast-2": "520713654638", - "ca-central-1": "520713654638", - "cn-north-1": "422961961927", - "cn-northwest-1": "423003514399", - "eu-central-1": "520713654638", - "eu-north-1": "520713654638", - "eu-west-1": "520713654638", - "eu-west-2": "520713654638", - "eu-west-3": "520713654638", - "me-south-1": "724002660598", - "sa-east-1": "520713654638", - "us-east-1": "520713654638", - "us-east-2": "520713654638", - "us-gov-west-1": "246785580436", - "us-iso-east-1": "744548109606", - "us-west-1": "520713654638", - "us-west-2": "520713654638" - }, - "repository": "sagemaker-tensorflow-scriptmode" - }, - "py3": { - "registries": { - "ap-east-1": "871362719292", - "ap-northeast-1": "763104351884", - "ap-northeast-2": "763104351884", - "ap-south-1": "763104351884", - "ap-southeast-1": "763104351884", - "ap-southeast-2": "763104351884", - "ca-central-1": "763104351884", - "cn-north-1": "727897471807", - "cn-northwest-1": "727897471807", - "eu-central-1": "763104351884", - "eu-north-1": "763104351884", - "eu-west-1": "763104351884", - "eu-west-2": "763104351884", - "eu-west-3": "763104351884", - "me-south-1": "217643126080", - "sa-east-1": "763104351884", - "us-east-1": "763104351884", - "us-east-2": "763104351884", - "us-gov-west-1": "442386744353", - "us-iso-east-1": "886529160074", - "us-west-1": "763104351884", - "us-west-2": "763104351884" - }, - "repository": "tensorflow-training" - } + "repository": "sagemaker-tensorflow-serving" }, - "1.14.0": { + "1.13.0": { "registries": { "ap-east-1": "871362719292", "ap-northeast-1": "763104351884", @@ -352,10 +339,9 @@ "us-west-1": "763104351884", "us-west-2": "763104351884" }, - "repository": "tensorflow-training", - "py_versions": ["py2", "py3"] + "repository": "tensorflow-inference" }, - "1.15.0": { + "1.14.0": { "registries": { "ap-east-1": "871362719292", "ap-northeast-1": "763104351884", @@ -380,10 +366,9 @@ "us-west-1": "763104351884", "us-west-2": "763104351884" }, - "repository": "tensorflow-training", - "py_versions": ["py2", "py3"] + "repository": "tensorflow-inference" }, - "1.15.2": { + "1.15.0": { "registries": { "ap-east-1": "871362719292", "ap-northeast-1": "763104351884", @@ -408,10 +393,9 @@ "us-west-1": "763104351884", "us-west-2": "763104351884" }, - "repository": "tensorflow-training", - "py_versions": ["py2", "py3", "py37"] + "repository": "tensorflow-inference" }, - "2.0.0": { + "1.15.2": { "registries": { "ap-east-1": "871362719292", "ap-northeast-1": "763104351884", @@ -436,115 +420,12 @@ "us-west-1": "763104351884", "us-west-2": "763104351884" }, - "repository": "tensorflow-training", - "py_versions": ["py2", "py3"] + "repository": "tensorflow-inference" }, - "2.0.1": { - "registries": { - "ap-east-1": "871362719292", - "ap-northeast-1": "763104351884", - "ap-northeast-2": "763104351884", - "ap-south-1": "763104351884", - "ap-southeast-1": "763104351884", - "ap-southeast-2": "763104351884", - "ca-central-1": "763104351884", - "cn-north-1": "727897471807", - "cn-northwest-1": "727897471807", - "eu-central-1": "763104351884", - "eu-north-1": "763104351884", - "eu-west-1": "763104351884", - "eu-west-2": "763104351884", - "eu-west-3": "763104351884", - "me-south-1": "217643126080", - "sa-east-1": "763104351884", - "us-east-1": "763104351884", - "us-east-2": "763104351884", - "us-gov-west-1": "442386744353", - "us-iso-east-1": "886529160074", - "us-west-1": "763104351884", - "us-west-2": "763104351884" - }, - "repository": "tensorflow-training", - "py_versions": ["py2", "py3"] - }, - "2.1.0": { - "registries": { - "ap-east-1": "871362719292", - "ap-northeast-1": "763104351884", - "ap-northeast-2": "763104351884", - "ap-south-1": "763104351884", - "ap-southeast-1": "763104351884", - "ap-southeast-2": "763104351884", - "ca-central-1": "763104351884", - "cn-north-1": "727897471807", - "cn-northwest-1": "727897471807", - "eu-central-1": "763104351884", - "eu-north-1": "763104351884", - "eu-west-1": "763104351884", - "eu-west-2": "763104351884", - "eu-west-3": "763104351884", - "me-south-1": "217643126080", - "sa-east-1": "763104351884", - "us-east-1": "763104351884", - "us-east-2": "763104351884", - "us-gov-west-1": "442386744353", - "us-iso-east-1": "886529160074", - "us-west-1": "763104351884", - "us-west-2": "763104351884" - }, - "repository": "tensorflow-training", - "py_versions": ["py2", "py3"] - }, - "2.2.0": { - "registries": { - "ap-east-1": "871362719292", - "ap-northeast-1": "763104351884", - "ap-northeast-2": "763104351884", - "ap-south-1": "763104351884", - "ap-southeast-1": "763104351884", - "ap-southeast-2": "763104351884", - "ca-central-1": "763104351884", - "cn-north-1": "727897471807", - "cn-northwest-1": "727897471807", - "eu-central-1": "763104351884", - "eu-north-1": "763104351884", - "eu-west-1": "763104351884", - "eu-west-2": "763104351884", - "eu-west-3": "763104351884", - "me-south-1": "217643126080", - "sa-east-1": "763104351884", - "us-east-1": "763104351884", - "us-east-2": "763104351884", - "us-gov-west-1": "442386744353", - "us-iso-east-1": "886529160074", - "us-west-1": "763104351884", - "us-west-2": "763104351884" - }, - "repository": "tensorflow-training", - "py_versions": ["py37"] - } - } - }, - "inference": { - "processors": ["cpu", "gpu"], - "version_aliases": { - "1.4": "1.4.1", - "1.5": "1.5.0", - "1.6": "1.6.0", - "1.7": "1.7.0", - "1.8": "1.8.0", - "1.9": "1.9.0", - "1.10": "1.10.0", - "1.11": "1.11.0", - "1.12": "1.12.0", - "1.13": "1.13.0", - "1.14": "1.14.0", - "1.15": "1.15.2", - "2.0": "2.0.1", - "2.1": "2.1.0" - }, - "versions": { - "1.4.1": { + "1.4.1": { + "py_versions": [ + "py2" + ], "registries": { "ap-east-1": "057415533634", "ap-northeast-1": "520713654638", @@ -569,10 +450,12 @@ "us-west-1": "520713654638", "us-west-2": "520713654638" }, - "repository": "sagemaker-tensorflow", - "py_versions": ["py2"] + "repository": "sagemaker-tensorflow" }, "1.5.0": { + "py_versions": [ + "py2" + ], "registries": { "ap-east-1": "057415533634", "ap-northeast-1": "520713654638", @@ -597,10 +480,12 @@ "us-west-1": "520713654638", "us-west-2": "520713654638" }, - "repository": "sagemaker-tensorflow", - "py_versions": ["py2"] + "repository": "sagemaker-tensorflow" }, "1.6.0": { + "py_versions": [ + "py2" + ], "registries": { "ap-east-1": "057415533634", "ap-northeast-1": "520713654638", @@ -625,10 +510,12 @@ "us-west-1": "520713654638", "us-west-2": "520713654638" }, - "repository": "sagemaker-tensorflow", - "py_versions": ["py2"] + "repository": "sagemaker-tensorflow" }, "1.7.0": { + "py_versions": [ + "py2" + ], "registries": { "ap-east-1": "057415533634", "ap-northeast-1": "520713654638", @@ -653,10 +540,12 @@ "us-west-1": "520713654638", "us-west-2": "520713654638" }, - "repository": "sagemaker-tensorflow", - "py_versions": ["py2"] + "repository": "sagemaker-tensorflow" }, "1.8.0": { + "py_versions": [ + "py2" + ], "registries": { "ap-east-1": "057415533634", "ap-northeast-1": "520713654638", @@ -681,10 +570,12 @@ "us-west-1": "520713654638", "us-west-2": "520713654638" }, - "repository": "sagemaker-tensorflow", - "py_versions": ["py2"] + "repository": "sagemaker-tensorflow" }, "1.9.0": { + "py_versions": [ + "py2" + ], "registries": { "ap-east-1": "057415533634", "ap-northeast-1": "520713654638", @@ -709,92 +600,9 @@ "us-west-1": "520713654638", "us-west-2": "520713654638" }, - "repository": "sagemaker-tensorflow", - "py_versions": ["py2"] - }, - "1.10.0": { - "registries": { - "ap-east-1": "057415533634", - "ap-northeast-1": "520713654638", - "ap-northeast-2": "520713654638", - "ap-south-1": "520713654638", - "ap-southeast-1": "520713654638", - "ap-southeast-2": "520713654638", - "ca-central-1": "520713654638", - "cn-north-1": "422961961927", - "cn-northwest-1": "423003514399", - "eu-central-1": "520713654638", - "eu-north-1": "520713654638", - "eu-west-1": "520713654638", - "eu-west-2": "520713654638", - "eu-west-3": "520713654638", - "me-south-1": "724002660598", - "sa-east-1": "520713654638", - "us-east-1": "520713654638", - "us-east-2": "520713654638", - "us-gov-west-1": "246785580436", - "us-iso-east-1": "744548109606", - "us-west-1": "520713654638", - "us-west-2": "520713654638" - }, - "repository": "sagemaker-tensorflow", - "py_versions": ["py2"] - }, - "1.11.0": { - "registries": { - "ap-east-1": "057415533634", - "ap-northeast-1": "520713654638", - "ap-northeast-2": "520713654638", - "ap-south-1": "520713654638", - "ap-southeast-1": "520713654638", - "ap-southeast-2": "520713654638", - "ca-central-1": "520713654638", - "cn-north-1": "422961961927", - "cn-northwest-1": "423003514399", - "eu-central-1": "520713654638", - "eu-north-1": "520713654638", - "eu-west-1": "520713654638", - "eu-west-2": "520713654638", - "eu-west-3": "520713654638", - "me-south-1": "724002660598", - "sa-east-1": "520713654638", - "us-east-1": "520713654638", - "us-east-2": "520713654638", - "us-gov-west-1": "246785580436", - "us-iso-east-1": "744548109606", - "us-west-1": "520713654638", - "us-west-2": "520713654638" - }, - "repository": "sagemaker-tensorflow-serving" - }, - "1.12.0": { - "registries": { - "ap-east-1": "057415533634", - "ap-northeast-1": "520713654638", - "ap-northeast-2": "520713654638", - "ap-south-1": "520713654638", - "ap-southeast-1": "520713654638", - "ap-southeast-2": "520713654638", - "ca-central-1": "520713654638", - "cn-north-1": "422961961927", - "cn-northwest-1": "423003514399", - "eu-central-1": "520713654638", - "eu-north-1": "520713654638", - "eu-west-1": "520713654638", - "eu-west-2": "520713654638", - "eu-west-3": "520713654638", - "me-south-1": "724002660598", - "sa-east-1": "520713654638", - "us-east-1": "520713654638", - "us-east-2": "520713654638", - "us-gov-west-1": "246785580436", - "us-iso-east-1": "744548109606", - "us-west-1": "520713654638", - "us-west-2": "520713654638" - }, - "repository": "sagemaker-tensorflow-serving" + "repository": "sagemaker-tensorflow" }, - "1.13.0": { + "2.0.0": { "registries": { "ap-east-1": "871362719292", "ap-northeast-1": "763104351884", @@ -821,7 +629,7 @@ }, "repository": "tensorflow-inference" }, - "1.14.0": { + "2.0.1": { "registries": { "ap-east-1": "871362719292", "ap-northeast-1": "763104351884", @@ -848,7 +656,7 @@ }, "repository": "tensorflow-inference" }, - "1.15.0": { + "2.1.0": { "registries": { "ap-east-1": "871362719292", "ap-northeast-1": "763104351884", @@ -875,7 +683,10 @@ }, "repository": "tensorflow-inference" }, - "1.15.2": { + "2.2.0": { + "py_versions": [ + "py3" + ], "registries": { "ap-east-1": "871362719292", "ap-northeast-1": "763104351884", @@ -901,8 +712,185 @@ "us-west-2": "763104351884" }, "repository": "tensorflow-inference" + } + } + }, + "training": { + "processors": [ + "cpu", + "gpu" + ], + "version_aliases": { + "1.10": "1.10.0", + "1.11": "1.11.0", + "1.12": "1.12.0", + "1.13": "1.13.1", + "1.14": "1.14.0", + "1.15": "1.15.2", + "1.4": "1.4.1", + "1.5": "1.5.0", + "1.6": "1.6.0", + "1.7": "1.7.0", + "1.8": "1.8.0", + "1.9": "1.9.0", + "2.0": "2.0.1", + "2.1": "2.1.0", + "2.2": "2.2.0" + }, + "versions": { + "1.10.0": { + "py_versions": [ + "py2" + ], + "registries": { + "ap-east-1": "057415533634", + "ap-northeast-1": "520713654638", + "ap-northeast-2": "520713654638", + "ap-south-1": "520713654638", + "ap-southeast-1": "520713654638", + "ap-southeast-2": "520713654638", + "ca-central-1": "520713654638", + "cn-north-1": "422961961927", + "cn-northwest-1": "423003514399", + "eu-central-1": "520713654638", + "eu-north-1": "520713654638", + "eu-west-1": "520713654638", + "eu-west-2": "520713654638", + "eu-west-3": "520713654638", + "me-south-1": "724002660598", + "sa-east-1": "520713654638", + "us-east-1": "520713654638", + "us-east-2": "520713654638", + "us-gov-west-1": "246785580436", + "us-iso-east-1": "744548109606", + "us-west-1": "520713654638", + "us-west-2": "520713654638" + }, + "repository": "sagemaker-tensorflow" }, - "2.0.0": { + "1.11.0": { + "py_versions": [ + "py2", + "py3" + ], + "registries": { + "ap-east-1": "057415533634", + "ap-northeast-1": "520713654638", + "ap-northeast-2": "520713654638", + "ap-south-1": "520713654638", + "ap-southeast-1": "520713654638", + "ap-southeast-2": "520713654638", + "ca-central-1": "520713654638", + "cn-north-1": "422961961927", + "cn-northwest-1": "423003514399", + "eu-central-1": "520713654638", + "eu-north-1": "520713654638", + "eu-west-1": "520713654638", + "eu-west-2": "520713654638", + "eu-west-3": "520713654638", + "me-south-1": "724002660598", + "sa-east-1": "520713654638", + "us-east-1": "520713654638", + "us-east-2": "520713654638", + "us-gov-west-1": "246785580436", + "us-iso-east-1": "744548109606", + "us-west-1": "520713654638", + "us-west-2": "520713654638" + }, + "repository": "sagemaker-tensorflow-scriptmode" + }, + "1.12.0": { + "py_versions": [ + "py2", + "py3" + ], + "registries": { + "ap-east-1": "057415533634", + "ap-northeast-1": "520713654638", + "ap-northeast-2": "520713654638", + "ap-south-1": "520713654638", + "ap-southeast-1": "520713654638", + "ap-southeast-2": "520713654638", + "ca-central-1": "520713654638", + "cn-north-1": "422961961927", + "cn-northwest-1": "423003514399", + "eu-central-1": "520713654638", + "eu-north-1": "520713654638", + "eu-west-1": "520713654638", + "eu-west-2": "520713654638", + "eu-west-3": "520713654638", + "me-south-1": "724002660598", + "sa-east-1": "520713654638", + "us-east-1": "520713654638", + "us-east-2": "520713654638", + "us-gov-west-1": "246785580436", + "us-iso-east-1": "744548109606", + "us-west-1": "520713654638", + "us-west-2": "520713654638" + }, + "repository": "sagemaker-tensorflow-scriptmode" + }, + "1.13.1": { + "py2": { + "registries": { + "ap-east-1": "057415533634", + "ap-northeast-1": "520713654638", + "ap-northeast-2": "520713654638", + "ap-south-1": "520713654638", + "ap-southeast-1": "520713654638", + "ap-southeast-2": "520713654638", + "ca-central-1": "520713654638", + "cn-north-1": "422961961927", + "cn-northwest-1": "423003514399", + "eu-central-1": "520713654638", + "eu-north-1": "520713654638", + "eu-west-1": "520713654638", + "eu-west-2": "520713654638", + "eu-west-3": "520713654638", + "me-south-1": "724002660598", + "sa-east-1": "520713654638", + "us-east-1": "520713654638", + "us-east-2": "520713654638", + "us-gov-west-1": "246785580436", + "us-iso-east-1": "744548109606", + "us-west-1": "520713654638", + "us-west-2": "520713654638" + }, + "repository": "sagemaker-tensorflow-scriptmode" + }, + "py3": { + "registries": { + "ap-east-1": "871362719292", + "ap-northeast-1": "763104351884", + "ap-northeast-2": "763104351884", + "ap-south-1": "763104351884", + "ap-southeast-1": "763104351884", + "ap-southeast-2": "763104351884", + "ca-central-1": "763104351884", + "cn-north-1": "727897471807", + "cn-northwest-1": "727897471807", + "eu-central-1": "763104351884", + "eu-north-1": "763104351884", + "eu-west-1": "763104351884", + "eu-west-2": "763104351884", + "eu-west-3": "763104351884", + "me-south-1": "217643126080", + "sa-east-1": "763104351884", + "us-east-1": "763104351884", + "us-east-2": "763104351884", + "us-gov-west-1": "442386744353", + "us-iso-east-1": "886529160074", + "us-west-1": "763104351884", + "us-west-2": "763104351884" + }, + "repository": "tensorflow-training" + } + }, + "1.14.0": { + "py_versions": [ + "py2", + "py3" + ], "registries": { "ap-east-1": "871362719292", "ap-northeast-1": "763104351884", @@ -927,9 +915,13 @@ "us-west-1": "763104351884", "us-west-2": "763104351884" }, - "repository": "tensorflow-inference" + "repository": "tensorflow-training" }, - "2.0.1": { + "1.15.0": { + "py_versions": [ + "py2", + "py3" + ], "registries": { "ap-east-1": "871362719292", "ap-northeast-1": "763104351884", @@ -954,9 +946,14 @@ "us-west-1": "763104351884", "us-west-2": "763104351884" }, - "repository": "tensorflow-inference" + "repository": "tensorflow-training" }, - "2.1.0": { + "1.15.2": { + "py_versions": [ + "py2", + "py3", + "py37" + ], "registries": { "ap-east-1": "871362719292", "ap-northeast-1": "763104351884", @@ -981,23 +978,12 @@ "us-west-1": "763104351884", "us-west-2": "763104351884" }, - "repository": "tensorflow-inference" - } - } - }, - "eia": { - "processors": ["cpu"], - "version_aliases": { - "1.10": "1.10.0", - "1.11": "1.11.0", - "1.12": "1.12.0", - "1.13": "1.13.0", - "1.14": "1.14.0", - "1.15": "1.15.0", - "2.0": "2.0.0" - }, - "versions": { - "1.10.0": { + "repository": "tensorflow-training" + }, + "1.4.1": { + "py_versions": [ + "py2" + ], "registries": { "ap-east-1": "057415533634", "ap-northeast-1": "520713654638", @@ -1022,10 +1008,12 @@ "us-west-1": "520713654638", "us-west-2": "520713654638" }, - "repository": "sagemaker-tensorflow-eia", - "py_versions": ["py2"] + "repository": "sagemaker-tensorflow" }, - "1.11.0": { + "1.5.0": { + "py_versions": [ + "py2" + ], "registries": { "ap-east-1": "057415533634", "ap-northeast-1": "520713654638", @@ -1050,9 +1038,12 @@ "us-west-1": "520713654638", "us-west-2": "520713654638" }, - "repository": "sagemaker-tensorflow-serving-eia" + "repository": "sagemaker-tensorflow" }, - "1.12.0": { + "1.6.0": { + "py_versions": [ + "py2" + ], "registries": { "ap-east-1": "057415533634", "ap-northeast-1": "520713654638", @@ -1077,9 +1068,12 @@ "us-west-1": "520713654638", "us-west-2": "520713654638" }, - "repository": "sagemaker-tensorflow-serving-eia" + "repository": "sagemaker-tensorflow" }, - "1.13.0": { + "1.7.0": { + "py_versions": [ + "py2" + ], "registries": { "ap-east-1": "057415533634", "ap-northeast-1": "520713654638", @@ -1104,9 +1098,73 @@ "us-west-1": "520713654638", "us-west-2": "520713654638" }, - "repository": "sagemaker-tensorflow-serving-eia" + "repository": "sagemaker-tensorflow" }, - "1.14.0": { + "1.8.0": { + "py_versions": [ + "py2" + ], + "registries": { + "ap-east-1": "057415533634", + "ap-northeast-1": "520713654638", + "ap-northeast-2": "520713654638", + "ap-south-1": "520713654638", + "ap-southeast-1": "520713654638", + "ap-southeast-2": "520713654638", + "ca-central-1": "520713654638", + "cn-north-1": "422961961927", + "cn-northwest-1": "423003514399", + "eu-central-1": "520713654638", + "eu-north-1": "520713654638", + "eu-west-1": "520713654638", + "eu-west-2": "520713654638", + "eu-west-3": "520713654638", + "me-south-1": "724002660598", + "sa-east-1": "520713654638", + "us-east-1": "520713654638", + "us-east-2": "520713654638", + "us-gov-west-1": "246785580436", + "us-iso-east-1": "744548109606", + "us-west-1": "520713654638", + "us-west-2": "520713654638" + }, + "repository": "sagemaker-tensorflow" + }, + "1.9.0": { + "py_versions": [ + "py2" + ], + "registries": { + "ap-east-1": "057415533634", + "ap-northeast-1": "520713654638", + "ap-northeast-2": "520713654638", + "ap-south-1": "520713654638", + "ap-southeast-1": "520713654638", + "ap-southeast-2": "520713654638", + "ca-central-1": "520713654638", + "cn-north-1": "422961961927", + "cn-northwest-1": "423003514399", + "eu-central-1": "520713654638", + "eu-north-1": "520713654638", + "eu-west-1": "520713654638", + "eu-west-2": "520713654638", + "eu-west-3": "520713654638", + "me-south-1": "724002660598", + "sa-east-1": "520713654638", + "us-east-1": "520713654638", + "us-east-2": "520713654638", + "us-gov-west-1": "246785580436", + "us-iso-east-1": "744548109606", + "us-west-1": "520713654638", + "us-west-2": "520713654638" + }, + "repository": "sagemaker-tensorflow" + }, + "2.0.0": { + "py_versions": [ + "py2", + "py3" + ], "registries": { "ap-east-1": "871362719292", "ap-northeast-1": "763104351884", @@ -1131,9 +1189,13 @@ "us-west-1": "763104351884", "us-west-2": "763104351884" }, - "repository": "tensorflow-inference-eia" + "repository": "tensorflow-training" }, - "1.15.0": { + "2.0.1": { + "py_versions": [ + "py2", + "py3" + ], "registries": { "ap-east-1": "871362719292", "ap-northeast-1": "763104351884", @@ -1158,9 +1220,13 @@ "us-west-1": "763104351884", "us-west-2": "763104351884" }, - "repository": "tensorflow-inference-eia" + "repository": "tensorflow-training" }, - "2.0.0": { + "2.1.0": { + "py_versions": [ + "py2", + "py3" + ], "registries": { "ap-east-1": "871362719292", "ap-northeast-1": "763104351884", @@ -1185,8 +1251,38 @@ "us-west-1": "763104351884", "us-west-2": "763104351884" }, - "repository": "tensorflow-inference-eia" + "repository": "tensorflow-training" + }, + "2.2.0": { + "py_versions": [ + "py37" + ], + "registries": { + "ap-east-1": "871362719292", + "ap-northeast-1": "763104351884", + "ap-northeast-2": "763104351884", + "ap-south-1": "763104351884", + "ap-southeast-1": "763104351884", + "ap-southeast-2": "763104351884", + "ca-central-1": "763104351884", + "cn-north-1": "727897471807", + "cn-northwest-1": "727897471807", + "eu-central-1": "763104351884", + "eu-north-1": "763104351884", + "eu-west-1": "763104351884", + "eu-west-2": "763104351884", + "eu-west-3": "763104351884", + "me-south-1": "217643126080", + "sa-east-1": "763104351884", + "us-east-1": "763104351884", + "us-east-2": "763104351884", + "us-gov-west-1": "442386744353", + "us-iso-east-1": "886529160074", + "us-west-1": "763104351884", + "us-west-2": "763104351884" + }, + "repository": "tensorflow-training" } } } -} +} \ No newline at end of file diff --git a/tests/unit/sagemaker/cli/test_framework_upgrade_tool.py b/tests/unit/sagemaker/cli/test_framework_upgrade_tool.py index ec56b99a20..8a15065d71 100644 --- a/tests/unit/sagemaker/cli/test_framework_upgrade_tool.py +++ b/tests/unit/sagemaker/cli/test_framework_upgrade_tool.py @@ -159,6 +159,38 @@ def algo_expected_region_content(): return region_info +@pytest.fixture +def dlc_no_optional_content(): + content = { + "inference": { + "processors": ["cpu"], + "version_aliases": {"1.0": "1.0.0"}, + "versions": { + "1.0.0": { + "registries": FRAMEWORK_REGION_REGISTRY, + "repository": "tensorflow-inference", + } + }, + } + } + return content + + +@pytest.fixture +def algo_no_optional_content(): + content = { + "processors": ["cpu"], + "scope": ["training"], + "versions": { + "0.10": { + "registries": FRAMEWORK_REGION_REGISTRY, + "repository": "sagemaker-rl-tensorflow", + } + }, + } + return content + + def test_add_tensorflow_eia(dlc_content, dlc_expected_content): latest_repository = "tensorflow-inference-eia" processors = ["cpu", "gpu"] @@ -237,3 +269,23 @@ def test_algo_get_latest_content(algo_content): assert registries == algo_content["versions"][latest_version]["registries"] assert py_versions == algo_content["versions"][latest_version]["py_versions"] assert repository == algo_content["versions"][latest_version]["repository"] + + +def test_dlc_get_latest_content_no_optional(dlc_no_optional_content): + latest_version = "1.0.0" + scope = "inference" + registries, py_versions, repository = framework_upgrade.get_latest_values( + dlc_no_optional_content, scope=scope + ) + assert py_versions is None + assert registries == dlc_no_optional_content[scope]["versions"][latest_version]["registries"] + assert repository == dlc_no_optional_content[scope]["versions"][latest_version]["repository"] + + +def test_algo_get_latest_content_no_optional(algo_no_optional_content): + latest_version = "0.10" + registries, py_versions, repository = framework_upgrade.get_latest_values(algo_no_optional_content) + assert py_versions is None + assert registries == algo_no_optional_content["versions"][latest_version]["registries"] + assert repository == algo_no_optional_content["versions"][latest_version]["repository"] + From e4c21ad714610665fab9189d869e5f2043027093 Mon Sep 17 00:00:00 2001 From: Chuyang Deng Date: Fri, 31 Jul 2020 14:08:33 -0700 Subject: [PATCH 2/2] update tests --- src/sagemaker/cli/framework_upgrade.py | 3 ++- src/sagemaker/image_uri_config/tensorflow.json | 3 --- tests/unit/sagemaker/cli/test_framework_upgrade_tool.py | 5 +++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/sagemaker/cli/framework_upgrade.py b/src/sagemaker/cli/framework_upgrade.py index 8f7dafae66..81247c5325 100644 --- a/src/sagemaker/cli/framework_upgrade.py +++ b/src/sagemaker/cli/framework_upgrade.py @@ -173,7 +173,8 @@ def add_version( py_versions (str): Supported Python versions (e.g. "py3,py37"). tag_prefix (str): Algorithm image's tag prefix. """ - py_versions = py_versions.split(",") + if py_versions: + py_versions = py_versions.split(",") processors = processors.split(",") latest_registries, latest_py_versions, latest_repository = get_latest_values( existing_content, scope diff --git a/src/sagemaker/image_uri_config/tensorflow.json b/src/sagemaker/image_uri_config/tensorflow.json index 6839483996..2271cf43dd 100644 --- a/src/sagemaker/image_uri_config/tensorflow.json +++ b/src/sagemaker/image_uri_config/tensorflow.json @@ -684,9 +684,6 @@ "repository": "tensorflow-inference" }, "2.2.0": { - "py_versions": [ - "py3" - ], "registries": { "ap-east-1": "871362719292", "ap-northeast-1": "763104351884", diff --git a/tests/unit/sagemaker/cli/test_framework_upgrade_tool.py b/tests/unit/sagemaker/cli/test_framework_upgrade_tool.py index 8a15065d71..d1026e3d1f 100644 --- a/tests/unit/sagemaker/cli/test_framework_upgrade_tool.py +++ b/tests/unit/sagemaker/cli/test_framework_upgrade_tool.py @@ -284,8 +284,9 @@ def test_dlc_get_latest_content_no_optional(dlc_no_optional_content): def test_algo_get_latest_content_no_optional(algo_no_optional_content): latest_version = "0.10" - registries, py_versions, repository = framework_upgrade.get_latest_values(algo_no_optional_content) + registries, py_versions, repository = framework_upgrade.get_latest_values( + algo_no_optional_content + ) assert py_versions is None assert registries == algo_no_optional_content["versions"][latest_version]["registries"] assert repository == algo_no_optional_content["versions"][latest_version]["repository"] -