From f383e88db3ea16181d89511900d568b7a3e28da9 Mon Sep 17 00:00:00 2001 From: Xinyun-123 Date: Wed, 15 Oct 2025 18:51:59 +0000 Subject: [PATCH] pysdk upgrade for python 3.10 and version mapping --- src/sagemaker/image_uri_config/sklearn.json | 96 +++++++++++++++++++ src/sagemaker/image_uri_config/xgboost.json | 84 ++++++++++++++++ src/sagemaker/image_uris.py | 12 ++- .../serve/detector/image_detector.py | 8 +- src/sagemaker/utils.py | 32 +++++++ tests/conftest.py | 3 +- 6 files changed, 226 insertions(+), 9 deletions(-) diff --git a/src/sagemaker/image_uri_config/sklearn.json b/src/sagemaker/image_uri_config/sklearn.json index 85114a11d2..6f7aed13d4 100644 --- a/src/sagemaker/image_uri_config/sklearn.json +++ b/src/sagemaker/image_uri_config/sklearn.json @@ -192,6 +192,54 @@ "us-west-2": "246618743249" }, "repository": "sagemaker-scikit-learn" + }, + "1.2-1-1": { + "processors": [ + "cpu" + ], + "py_versions": [ + "py3" + ], + "registries": { + "af-south-1": "510948584623", + "ap-east-1": "651117190479", + "ap-northeast-1": "354813040037", + "ap-northeast-2": "366743142698", + "ap-northeast-3": "867004704886", + "ap-south-1": "720646828776", + "ap-south-2": "628508329040", + "ap-southeast-1": "121021644041", + "ap-southeast-2": "783357654285", + "ap-southeast-3": "951798379941", + "ap-southeast-4": "106583098589", + "ca-central-1": "341280168497", + "ca-west-1": "190319476487", + "cn-north-1": "450853457545", + "cn-northwest-1": "451049120500", + "eu-central-1": "492215442770", + "eu-central-2": "680994064768", + "eu-north-1": "662702820516", + "eu-south-1": "978288397137", + "eu-south-2": "104374241257", + "eu-west-1": "141502667606", + "eu-west-2": "764974769150", + "eu-west-3": "659782779980", + "il-central-1": "898809789911", + "me-central-1": "272398656194", + "me-south-1": "801668240914", + "sa-east-1": "737474898029", + "us-east-1": "683313688378", + "us-east-2": "257758044811", + "us-gov-east-1": "237065988967", + "us-gov-west-1": "414596584902", + "us-iso-east-1": "833128469047", + "us-isob-east-1": "281123927165", + "us-isof-east-1": "108575199400", + "us-isof-south-1": "124985052026", + "us-west-1": "746614075791", + "us-west-2": "246618743249" + }, + "repository": "sagemaker-scikit-learn" } } }, @@ -388,6 +436,54 @@ "us-west-2": "246618743249" }, "repository": "sagemaker-scikit-learn" + }, + "1.2-1-1": { + "processors": [ + "cpu" + ], + "py_versions": [ + "py3" + ], + "registries": { + "af-south-1": "510948584623", + "ap-east-1": "651117190479", + "ap-northeast-1": "354813040037", + "ap-northeast-2": "366743142698", + "ap-northeast-3": "867004704886", + "ap-south-1": "720646828776", + "ap-south-2": "628508329040", + "ap-southeast-1": "121021644041", + "ap-southeast-2": "783357654285", + "ap-southeast-3": "951798379941", + "ap-southeast-4": "106583098589", + "ca-central-1": "341280168497", + "ca-west-1": "190319476487", + "cn-north-1": "450853457545", + "cn-northwest-1": "451049120500", + "eu-central-1": "492215442770", + "eu-central-2": "680994064768", + "eu-north-1": "662702820516", + "eu-south-1": "978288397137", + "eu-south-2": "104374241257", + "eu-west-1": "141502667606", + "eu-west-2": "764974769150", + "eu-west-3": "659782779980", + "il-central-1": "898809789911", + "me-central-1": "272398656194", + "me-south-1": "801668240914", + "sa-east-1": "737474898029", + "us-east-1": "683313688378", + "us-east-2": "257758044811", + "us-gov-east-1": "237065988967", + "us-gov-west-1": "414596584902", + "us-iso-east-1": "833128469047", + "us-isob-east-1": "281123927165", + "us-isof-east-1": "108575199400", + "us-isof-south-1": "124985052026", + "us-west-1": "746614075791", + "us-west-2": "246618743249" + }, + "repository": "sagemaker-scikit-learn" } } }, diff --git a/src/sagemaker/image_uri_config/xgboost.json b/src/sagemaker/image_uri_config/xgboost.json index 88d621af49..acd68e1ed4 100644 --- a/src/sagemaker/image_uri_config/xgboost.json +++ b/src/sagemaker/image_uri_config/xgboost.json @@ -395,6 +395,48 @@ "us-west-2": "246618743249" }, "repository": "sagemaker-xgboost" + }, + "1.7-1-1": { + "registries": { + "af-south-1": "510948584623", + "ap-east-1": "651117190479", + "ap-northeast-1": "354813040037", + "ap-northeast-2": "366743142698", + "ap-northeast-3": "867004704886", + "ap-south-1": "720646828776", + "ap-south-2": "628508329040", + "ap-southeast-1": "121021644041", + "ap-southeast-2": "783357654285", + "ap-southeast-3": "951798379941", + "ap-southeast-4": "106583098589", + "ca-central-1": "341280168497", + "ca-west-1": "190319476487", + "cn-north-1": "450853457545", + "cn-northwest-1": "451049120500", + "eu-central-1": "492215442770", + "eu-central-2": "680994064768", + "eu-north-1": "662702820516", + "eu-south-1": "978288397137", + "eu-south-2": "104374241257", + "eu-west-1": "141502667606", + "eu-west-2": "764974769150", + "eu-west-3": "659782779980", + "il-central-1": "898809789911", + "me-central-1": "272398656194", + "me-south-1": "801668240914", + "sa-east-1": "737474898029", + "us-east-1": "683313688378", + "us-east-2": "257758044811", + "us-gov-east-1": "237065988967", + "us-gov-west-1": "414596584902", + "us-iso-east-1": "833128469047", + "us-isob-east-1": "281123927165", + "us-isof-east-1": "108575199400", + "us-isof-south-1": "124985052026", + "us-west-1": "746614075791", + "us-west-2": "246618743249" + }, + "repository": "sagemaker-xgboost" } } }, @@ -794,6 +836,48 @@ "us-west-2": "246618743249" }, "repository": "sagemaker-xgboost" + }, + "1.7-1-1": { + "registries": { + "af-south-1": "510948584623", + "ap-east-1": "651117190479", + "ap-northeast-1": "354813040037", + "ap-northeast-2": "366743142698", + "ap-northeast-3": "867004704886", + "ap-south-1": "720646828776", + "ap-south-2": "628508329040", + "ap-southeast-1": "121021644041", + "ap-southeast-2": "783357654285", + "ap-southeast-3": "951798379941", + "ap-southeast-4": "106583098589", + "ca-central-1": "341280168497", + "ca-west-1": "190319476487", + "cn-north-1": "450853457545", + "cn-northwest-1": "451049120500", + "eu-central-1": "492215442770", + "eu-central-2": "680994064768", + "eu-north-1": "662702820516", + "eu-south-1": "978288397137", + "eu-south-2": "104374241257", + "eu-west-1": "141502667606", + "eu-west-2": "764974769150", + "eu-west-3": "659782779980", + "il-central-1": "898809789911", + "me-central-1": "272398656194", + "me-south-1": "801668240914", + "sa-east-1": "737474898029", + "us-east-1": "683313688378", + "us-east-2": "257758044811", + "us-gov-east-1": "237065988967", + "us-gov-west-1": "414596584902", + "us-iso-east-1": "833128469047", + "us-isob-east-1": "281123927165", + "us-isof-east-1": "108575199400", + "us-isof-south-1": "124985052026", + "us-west-1": "746614075791", + "us-west-2": "246618743249" + }, + "repository": "sagemaker-xgboost" } } }, diff --git a/src/sagemaker/image_uris.py b/src/sagemaker/image_uris.py index de6d622f78..eb5fbb1cba 100644 --- a/src/sagemaker/image_uris.py +++ b/src/sagemaker/image_uris.py @@ -18,7 +18,7 @@ import os import re from typing import Optional -from packaging.version import Version +from sagemaker.utils import parse_sagemaker_version from sagemaker import utils from sagemaker.jumpstart.constants import DEFAULT_JUMPSTART_SAGEMAKER_SESSION, JUMPSTART_LOGGER @@ -403,8 +403,12 @@ def _config_for_framework_and_scope(framework, image_scope, accelerator_type=Non def _validate_instance_deprecation(framework, instance_type, version): """Check if instance type is deprecated for a certain framework with a certain version""" if utils.get_instance_type_family(instance_type) == "p2": - if (framework == "pytorch" and Version(version) >= Version("1.13")) or ( - framework == "tensorflow" and Version(version) >= Version("2.12") + if ( + framework == "pytorch" + and parse_sagemaker_version(version) >= parse_sagemaker_version("1.13") + ) or ( + framework == "tensorflow" + and parse_sagemaker_version(version) >= parse_sagemaker_version("2.12") ): raise ValueError( "P2 instances have been deprecated for sagemaker jobs starting PyTorch 1.13 and TensorFlow 2.12" @@ -804,7 +808,7 @@ def _fetch_latest_version_from_config( # pylint: disable=R0911 bottom_number = int(bottom_version[:-2]) max_version = max(top_number, bottom_number) return f"{max_version}.x" - if Version(top_version) >= Version(bottom_version): + if parse_sagemaker_version(top_version) >= parse_sagemaker_version(bottom_version): return top_version return bottom_version diff --git a/src/sagemaker/serve/detector/image_detector.py b/src/sagemaker/serve/detector/image_detector.py index d8bee9deb8..d575649734 100644 --- a/src/sagemaker/serve/detector/image_detector.py +++ b/src/sagemaker/serve/detector/image_detector.py @@ -5,7 +5,7 @@ import platform import re import logging -from packaging import version +from sagemaker.utils import parse_sagemaker_version from sagemaker import image_uris logger = logging.getLogger(__name__) @@ -74,7 +74,7 @@ def auto_detect_container(model, region: str, instance_type: str) -> str: def _cast_to_compatible_version(framework: str, fw_version: str) -> Tuple[str]: """Given fw_version, detect the available versions""" config = image_uris._config_for_framework_and_scope(framework, "inference", None) - available_versions = [version.parse(ver) for ver in list(config["versions"].keys())] + available_versions = [parse_sagemaker_version(ver) for ver in list(config["versions"].keys())] available_versions.sort() earliest_upcast_version = None @@ -82,7 +82,7 @@ def _cast_to_compatible_version(framework: str, fw_version: str) -> Tuple[str]: latest_downcast_version = None major_version_pattern = r"^(\d+)" - parsed_fw_version = version.parse(fw_version) + parsed_fw_version = parse_sagemaker_version(fw_version) major_verson = int(re.match(major_version_pattern, str(parsed_fw_version)).group(1)) for available_version in available_versions: @@ -119,7 +119,7 @@ def _cast_to_compatible_version(framework: str, fw_version: str) -> Tuple[str]: return (exact_match_version, latest_downcast_version, earliest_upcast_version) -def _process_version(ver: version.Version) -> str: +def _process_version(ver) -> str: """Placeholder docstring""" if not ver: return None diff --git a/src/sagemaker/utils.py b/src/sagemaker/utils.py index af3cc16f1e..041a252159 100644 --- a/src/sagemaker/utils.py +++ b/src/sagemaker/utils.py @@ -53,6 +53,12 @@ from sagemaker.workflow import is_pipeline_parameter_string, is_pipeline_variable from sagemaker.workflow.entities import PipelineVariable +try: + from packaging.version import Version, InvalidVersion +except ImportError: + Version = None + InvalidVersion = None + ALTERNATE_DOMAINS = { "cn-north-1": "amazonaws.com.cn", "cn-northwest-1": "amazonaws.com.cn", @@ -1959,3 +1965,29 @@ def get_domain_for_region(region: str) -> str: region (str): AWS region name. """ return ALTERNATE_DOMAINS.get(region, "amazonaws.com") + + +def parse_sagemaker_version(version_str): + """Parse SageMaker version strings, handling custom formats like '1.7-1-1' + + Args: + version_str (str): Version string to parse + + Returns: + packaging.version.Version: Parsed version object + + Raises: + InvalidVersion: If the version string cannot be parsed + ImportError: If packaging library is not available + """ + if Version is None or InvalidVersion is None: + raise ImportError("packaging library is required for version parsing") + + try: + return Version(version_str) + except InvalidVersion: + # Handle SageMaker custom format: '1.7-1-1' -> '1.7.1.1' + if version_str.count("-") >= 2: + normalized = version_str.replace("-", ".", 2) + return Version(normalized) + raise diff --git a/tests/conftest.py b/tests/conftest.py index 34f5c5306d..a4873c823c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,6 +23,7 @@ from botocore.config import Config from packaging.version import Version from packaging.specifiers import SpecifierSet +from sagemaker.utils import parse_sagemaker_version from sagemaker import Session, image_uris, utils, get_execution_role from sagemaker.local import LocalSession @@ -790,7 +791,7 @@ def _parametrize_framework_version_fixtures(metafunc, fixture_prefix, config): versions = list(config["versions"].keys()) + list(config.get("version_aliases", {}).keys()) metafunc.parametrize(fixture_name, versions, scope="session") - latest_version = sorted(config["versions"].keys(), key=lambda v: Version(v))[-1] + latest_version = sorted(config["versions"].keys(), key=parse_sagemaker_version)[-1] fixture_name = "{}_latest_version".format(fixture_prefix) if fixture_name in metafunc.fixturenames: