Skip to content

pysdk upgrade for python 3.10 and version mapping #5310

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

pysdk upgrade for python 3.10 and version mapping #5310

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 96 additions & 0 deletions src/sagemaker/image_uri_config/sklearn.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,54 @@
"us-west-2": "246618743249"
},
"repository": "sagemaker-scikit-learn"
},
"1.2-1-1": {
"processors": [
"cpu"
],
"py_versions": [
"py3"
],
"registries": {
"af-south-1": "510948584623",
"ap-east-1": "651117190479",
"ap-northeast-1": "354813040037",
"ap-northeast-2": "366743142698",
"ap-northeast-3": "867004704886",
"ap-south-1": "720646828776",
"ap-south-2": "628508329040",
"ap-southeast-1": "121021644041",
"ap-southeast-2": "783357654285",
"ap-southeast-3": "951798379941",
"ap-southeast-4": "106583098589",
"ca-central-1": "341280168497",
"ca-west-1": "190319476487",
"cn-north-1": "450853457545",
"cn-northwest-1": "451049120500",
"eu-central-1": "492215442770",
"eu-central-2": "680994064768",
"eu-north-1": "662702820516",
"eu-south-1": "978288397137",
"eu-south-2": "104374241257",
"eu-west-1": "141502667606",
"eu-west-2": "764974769150",
"eu-west-3": "659782779980",
"il-central-1": "898809789911",
"me-central-1": "272398656194",
"me-south-1": "801668240914",
"sa-east-1": "737474898029",
"us-east-1": "683313688378",
"us-east-2": "257758044811",
"us-gov-east-1": "237065988967",
"us-gov-west-1": "414596584902",
"us-iso-east-1": "833128469047",
"us-isob-east-1": "281123927165",
"us-isof-east-1": "108575199400",
"us-isof-south-1": "124985052026",
"us-west-1": "746614075791",
"us-west-2": "246618743249"
},
"repository": "sagemaker-scikit-learn"
}
}
},
Expand Down Expand Up @@ -388,6 +436,54 @@
"us-west-2": "246618743249"
},
"repository": "sagemaker-scikit-learn"
},
"1.2-1-1": {
"processors": [
"cpu"
],
"py_versions": [
"py3"
],
"registries": {
"af-south-1": "510948584623",
"ap-east-1": "651117190479",
"ap-northeast-1": "354813040037",
"ap-northeast-2": "366743142698",
"ap-northeast-3": "867004704886",
"ap-south-1": "720646828776",
"ap-south-2": "628508329040",
"ap-southeast-1": "121021644041",
"ap-southeast-2": "783357654285",
"ap-southeast-3": "951798379941",
"ap-southeast-4": "106583098589",
"ca-central-1": "341280168497",
"ca-west-1": "190319476487",
"cn-north-1": "450853457545",
"cn-northwest-1": "451049120500",
"eu-central-1": "492215442770",
"eu-central-2": "680994064768",
"eu-north-1": "662702820516",
"eu-south-1": "978288397137",
"eu-south-2": "104374241257",
"eu-west-1": "141502667606",
"eu-west-2": "764974769150",
"eu-west-3": "659782779980",
"il-central-1": "898809789911",
"me-central-1": "272398656194",
"me-south-1": "801668240914",
"sa-east-1": "737474898029",
"us-east-1": "683313688378",
"us-east-2": "257758044811",
"us-gov-east-1": "237065988967",
"us-gov-west-1": "414596584902",
"us-iso-east-1": "833128469047",
"us-isob-east-1": "281123927165",
"us-isof-east-1": "108575199400",
"us-isof-south-1": "124985052026",
"us-west-1": "746614075791",
"us-west-2": "246618743249"
},
"repository": "sagemaker-scikit-learn"
}
}
},
Expand Down
84 changes: 84 additions & 0 deletions src/sagemaker/image_uri_config/xgboost.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -395,6 +395,48 @@
"us-west-2": "246618743249"
},
"repository": "sagemaker-xgboost"
},
"1.7-1-1": {
"registries": {
"af-south-1": "510948584623",
"ap-east-1": "651117190479",
"ap-northeast-1": "354813040037",
"ap-northeast-2": "366743142698",
"ap-northeast-3": "867004704886",
"ap-south-1": "720646828776",
"ap-south-2": "628508329040",
"ap-southeast-1": "121021644041",
"ap-southeast-2": "783357654285",
"ap-southeast-3": "951798379941",
"ap-southeast-4": "106583098589",
"ca-central-1": "341280168497",
"ca-west-1": "190319476487",
"cn-north-1": "450853457545",
"cn-northwest-1": "451049120500",
"eu-central-1": "492215442770",
"eu-central-2": "680994064768",
"eu-north-1": "662702820516",
"eu-south-1": "978288397137",
"eu-south-2": "104374241257",
"eu-west-1": "141502667606",
"eu-west-2": "764974769150",
"eu-west-3": "659782779980",
"il-central-1": "898809789911",
"me-central-1": "272398656194",
"me-south-1": "801668240914",
"sa-east-1": "737474898029",
"us-east-1": "683313688378",
"us-east-2": "257758044811",
"us-gov-east-1": "237065988967",
"us-gov-west-1": "414596584902",
"us-iso-east-1": "833128469047",
"us-isob-east-1": "281123927165",
"us-isof-east-1": "108575199400",
"us-isof-south-1": "124985052026",
"us-west-1": "746614075791",
"us-west-2": "246618743249"
},
"repository": "sagemaker-xgboost"
}
}
},
Expand Down Expand Up @@ -794,6 +836,48 @@
"us-west-2": "246618743249"
},
"repository": "sagemaker-xgboost"
},
"1.7-1-1": {
"registries": {
"af-south-1": "510948584623",
"ap-east-1": "651117190479",
"ap-northeast-1": "354813040037",
"ap-northeast-2": "366743142698",
"ap-northeast-3": "867004704886",
"ap-south-1": "720646828776",
"ap-south-2": "628508329040",
"ap-southeast-1": "121021644041",
"ap-southeast-2": "783357654285",
"ap-southeast-3": "951798379941",
"ap-southeast-4": "106583098589",
"ca-central-1": "341280168497",
"ca-west-1": "190319476487",
"cn-north-1": "450853457545",
"cn-northwest-1": "451049120500",
"eu-central-1": "492215442770",
"eu-central-2": "680994064768",
"eu-north-1": "662702820516",
"eu-south-1": "978288397137",
"eu-south-2": "104374241257",
"eu-west-1": "141502667606",
"eu-west-2": "764974769150",
"eu-west-3": "659782779980",
"il-central-1": "898809789911",
"me-central-1": "272398656194",
"me-south-1": "801668240914",
"sa-east-1": "737474898029",
"us-east-1": "683313688378",
"us-east-2": "257758044811",
"us-gov-east-1": "237065988967",
"us-gov-west-1": "414596584902",
"us-iso-east-1": "833128469047",
"us-isob-east-1": "281123927165",
"us-isof-east-1": "108575199400",
"us-isof-south-1": "124985052026",
"us-west-1": "746614075791",
"us-west-2": "246618743249"
},
"repository": "sagemaker-xgboost"
}
}
},
Expand Down
12 changes: 8 additions & 4 deletions src/sagemaker/image_uris.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import os
import re
from typing import Optional
from packaging.version import Version
from sagemaker.utils import parse_sagemaker_version

from sagemaker import utils
from sagemaker.jumpstart.constants import DEFAULT_JUMPSTART_SAGEMAKER_SESSION, JUMPSTART_LOGGER
Expand Down Expand Up @@ -403,8 +403,12 @@ def _config_for_framework_and_scope(framework, image_scope, accelerator_type=Non
def _validate_instance_deprecation(framework, instance_type, version):
"""Check if instance type is deprecated for a certain framework with a certain version"""
if utils.get_instance_type_family(instance_type) == "p2":
if (framework == "pytorch" and Version(version) >= Version("1.13")) or (
framework == "tensorflow" and Version(version) >= Version("2.12")
if (
framework == "pytorch"
and parse_sagemaker_version(version) >= parse_sagemaker_version("1.13")
) or (
framework == "tensorflow"
and parse_sagemaker_version(version) >= parse_sagemaker_version("2.12")
):
raise ValueError(
"P2 instances have been deprecated for sagemaker jobs starting PyTorch 1.13 and TensorFlow 2.12"
Expand Down Expand Up @@ -804,7 +808,7 @@ def _fetch_latest_version_from_config( # pylint: disable=R0911
bottom_number = int(bottom_version[:-2])
max_version = max(top_number, bottom_number)
return f"{max_version}.x"
if Version(top_version) >= Version(bottom_version):
if parse_sagemaker_version(top_version) >= parse_sagemaker_version(bottom_version):
return top_version
return bottom_version

Expand Down
8 changes: 4 additions & 4 deletions src/sagemaker/serve/detector/image_detector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import platform
import re
import logging
from packaging import version
from sagemaker.utils import parse_sagemaker_version
from sagemaker import image_uris

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -74,15 +74,15 @@ def auto_detect_container(model, region: str, instance_type: str) -> str:
def _cast_to_compatible_version(framework: str, fw_version: str) -> Tuple[str]:
"""Given fw_version, detect the available versions"""
config = image_uris._config_for_framework_and_scope(framework, "inference", None)
available_versions = [version.parse(ver) for ver in list(config["versions"].keys())]
available_versions = [parse_sagemaker_version(ver) for ver in list(config["versions"].keys())]
available_versions.sort()

earliest_upcast_version = None
exact_match_version = None
latest_downcast_version = None

major_version_pattern = r"^(\d+)"
parsed_fw_version = version.parse(fw_version)
parsed_fw_version = parse_sagemaker_version(fw_version)
major_verson = int(re.match(major_version_pattern, str(parsed_fw_version)).group(1))

for available_version in available_versions:
Expand Down Expand Up @@ -119,7 +119,7 @@ def _cast_to_compatible_version(framework: str, fw_version: str) -> Tuple[str]:
return (exact_match_version, latest_downcast_version, earliest_upcast_version)


def _process_version(ver: version.Version) -> str:
def _process_version(ver) -> str:
"""Placeholder docstring"""
if not ver:
return None
Expand Down
32 changes: 32 additions & 0 deletions src/sagemaker/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,12 @@
from sagemaker.workflow import is_pipeline_parameter_string, is_pipeline_variable
from sagemaker.workflow.entities import PipelineVariable

try:
from packaging.version import Version, InvalidVersion
except ImportError:
Version = None
InvalidVersion = None

ALTERNATE_DOMAINS = {
"cn-north-1": "amazonaws.com.cn",
"cn-northwest-1": "amazonaws.com.cn",
Expand Down Expand Up @@ -1959,3 +1965,29 @@ def get_domain_for_region(region: str) -> str:
region (str): AWS region name.
"""
return ALTERNATE_DOMAINS.get(region, "amazonaws.com")


def parse_sagemaker_version(version_str):
"""Parse SageMaker version strings, handling custom formats like '1.7-1-1'

Args:
version_str (str): Version string to parse

Returns:
packaging.version.Version: Parsed version object

Raises:
InvalidVersion: If the version string cannot be parsed
ImportError: If packaging library is not available
"""
if Version is None or InvalidVersion is None:
raise ImportError("packaging library is required for version parsing")

try:
return Version(version_str)
except InvalidVersion:
# Handle SageMaker custom format: '1.7-1-1' -> '1.7.1.1'
if version_str.count("-") >= 2:
normalized = version_str.replace("-", ".", 2)
return Version(normalized)
raise
3 changes: 2 additions & 1 deletion tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from botocore.config import Config
from packaging.version import Version
from packaging.specifiers import SpecifierSet
from sagemaker.utils import parse_sagemaker_version

from sagemaker import Session, image_uris, utils, get_execution_role
from sagemaker.local import LocalSession
Expand Down Expand Up @@ -790,7 +791,7 @@ def _parametrize_framework_version_fixtures(metafunc, fixture_prefix, config):
versions = list(config["versions"].keys()) + list(config.get("version_aliases", {}).keys())
metafunc.parametrize(fixture_name, versions, scope="session")

latest_version = sorted(config["versions"].keys(), key=lambda v: Version(v))[-1]
latest_version = sorted(config["versions"].keys(), key=parse_sagemaker_version)[-1]

fixture_name = "{}_latest_version".format(fixture_prefix)
if fixture_name in metafunc.fixturenames:
Expand Down
Loading