From 5ebe2258c9d6ff8285273b497027691d35b50973 Mon Sep 17 00:00:00 2001 From: Lauren Yu <6631887+laurenyu@users.noreply.github.com> Date: Wed, 29 Jul 2020 14:20:15 -0700 Subject: [PATCH] breaking: deprecate unused functions from utils and fw_utils This also deprecates the unused class sagemaker.session.ModelContainer --- src/sagemaker/fw_utils.py | 53 +------------- src/sagemaker/session.py | 23 ------- src/sagemaker/utils.py | 79 --------------------- tests/unit/test_pipeline_model.py | 3 - tests/unit/test_utils.py | 111 ------------------------------ 5 files changed, 1 insertion(+), 268 deletions(-) diff --git a/src/sagemaker/fw_utils.py b/src/sagemaker/fw_utils.py index f757087fdb..1a8eabda0c 100644 --- a/src/sagemaker/fw_utils.py +++ b/src/sagemaker/fw_utils.py @@ -33,15 +33,6 @@ instantiated with positional or keyword arguments. """ -EMPTY_FRAMEWORK_VERSION_WARNING = ( - "No framework_version specified, defaulting to version {}. " - "framework_version will be required in SageMaker Python SDK v2." -) -LATER_FRAMEWORK_VERSION_WARNING = ( - "This is not the latest supported version. " - "If you would like to use version {latest}, " - "please add framework_version={latest} to your constructor." -) PYTHON_2_DEPRECATION_WARNING = ( "{latest_supported_version} is the latest version of {framework} that supports " "Python 2. Newer versions of {framework} will only be available for Python 3." @@ -54,21 +45,10 @@ "fully leverage all GPU cores; the parameter server will be configured to run " "only one worker per host regardless of the number of GPUs." ) -PARAMETER_V2_RENAME_WARNING = ( - "Parameter {v1_parameter_name} will be renamed to {v2_parameter_name} " - "in SageMaker Python SDK v2." -) - - -EMPTY_FRAMEWORK_VERSION_ERROR = ( - "framework_version is required for script mode estimator. " - "Please add framework_version={} to your constructor to avoid this error." -) +DEBUGGER_UNSUPPORTED_REGIONS = ("us-gov-west-1", "us-iso-east-1") SINGLE_GPU_INSTANCE_TYPES = ("ml.p2.xlarge", "ml.p3.2xlarge") -DEBUGGER_UNSUPPORTED_REGIONS = ["us-gov-west-1", "us-iso-east-1"] - def is_version_equal_or_higher(lowest_version, framework_version): """Determine whether the ``framework_version`` is equal to or higher than @@ -294,26 +274,6 @@ def model_code_key_prefix(code_location_key_prefix, model_name, image): return "/".join(filter(None, [code_location_key_prefix, model_name or training_job_name])) -def empty_framework_version_warning(default_version, latest_version): - """ - Args: - default_version: - latest_version: - """ - msgs = [EMPTY_FRAMEWORK_VERSION_WARNING.format(default_version)] - if default_version != latest_version: - msgs.append(later_framework_version_warning(latest_version)) - return " ".join(msgs) - - -def later_framework_version_warning(latest_version): - """ - Args: - latest_version: - """ - return LATER_FRAMEWORK_VERSION_WARNING.format(latest=latest_version) - - def warn_if_parameter_server_with_multi_gpu(training_instance_type, distribution): """Warn the user that training will not fully leverage all the GPU cores if parameter server is enabled and a multi-GPU instance is selected. @@ -363,17 +323,6 @@ def python_deprecation_warning(framework, latest_supported_version): ) -def parameter_v2_rename_warning(v1_parameter_name, v2_parameter_name): - """ - Args: - v1_parameter_name: parameter name used in SageMaker Python SDK v1 - v2_parameter_name: parameter name used in SageMaker Python SDK v2 - """ - return PARAMETER_V2_RENAME_WARNING.format( - v1_parameter_name=v1_parameter_name, v2_parameter_name=v2_parameter_name - ) - - def _region_supports_debugger(region_name): """Returns boolean indicating whether the region supports Amazon SageMaker Debugger. diff --git a/src/sagemaker/session.py b/src/sagemaker/session.py index f50028a2cc..cfd0ca7fa1 100644 --- a/src/sagemaker/session.py +++ b/src/sagemaker/session.py @@ -3370,29 +3370,6 @@ def __init__(self, seed): self.seed = seed -class ModelContainer(object): - """Amazon SageMaker Model configurations for inference pipelines. - - Attributes: - model_data (str): S3 Model artifact location - image_uri (str): Docker image URL in ECR - env (dict[str,str]): Environment variable mapping - """ - - def __init__(self, model_data, image_uri, env=None): - """Create a definition of a model which can be part of an Inference Pipeline - - Args: - model_data (str): The S3 location of a SageMaker model data ``.tar.gz`` file. - image_uri (str): A Docker image URI. - env (dict[str, str]): Environment variables to run with ``image_uri`` when hosted in - SageMaker (default: None). - """ - self.model_data = model_data - self.image_uri = image_uri - self.env = env - - def _create_model_request( name, role, container_def=None, tags=None ): # pylint: disable=redefined-outer-name diff --git a/src/sagemaker/utils.py b/src/sagemaker/utils.py index f7e132dfed..e1aa558287 100644 --- a/src/sagemaker/utils.py +++ b/src/sagemaker/utils.py @@ -24,10 +24,8 @@ import tempfile import time from datetime import datetime -from functools import wraps import botocore -import six from six.moves.urllib import parse @@ -132,21 +130,6 @@ def sagemaker_short_timestamp(): return time.strftime("%y%m%d-%H%M") -def debug(func): - """Print the function name and arguments for debugging. - - Args: - func: - """ - - @wraps(func) - def wrapper(*args, **kwargs): - print("{} args: {} kwargs: {}".format(func.__name__, args, kwargs)) - return func(*args, **kwargs) - - return wrapper - - def get_config_value(key_path, config): """ Args: @@ -178,19 +161,6 @@ def get_short_version(framework_version): return ".".join(framework_version.split(".")[:2]) -def extract_name_from_job_arn(arn): - """Returns the name used in the API given a full ARN for a training job or - hyperparameter tuning job. - - Args: - arn: - """ - slash_pos = arn.find("/") - if slash_pos == -1: - raise ValueError("Cannot parse invalid ARN: %s" % arn) - return arn[(slash_pos + 1) :] - - def secondary_training_status_changed(current_job_description, prev_job_description): """Returns true if training job's secondary status message has changed. @@ -276,55 +246,6 @@ def secondary_training_status_message(job_description, prev_description): return "\n".join(status_strs) -def generate_tensorboard_url(domain, bucket_paths): - """Generate Tensorboard URL for given list of s3 buckets - - Args: - domain: JupyterLab app domain - bucket_paths: List of S3 bucket paths in format `bucket/path` - or a single string in the same format - - Returns: - str: Tensorboard URL - - Raises: - AttributeError if invalid inputs are passed - """ - - def trim_prefix(s, prefix): - if s.startswith(prefix): - return s[len(prefix) :] - return s - - def encode_s3_url(s3_url): - if not s3_url: - raise AttributeError("bucket_paths element should not be empty") - s3_url = trim_prefix(s3_url, S3_PREFIX) - return parse.quote_plus("{}{}".format(S3_PREFIX, s3_url)) - - if not isinstance(domain, six.string_types): - raise AttributeError("domain parameter should be string") - - if len(domain) == 0: - raise AttributeError("domain parameter should not be empty") - - if isinstance(bucket_paths, six.string_types): - bucket_paths = [bucket_paths] - elif not isinstance(bucket_paths, list): - raise AttributeError("bucket paths should be a list or a string") - - if len(bucket_paths) == 0: - raise AttributeError("bucket_paths parameter should not be empty list") - - domain = trim_prefix(domain, HTTPS_PREFIX) - domain = trim_prefix(domain, HTTP_PREFIX) - - s3_urls = map(encode_s3_url, bucket_paths) - query = ",".join(s3_urls) - - return "https://{}/tensorboard/default?s3urls={}".format(domain, query) - - def download_folder(bucket_name, prefix, target, sagemaker_session): """Download a folder from S3 to a local path diff --git a/tests/unit/test_pipeline_model.py b/tests/unit/test_pipeline_model.py index c53f96f3e0..fed0fde01f 100644 --- a/tests/unit/test_pipeline_model.py +++ b/tests/unit/test_pipeline_model.py @@ -18,7 +18,6 @@ from sagemaker.model import FrameworkModel from sagemaker.pipeline import PipelineModel from sagemaker.predictor import Predictor -from sagemaker.session import ModelContainer from sagemaker.sparkml import SparkMLModel ENTRY_POINT = "blah.py" @@ -30,8 +29,6 @@ ROLE = "some-role" ENV_1 = {"SAGEMAKER_DEFAULT_INVOCATIONS_ACCEPT": "application/json"} ENV_2 = {"SAGEMAKER_DEFAULT_INVOCATIONS_ACCEPT": "text/csv"} -MODEL_CONTAINER_1 = ModelContainer(image_uri=MODEL_IMAGE_1, model_data=MODEL_DATA_1, env=ENV_1) -MODEL_CONTAINER_2 = ModelContainer(image_uri=MODEL_IMAGE_2, model_data=MODEL_DATA_2, env=ENV_2) ENDPOINT = "some-ep" diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 9f9415b9f9..47a4a5f4e2 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -103,18 +103,6 @@ def test_unique_name_from_base_truncated(): ) -def test_name_from_tuning_arn(): - arn = "arn:aws:sagemaker:us-west-2:968277160000:hyper-parameter-tuning-job/resnet-sgd-tuningjob-11-07-34-11" - name = sagemaker.utils.extract_name_from_job_arn(arn) - assert name == "resnet-sgd-tuningjob-11-07-34-11" - - -def test_name_from_training_arn(): - arn = "arn:aws:sagemaker:us-west-2:968277160000:training-job/resnet-sgd-tuningjob-11-22-38-46-002-2927640b" - name = sagemaker.utils.extract_name_from_job_arn(arn) - assert name == "resnet-sgd-tuningjob-11-22-38-46-002-2927640b" - - def test_base_from_name(): name = "mxnet-training-2020-06-29-15-19-25-475" assert "mxnet-training" == sagemaker.utils.base_from_name(name) @@ -217,105 +205,6 @@ def test_secondary_training_status_message_prev_missing(): ) -def test_generate_tensorboard_url_valid_domain_and_bucket_paths(): - domain = "jupyterlab.us-east-2.abcdefgh.com" - bucket_paths = ["bucket1/path1", "bucket2/path2"] - expected = "https://{}/tensorboard/default?{}".format( - domain, "s3urls=s3%3A%2F%2Fbucket1%2Fpath1,s3%3A%2F%2Fbucket2%2Fpath2" - ) - assert sagemaker.utils.generate_tensorboard_url(domain, bucket_paths) == expected - - -def test_generate_tensorboard_url_valid_domain_and_bucket_paths_with_s3_prefixes(): - domain = "jupyterlab.us-east-2.abcdefgh.com" - bucket_paths = ["s3://bucket1/path1", "s3://bucket2/path2"] - expected = "https://{}/tensorboard/default?{}".format( - domain, "s3urls=s3%3A%2F%2Fbucket1%2Fpath1,s3%3A%2F%2Fbucket2%2Fpath2" - ) - assert sagemaker.utils.generate_tensorboard_url(domain, bucket_paths) == expected - - -def test_generate_tensorboard_url_valid_domain_and_bucket_paths_single(): - domain = "jupyterlab.us-east-2.abcdefgh.com" - bucket_paths = ["bucket1/path1"] - expected = "https://{}/tensorboard/default?{}".format( - domain, "s3urls=s3%3A%2F%2Fbucket1%2Fpath1" - ) - assert sagemaker.utils.generate_tensorboard_url(domain, bucket_paths) == expected - - -def test_generate_tensorboard_url_valid_domain_and_bucket_paths_string(): - domain = "jupyterlab.us-east-2.abcdefgh.com" - bucket_paths = "bucket1/path1" - expected = "https://{}/tensorboard/default?{}".format( - domain, "s3urls=s3%3A%2F%2Fbucket1%2Fpath1" - ) - assert sagemaker.utils.generate_tensorboard_url(domain, bucket_paths) == expected - - -def test_generate_tensorboard_url_valid_domain_with_http_prefix_and_bucket_paths(): - domain = "http://jupyterlab.us-east-2.abcdefgh.com" - bucket_paths = ["bucket1/path1"] - expected = "https://{}/tensorboard/default?{}".format( - "jupyterlab.us-east-2.abcdefgh.com", "s3urls=s3%3A%2F%2Fbucket1%2Fpath1" - ) - assert sagemaker.utils.generate_tensorboard_url(domain, bucket_paths) == expected - - -def test_generate_tensorboard_url_valid_domain_with_https_prefix_and_bucket_paths(): - domain = "https://jupyterlab.us-east-2.abcdefgh.com" - bucket_paths = ["bucket1/path1"] - expected = "https://{}/tensorboard/default?{}".format( - "jupyterlab.us-east-2.abcdefgh.com", "s3urls=s3%3A%2F%2Fbucket1%2Fpath1" - ) - assert sagemaker.utils.generate_tensorboard_url(domain, bucket_paths) == expected - - -def test_generate_tensorboard_url_bucket_path_neither_string_nor_list(): - domain = "jupyterlab.us-east-2.abcdefgh.com" - bucket_paths = None - try: - sagemaker.utils.generate_tensorboard_url(domain, bucket_paths) - except AttributeError as error: - assert str(error) == "bucket paths should be a list or a string" - - -def test_generate_tensorboard_url_empty_domain(): - domain = "" - bucket_paths = ["bucket1/path1"] - try: - sagemaker.utils.generate_tensorboard_url(domain, bucket_paths) - except AttributeError as error: - assert str(error) == "domain parameter should not be empty" - - -def test_generate_tensorboard_url_empty_bucket_paths(): - domain = "jupyterlab.us-east-2.abcdefgh.com" - bucket_paths = [] - try: - sagemaker.utils.generate_tensorboard_url(domain, bucket_paths) - except AttributeError as error: - assert str(error) == "bucket_paths parameter should not be empty list" - - -def test_generate_tensorboard_url_bucket_paths_with_empty_string(): - domain = "jupyterlab.us-east-2.abcdefgh.com" - bucket_paths = [""] - try: - sagemaker.utils.generate_tensorboard_url(domain, bucket_paths) - except AttributeError as error: - assert str(error) == "bucket_paths element should not be empty" - - -def test_generate_tensorboard_url_domain_non_string(): - domain = None - bucket_paths = ["bucket1/path1"] - try: - sagemaker.utils.generate_tensorboard_url(domain, bucket_paths) - except AttributeError as error: - assert str(error) == "domain parameter should be string" - - @patch("os.makedirs") def test_download_folder(makedirs): boto_mock = Mock(name="boto_session")