From a7a596e7a3945962b2ee39bfa04fd1f6f5a89726 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Fri, 17 Sep 2021 15:44:35 -0700 Subject: [PATCH 01/30] Support env variables for configure batchSize, maxBatchDelay etc. for the single model in torchserve --- .../torchserve.py | 17 ++++ .../ts_environment.py | 81 +++++++++++++++++++ .../ts_parameters.py | 21 +++++ 3 files changed, 119 insertions(+) create mode 100644 src/sagemaker_pytorch_serving_container/ts_environment.py create mode 100644 src/sagemaker_pytorch_serving_container/ts_parameters.py diff --git a/src/sagemaker_pytorch_serving_container/torchserve.py b/src/sagemaker_pytorch_serving_container/torchserve.py index d90e0000..c3a0e305 100644 --- a/src/sagemaker_pytorch_serving_container/torchserve.py +++ b/src/sagemaker_pytorch_serving_container/torchserve.py @@ -13,6 +13,7 @@ """This module contains functionality to configure and start Torchserve.""" from __future__ import absolute_import +import json import os import signal import subprocess @@ -24,6 +25,7 @@ from retrying import retry import sagemaker_pytorch_serving_container +from sagemaker_pytorch_serving_container import ts_environment from sagemaker_inference import default_handler_service, environment, utils from sagemaker_inference.environment import code_dir @@ -149,8 +151,23 @@ def _create_torchserve_config_file(): def _generate_ts_config_properties(): env = environment.Environment() + ts_env = ts_environment.TorchServeEnvironment() + models = { + DEFAULT_TS_MODEL_NAME: { + "1.0": { + "defaultVersion": "true", + "marName": f"{DEFAULT_TS_MODEL_NAME}.mar", + "minWorkers": ts_env._min_workers, + "maxWorkers": ts_env._max_workers, + "batchSize": ts_env._batch_size, + "maxBatchDelay": ts_env._max_batch_delay, + "responseTimeout": ts_env._response_timeout + } + } + } user_defined_configuration = { + "models": json.dumps(models), "default_response_timeout": env.model_server_timeout, "default_workers_per_model": env.model_server_workers, "inference_address": "http://0.0.0.0:{}".format(env.inference_http_port), diff --git a/src/sagemaker_pytorch_serving_container/ts_environment.py b/src/sagemaker_pytorch_serving_container/ts_environment.py new file mode 100644 index 00000000..f24d928b --- /dev/null +++ b/src/sagemaker_pytorch_serving_container/ts_environment.py @@ -0,0 +1,81 @@ +# Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the 'license' file accompanying this file. This file is +# distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""This module contains functionality that provides access to system +characteristics, environment variables and configuration settings. +""" + +from __future__ import absolute_import + +from sagemaker_pytorch_serving_container import ts_parameters + +import os +import logging + +logger = logging.getLogger() + +DEFAULT_TS_BATCH_SIZE = 1 +DEFAULT_TS_MAX_BATCH_DELAY = 100 +DEFAULT_TS_MIN_WORKERS = 1 +DEFAULT_TS_MAX_WORKERS = 4 +DEFAULT_TS_RESPONSE_TIMEOUT = 120 + +class TorchServeEnvironment(): + """Provides access to aspects of the torchserve environment relevant to serving containers, + including system characteristics, environment variables and configuration settings. + + The Environment is a read-only snapshot of the container environment. + It is a dictionary-like object, allowing any builtin function that works with dictionary. + + Attributes: + batch_size (int): This is the maximum batch size in ms that a model is expected to handle + max_batch_delay (int): This is the maximum batch delay time TorchServe waits to receive + batch_size number of requests. If TorchServe doesn’t receive batch_size number of requests + before this timer time’s out, it sends what ever requests that were received to the model handler + """ + def __init__(self): + self._batch_size = os.environ.get(ts_parameters.MODEL_SERVER_BATCH_SIZE_ENV, DEFAULT_TS_BATCH_SIZE) + self._max_batch_delay = os.environ.get(ts_parameters.MODEL_SERVER_MAX_BATCH_SIZE, DEFAULT_TS_MAX_BATCH_DELAY) + self._min_workers = os.environ.get(ts_parameters.MODEL_SERVER_MIN_WORKERS, DEFAULT_TS_MIN_WORKERS) + self._max_workers = os.environ.get(ts_parameters.MODEL_SERVER_MAX_WORKERS, DEFAULT_TS_MAX_WORKERS) + self._response_timeout = os.environ.get(ts_parameters.MODEL_SERVER_RESPONSE_TIMEOUT, DEFAULT_TS_RESPONSE_TIMEOUT) + + @property + def batch_size(self): # type: () -> int + """int: number of requests to batch before running inference on the server""" + return self._batch_size + + @property + def max_batch_delay(self): # type: () -> int + """int: time delay in milliseconds, to wait for incoming requests to be batched, + before running inference on the server + """ + return self._max_batch_delay + + @property + def min_workers(self): #type:() -> int + """int: minimum number of worker for model + """ + return self._min_workers + + @property + def max_workers(self): #type() -> int + """int: maximum number of workers for model + """ + return self._max_workers + + @property + def response_timeout(self): #type() -> int + """int: time delay after which inference will timeout in absense of a response + """ + return self._response_timeout + diff --git a/src/sagemaker_pytorch_serving_container/ts_parameters.py b/src/sagemaker_pytorch_serving_container/ts_parameters.py new file mode 100644 index 00000000..d7acc410 --- /dev/null +++ b/src/sagemaker_pytorch_serving_container/ts_parameters.py @@ -0,0 +1,21 @@ +# Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""This module contains string constants that define inference toolkit +parameters.""" +from __future__ import absolute_import + +MODEL_SERVER_BATCH_SIZE_ENV = "SAGEMAKER_TS_BATCH_SIZE" # type: str +MODEL_SERVER_MAX_BATCH_SIZE = "SAGEMAKER_TS_MAX_BATCH_DELAY" # type: str +MODEL_SERVER_MIN_WORKERS = "SAGEMAKER_TS_MIN_WORKERS" # type: str +MODEL_SERVER_MAX_WORKERS = "SAGEMAKER_TS_MAX_WORKERS" # type: str +MODEL_SERVER_RESPONSE_TIMEOUT = "SAGEMAKER_TS_RESPONSE_TIMEOUT" \ No newline at end of file From 27ca279a9d68ba943ca51c6275c8f8b337364819 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Tue, 21 Sep 2021 00:56:06 -0700 Subject: [PATCH 02/30] Add modified version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b64afe32..ba80f2ce 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.6.dev0 +2.0.6.nskool From 9415fd7e7c606a08956f3a4854904e59136a5622 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Tue, 21 Sep 2021 17:02:34 -0700 Subject: [PATCH 03/30] fix flake8 --- .../ts_environment.py | 31 ++++++++++--------- .../ts_parameters.py | 10 +++--- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/sagemaker_pytorch_serving_container/ts_environment.py b/src/sagemaker_pytorch_serving_container/ts_environment.py index f24d928b..96cd5b94 100644 --- a/src/sagemaker_pytorch_serving_container/ts_environment.py +++ b/src/sagemaker_pytorch_serving_container/ts_environment.py @@ -29,6 +29,7 @@ DEFAULT_TS_MAX_WORKERS = 4 DEFAULT_TS_RESPONSE_TIMEOUT = 120 + class TorchServeEnvironment(): """Provides access to aspects of the torchserve environment relevant to serving containers, including system characteristics, environment variables and configuration settings. @@ -38,44 +39,44 @@ class TorchServeEnvironment(): Attributes: batch_size (int): This is the maximum batch size in ms that a model is expected to handle - max_batch_delay (int): This is the maximum batch delay time TorchServe waits to receive - batch_size number of requests. If TorchServe doesn’t receive batch_size number of requests + max_batch_delay (int): This is the maximum batch delay time TorchServe waits to receive + batch_size number of requests. If TorchServe doesn’t receive batch_size number of requests before this timer time’s out, it sends what ever requests that were received to the model handler """ - def __init__(self): + def __init__(self): self._batch_size = os.environ.get(ts_parameters.MODEL_SERVER_BATCH_SIZE_ENV, DEFAULT_TS_BATCH_SIZE) - self._max_batch_delay = os.environ.get(ts_parameters.MODEL_SERVER_MAX_BATCH_SIZE, DEFAULT_TS_MAX_BATCH_DELAY) + self._max_batch_delay = os.environ.get(ts_parameters.MODEL_SERVER_MAX_BATCH_SIZE, DEFAULT_TS_MAX_BATCH_DELAY) self._min_workers = os.environ.get(ts_parameters.MODEL_SERVER_MIN_WORKERS, DEFAULT_TS_MIN_WORKERS) self._max_workers = os.environ.get(ts_parameters.MODEL_SERVER_MAX_WORKERS, DEFAULT_TS_MAX_WORKERS) - self._response_timeout = os.environ.get(ts_parameters.MODEL_SERVER_RESPONSE_TIMEOUT, DEFAULT_TS_RESPONSE_TIMEOUT) - + self._response_timeout = os.environ.get(ts_parameters.MODEL_SERVER_RESPONSE_TIMEOUT, + DEFAULT_TS_RESPONSE_TIMEOUT) + @property - def batch_size(self): # type: () -> int + def batch_size(self): # type: () -> int """int: number of requests to batch before running inference on the server""" return self._batch_size - + @property - def max_batch_delay(self): # type: () -> int + def max_batch_delay(self): # type: () -> int """int: time delay in milliseconds, to wait for incoming requests to be batched, before running inference on the server """ return self._max_batch_delay - + @property - def min_workers(self): #type:() -> int + def min_workers(self): # type:() -> int """int: minimum number of worker for model """ return self._min_workers - + @property - def max_workers(self): #type() -> int + def max_workers(self): # type() -> int """int: maximum number of workers for model """ return self._max_workers @property - def response_timeout(self): #type() -> int + def response_timeout(self): # type() -> int """int: time delay after which inference will timeout in absense of a response """ return self._response_timeout - diff --git a/src/sagemaker_pytorch_serving_container/ts_parameters.py b/src/sagemaker_pytorch_serving_container/ts_parameters.py index d7acc410..2354580d 100644 --- a/src/sagemaker_pytorch_serving_container/ts_parameters.py +++ b/src/sagemaker_pytorch_serving_container/ts_parameters.py @@ -14,8 +14,8 @@ parameters.""" from __future__ import absolute_import -MODEL_SERVER_BATCH_SIZE_ENV = "SAGEMAKER_TS_BATCH_SIZE" # type: str -MODEL_SERVER_MAX_BATCH_SIZE = "SAGEMAKER_TS_MAX_BATCH_DELAY" # type: str -MODEL_SERVER_MIN_WORKERS = "SAGEMAKER_TS_MIN_WORKERS" # type: str -MODEL_SERVER_MAX_WORKERS = "SAGEMAKER_TS_MAX_WORKERS" # type: str -MODEL_SERVER_RESPONSE_TIMEOUT = "SAGEMAKER_TS_RESPONSE_TIMEOUT" \ No newline at end of file +MODEL_SERVER_BATCH_SIZE_ENV = "SAGEMAKER_TS_BATCH_SIZE" # type: str +MODEL_SERVER_MAX_BATCH_SIZE = "SAGEMAKER_TS_MAX_BATCH_DELAY" # type: str +MODEL_SERVER_MIN_WORKERS = "SAGEMAKER_TS_MIN_WORKERS" # type: str +MODEL_SERVER_MAX_WORKERS = "SAGEMAKER_TS_MAX_WORKERS" # type: str +MODEL_SERVER_RESPONSE_TIMEOUT = "SAGEMAKER_TS_RESPONSE_TIMEOUT" # type: str From 58c9fc275d77f20f7cf0b79d9922ee8e8ec11534 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Tue, 21 Sep 2021 17:56:54 -0700 Subject: [PATCH 04/30] Edit version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ba80f2ce..736e0a64 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.6.nskool +2.0.6.nskool1 From a3f24c66f1aa1499ec2ce18f58bb7d86950fc235 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Wed, 22 Sep 2021 02:41:22 -0700 Subject: [PATCH 05/30] Correct type --- .../ts_environment.py | 12 ++++++------ .../ts_parameters.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/sagemaker_pytorch_serving_container/ts_environment.py b/src/sagemaker_pytorch_serving_container/ts_environment.py index 96cd5b94..c6b9f21e 100644 --- a/src/sagemaker_pytorch_serving_container/ts_environment.py +++ b/src/sagemaker_pytorch_serving_container/ts_environment.py @@ -44,12 +44,12 @@ class TorchServeEnvironment(): before this timer time’s out, it sends what ever requests that were received to the model handler """ def __init__(self): - self._batch_size = os.environ.get(ts_parameters.MODEL_SERVER_BATCH_SIZE_ENV, DEFAULT_TS_BATCH_SIZE) - self._max_batch_delay = os.environ.get(ts_parameters.MODEL_SERVER_MAX_BATCH_SIZE, DEFAULT_TS_MAX_BATCH_DELAY) - self._min_workers = os.environ.get(ts_parameters.MODEL_SERVER_MIN_WORKERS, DEFAULT_TS_MIN_WORKERS) - self._max_workers = os.environ.get(ts_parameters.MODEL_SERVER_MAX_WORKERS, DEFAULT_TS_MAX_WORKERS) - self._response_timeout = os.environ.get(ts_parameters.MODEL_SERVER_RESPONSE_TIMEOUT, - DEFAULT_TS_RESPONSE_TIMEOUT) + self._batch_size = int(os.environ.get(ts_parameters.MODEL_SERVER_BATCH_SIZE, DEFAULT_TS_BATCH_SIZE)) + self._max_batch_delay = int(os.environ.get(ts_parameters.MODEL_SERVER_MAX_BATCH_DELAY, DEFAULT_TS_MAX_BATCH_DELAY)) + self._min_workers = int(os.environ.get(ts_parameters.MODEL_SERVER_MIN_WORKERS, DEFAULT_TS_MIN_WORKERS)) + self._max_workers = int(os.environ.get(ts_parameters.MODEL_SERVER_MAX_WORKERS, DEFAULT_TS_MAX_WORKERS)) + self._response_timeout = int(os.environ.get(ts_parameters.MODEL_SERVER_RESPONSE_TIMEOUT, + DEFAULT_TS_RESPONSE_TIMEOUT)) @property def batch_size(self): # type: () -> int diff --git a/src/sagemaker_pytorch_serving_container/ts_parameters.py b/src/sagemaker_pytorch_serving_container/ts_parameters.py index 2354580d..6aa961a1 100644 --- a/src/sagemaker_pytorch_serving_container/ts_parameters.py +++ b/src/sagemaker_pytorch_serving_container/ts_parameters.py @@ -14,8 +14,8 @@ parameters.""" from __future__ import absolute_import -MODEL_SERVER_BATCH_SIZE_ENV = "SAGEMAKER_TS_BATCH_SIZE" # type: str -MODEL_SERVER_MAX_BATCH_SIZE = "SAGEMAKER_TS_MAX_BATCH_DELAY" # type: str +MODEL_SERVER_BATCH_SIZE = "SAGEMAKER_TS_BATCH_SIZE" # type: str +MODEL_SERVER_MAX_BATCH_DELAY= "SAGEMAKER_TS_MAX_BATCH_DELAY" # type: str MODEL_SERVER_MIN_WORKERS = "SAGEMAKER_TS_MIN_WORKERS" # type: str MODEL_SERVER_MAX_WORKERS = "SAGEMAKER_TS_MAX_WORKERS" # type: str MODEL_SERVER_RESPONSE_TIMEOUT = "SAGEMAKER_TS_RESPONSE_TIMEOUT" # type: str From 6771e2bdfc536857c8eeaf75d66f9c6dc9c25041 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Wed, 22 Sep 2021 03:31:26 -0700 Subject: [PATCH 06/30] Add condition to including env variables in model config --- .../torchserve.py | 17 ++++++++++------- .../ts_environment.py | 13 +++++++++++-- .../ts_parameters.py | 2 +- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/sagemaker_pytorch_serving_container/torchserve.py b/src/sagemaker_pytorch_serving_container/torchserve.py index c3a0e305..3ef8b620 100644 --- a/src/sagemaker_pytorch_serving_container/torchserve.py +++ b/src/sagemaker_pytorch_serving_container/torchserve.py @@ -151,7 +151,15 @@ def _create_torchserve_config_file(): def _generate_ts_config_properties(): env = environment.Environment() + user_defined_configuration = { + "default_response_timeout": env.model_server_timeout, + "default_workers_per_model": env.model_server_workers, + "inference_address": "http://0.0.0.0:{}".format(env.inference_http_port), + "management_address": "http://0.0.0.0:{}".format(env.management_http_port), + } + ts_env = ts_environment.TorchServeEnvironment() + models = { DEFAULT_TS_MODEL_NAME: { "1.0": { @@ -166,13 +174,8 @@ def _generate_ts_config_properties(): } } - user_defined_configuration = { - "models": json.dumps(models), - "default_response_timeout": env.model_server_timeout, - "default_workers_per_model": env.model_server_workers, - "inference_address": "http://0.0.0.0:{}".format(env.inference_http_port), - "management_address": "http://0.0.0.0:{}".format(env.management_http_port), - } + if ts_env.is_env_set(): + user_defined_configuration["models"] = json.dumps(models) custom_configuration = str() diff --git a/src/sagemaker_pytorch_serving_container/ts_environment.py b/src/sagemaker_pytorch_serving_container/ts_environment.py index c6b9f21e..0f79d5f4 100644 --- a/src/sagemaker_pytorch_serving_container/ts_environment.py +++ b/src/sagemaker_pytorch_serving_container/ts_environment.py @@ -45,11 +45,20 @@ class TorchServeEnvironment(): """ def __init__(self): self._batch_size = int(os.environ.get(ts_parameters.MODEL_SERVER_BATCH_SIZE, DEFAULT_TS_BATCH_SIZE)) - self._max_batch_delay = int(os.environ.get(ts_parameters.MODEL_SERVER_MAX_BATCH_DELAY, DEFAULT_TS_MAX_BATCH_DELAY)) + self._max_batch_delay = int(os.environ.get(ts_parameters.MODEL_SERVER_MAX_BATCH_DELAY, + DEFAULT_TS_MAX_BATCH_DELAY)) self._min_workers = int(os.environ.get(ts_parameters.MODEL_SERVER_MIN_WORKERS, DEFAULT_TS_MIN_WORKERS)) self._max_workers = int(os.environ.get(ts_parameters.MODEL_SERVER_MAX_WORKERS, DEFAULT_TS_MAX_WORKERS)) self._response_timeout = int(os.environ.get(ts_parameters.MODEL_SERVER_RESPONSE_TIMEOUT, - DEFAULT_TS_RESPONSE_TIMEOUT)) + DEFAULT_TS_RESPONSE_TIMEOUT)) + + def is_env_set(self): # type: () -> bool + """bool: whether or not the environment variables have been set""" + ts_env_list = [ts_parameters.MODEL_SERVER_BATCH_SIZE, ts_parameters.MODEL_SERVER_MAX_BATCH_DELAY, + ts_parameters.MODEL_SERVER_MIN_WORKERS, ts_parameters.MODEL_SERVER_MAX_WORKERS, + ts_parameters.MODEL_SERVER_RESPONSE_TIMEOUT] + if any(env in ts_env_list for env in os.environ): + return True @property def batch_size(self): # type: () -> int diff --git a/src/sagemaker_pytorch_serving_container/ts_parameters.py b/src/sagemaker_pytorch_serving_container/ts_parameters.py index 6aa961a1..a2be2125 100644 --- a/src/sagemaker_pytorch_serving_container/ts_parameters.py +++ b/src/sagemaker_pytorch_serving_container/ts_parameters.py @@ -15,7 +15,7 @@ from __future__ import absolute_import MODEL_SERVER_BATCH_SIZE = "SAGEMAKER_TS_BATCH_SIZE" # type: str -MODEL_SERVER_MAX_BATCH_DELAY= "SAGEMAKER_TS_MAX_BATCH_DELAY" # type: str +MODEL_SERVER_MAX_BATCH_DELAY = "SAGEMAKER_TS_MAX_BATCH_DELAY" # type: str MODEL_SERVER_MIN_WORKERS = "SAGEMAKER_TS_MIN_WORKERS" # type: str MODEL_SERVER_MAX_WORKERS = "SAGEMAKER_TS_MAX_WORKERS" # type: str MODEL_SERVER_RESPONSE_TIMEOUT = "SAGEMAKER_TS_RESPONSE_TIMEOUT" # type: str From 76e700f8b966b10c5561ec8f26d18d7de219a5bf Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Wed, 22 Sep 2021 03:32:19 -0700 Subject: [PATCH 07/30] Add version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 736e0a64..3d08a966 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.6.nskool1 +2.0.6.nskool2 From 21fb1273396ced0a2aabad0436ec35e6b456865d Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Thu, 23 Sep 2021 17:53:10 -0700 Subject: [PATCH 08/30] Update version and remove env support --- VERSION | 2 +- src/sagemaker_pytorch_serving_container/torchserve.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index 3d08a966..bccc9a72 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.6.nskool2 +2.0.6.nskool3 diff --git a/src/sagemaker_pytorch_serving_container/torchserve.py b/src/sagemaker_pytorch_serving_container/torchserve.py index 3ef8b620..084eed4e 100644 --- a/src/sagemaker_pytorch_serving_container/torchserve.py +++ b/src/sagemaker_pytorch_serving_container/torchserve.py @@ -174,8 +174,8 @@ def _generate_ts_config_properties(): } } - if ts_env.is_env_set(): - user_defined_configuration["models"] = json.dumps(models) + # if ts_env.is_env_set(): + # user_defined_configuration["models"] = json.dumps(models) custom_configuration = str() From e44f383353e78869354b79fed13a7a0b781b2495 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Fri, 24 Sep 2021 01:05:53 -0700 Subject: [PATCH 09/30] Try converting config to string --- .../torchserve.py | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/sagemaker_pytorch_serving_container/torchserve.py b/src/sagemaker_pytorch_serving_container/torchserve.py index 084eed4e..28f72e68 100644 --- a/src/sagemaker_pytorch_serving_container/torchserve.py +++ b/src/sagemaker_pytorch_serving_container/torchserve.py @@ -160,22 +160,21 @@ def _generate_ts_config_properties(): ts_env = ts_environment.TorchServeEnvironment() - models = { - DEFAULT_TS_MODEL_NAME: { - "1.0": { - "defaultVersion": "true", - "marName": f"{DEFAULT_TS_MODEL_NAME}.mar", - "minWorkers": ts_env._min_workers, - "maxWorkers": ts_env._max_workers, - "batchSize": ts_env._batch_size, - "maxBatchDelay": ts_env._max_batch_delay, - "responseTimeout": ts_env._response_timeout + if ts_env.is_env_set(): + models = { + DEFAULT_TS_MODEL_NAME: { + "1.0": { + "defaultVersion": "true", + "marName": f"{DEFAULT_TS_MODEL_NAME}.mar", + "minWorkers": str(ts_env._min_workers), + "maxWorkers": str(ts_env._max_workers), + "batchSize": str(ts_env._batch_size), + "maxBatchDelay": str(ts_env._max_batch_delay), + "responseTimeout": str(ts_env._response_timeout) + } } } - } - - # if ts_env.is_env_set(): - # user_defined_configuration["models"] = json.dumps(models) + user_defined_configuration["models"] = json.dumps(models) custom_configuration = str() From 6df9a20660835132964ecc2e47f8bba6d58d580b Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Fri, 24 Sep 2021 01:25:26 -0700 Subject: [PATCH 10/30] Reverse str and update version --- VERSION | 2 +- src/sagemaker_pytorch_serving_container/torchserve.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/VERSION b/VERSION index bccc9a72..aee14265 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.6.nskool3 +2.0.6.nskool4 diff --git a/src/sagemaker_pytorch_serving_container/torchserve.py b/src/sagemaker_pytorch_serving_container/torchserve.py index 28f72e68..a063dc03 100644 --- a/src/sagemaker_pytorch_serving_container/torchserve.py +++ b/src/sagemaker_pytorch_serving_container/torchserve.py @@ -166,11 +166,11 @@ def _generate_ts_config_properties(): "1.0": { "defaultVersion": "true", "marName": f"{DEFAULT_TS_MODEL_NAME}.mar", - "minWorkers": str(ts_env._min_workers), - "maxWorkers": str(ts_env._max_workers), - "batchSize": str(ts_env._batch_size), - "maxBatchDelay": str(ts_env._max_batch_delay), - "responseTimeout": str(ts_env._response_timeout) + "minWorkers": ts_env._min_workers, + "maxWorkers": ts_env._max_workers, + "batchSize": ts_env._batch_size, + "maxBatchDelay": ts_env._max_batch_delay, + "responseTimeout": ts_env._response_timeout } } } From 6ed240fe7b54b7bfa77f66cfb76ca7475b21106a Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Fri, 24 Sep 2021 01:39:33 -0700 Subject: [PATCH 11/30] Fix true --- src/sagemaker_pytorch_serving_container/torchserve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sagemaker_pytorch_serving_container/torchserve.py b/src/sagemaker_pytorch_serving_container/torchserve.py index a063dc03..e97f7b79 100644 --- a/src/sagemaker_pytorch_serving_container/torchserve.py +++ b/src/sagemaker_pytorch_serving_container/torchserve.py @@ -164,7 +164,7 @@ def _generate_ts_config_properties(): models = { DEFAULT_TS_MODEL_NAME: { "1.0": { - "defaultVersion": "true", + "defaultVersion": True, "marName": f"{DEFAULT_TS_MODEL_NAME}.mar", "minWorkers": ts_env._min_workers, "maxWorkers": ts_env._max_workers, From e99fff95a4ff1e22248aa0af88f4afb9790781e8 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Fri, 24 Sep 2021 01:58:42 -0700 Subject: [PATCH 12/30] Experiment with default config --- VERSION | 2 +- .../etc/default-ts.properties | 35 +++++++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index aee14265..80206878 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.6.nskool4 +2.0.6.nskool5 diff --git a/src/sagemaker_pytorch_serving_container/etc/default-ts.properties b/src/sagemaker_pytorch_serving_container/etc/default-ts.properties index bc0f996a..f437e265 100644 --- a/src/sagemaker_pytorch_serving_container/etc/default-ts.properties +++ b/src/sagemaker_pytorch_serving_container/etc/default-ts.properties @@ -1,4 +1,33 @@ # Based on https://github.com/pytorch/serve/blob/master/docs/configuration.md -enable_envvars_config=true -decode_input_request=false -load_models=ALL +#enable_envvars_config=true +#decode_input_request=false +#load_models=ALL +# + +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +inference_address=http://0.0.0.0:8080 +management_address=http://0.0.0.0:8081 +number_of_netty_threads=32 +job_queue_size=1000 +model_store=/opt/ml/model +#decode_input_request=false +#load_models= flores_modified.mar +install_py_dep_per_model=true +load_models=BERTSeqClassification.mar +models={\ + "BERTSeqClassification": {\ + "1.0": {\ + "defaultVersion": true,\ + "marName": "BERTSeqClassification.mar",\ + "minWorkers": 1,\ + "maxWorkers": 1,\ + "batchSize": 3,\ + "maxBatchDelay": 100000,\ + "responseTimeout": 120\ + }\ + }\ +} From 112a08a7f6d8b5d3263da99d050e293ce1c4f799 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Fri, 24 Sep 2021 02:23:50 -0700 Subject: [PATCH 13/30] Complete --- VERSION | 2 +- .../etc/default-ts.properties | 35 ++----------------- 2 files changed, 4 insertions(+), 33 deletions(-) diff --git a/VERSION b/VERSION index 80206878..b64afe32 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.6.nskool5 +2.0.6.dev0 diff --git a/src/sagemaker_pytorch_serving_container/etc/default-ts.properties b/src/sagemaker_pytorch_serving_container/etc/default-ts.properties index f437e265..bc0f996a 100644 --- a/src/sagemaker_pytorch_serving_container/etc/default-ts.properties +++ b/src/sagemaker_pytorch_serving_container/etc/default-ts.properties @@ -1,33 +1,4 @@ # Based on https://github.com/pytorch/serve/blob/master/docs/configuration.md -#enable_envvars_config=true -#decode_input_request=false -#load_models=ALL -# - -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -inference_address=http://0.0.0.0:8080 -management_address=http://0.0.0.0:8081 -number_of_netty_threads=32 -job_queue_size=1000 -model_store=/opt/ml/model -#decode_input_request=false -#load_models= flores_modified.mar -install_py_dep_per_model=true -load_models=BERTSeqClassification.mar -models={\ - "BERTSeqClassification": {\ - "1.0": {\ - "defaultVersion": true,\ - "marName": "BERTSeqClassification.mar",\ - "minWorkers": 1,\ - "maxWorkers": 1,\ - "batchSize": 3,\ - "maxBatchDelay": 100000,\ - "responseTimeout": 120\ - }\ - }\ -} +enable_envvars_config=true +decode_input_request=false +load_models=ALL From 82c0919c8b1809cfc1840b6e86d32d907155ce09 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Fri, 24 Sep 2021 02:37:57 -0700 Subject: [PATCH 14/30] Include load models --- src/sagemaker_pytorch_serving_container/torchserve.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sagemaker_pytorch_serving_container/torchserve.py b/src/sagemaker_pytorch_serving_container/torchserve.py index e97f7b79..a931d991 100644 --- a/src/sagemaker_pytorch_serving_container/torchserve.py +++ b/src/sagemaker_pytorch_serving_container/torchserve.py @@ -175,6 +175,7 @@ def _generate_ts_config_properties(): } } user_defined_configuration["models"] = json.dumps(models) + user_defined_configuration["load_models"] = f"{DEFAULT_TS_MODEL_NAME}.mar" custom_configuration = str() From bfe053d4cec27b5c3da7f9e674ec7060b1c24f86 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Mon, 27 Sep 2021 15:20:54 -0700 Subject: [PATCH 15/30] Set max workers to 1 --- src/sagemaker_pytorch_serving_container/ts_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sagemaker_pytorch_serving_container/ts_environment.py b/src/sagemaker_pytorch_serving_container/ts_environment.py index 0f79d5f4..808d9b0b 100644 --- a/src/sagemaker_pytorch_serving_container/ts_environment.py +++ b/src/sagemaker_pytorch_serving_container/ts_environment.py @@ -26,7 +26,7 @@ DEFAULT_TS_BATCH_SIZE = 1 DEFAULT_TS_MAX_BATCH_DELAY = 100 DEFAULT_TS_MIN_WORKERS = 1 -DEFAULT_TS_MAX_WORKERS = 4 +DEFAULT_TS_MAX_WORKERS = 1 DEFAULT_TS_RESPONSE_TIMEOUT = 120 From 9b36af5d4f69cfb98469930dc6c071610f278b9d Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Tue, 28 Sep 2021 11:44:42 -0700 Subject: [PATCH 16/30] Set default response timeout to 60, and improve docstring --- src/sagemaker_pytorch_serving_container/ts_environment.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sagemaker_pytorch_serving_container/ts_environment.py b/src/sagemaker_pytorch_serving_container/ts_environment.py index 808d9b0b..c1a9eeb5 100644 --- a/src/sagemaker_pytorch_serving_container/ts_environment.py +++ b/src/sagemaker_pytorch_serving_container/ts_environment.py @@ -27,7 +27,7 @@ DEFAULT_TS_MAX_BATCH_DELAY = 100 DEFAULT_TS_MIN_WORKERS = 1 DEFAULT_TS_MAX_WORKERS = 1 -DEFAULT_TS_RESPONSE_TIMEOUT = 120 +DEFAULT_TS_RESPONSE_TIMEOUT = 60 class TorchServeEnvironment(): @@ -42,6 +42,9 @@ class TorchServeEnvironment(): max_batch_delay (int): This is the maximum batch delay time TorchServe waits to receive batch_size number of requests. If TorchServe doesn’t receive batch_size number of requests before this timer time’s out, it sends what ever requests that were received to the model handler + min_workers (int): Minimum number of workers that torchserve is allowed to scale down to. + max_workers (int): Minimum number of workers that torchserve is allowed to scale up to. + response_timeout (int): Time delay after which inference will timeout in absence of a response """ def __init__(self): self._batch_size = int(os.environ.get(ts_parameters.MODEL_SERVER_BATCH_SIZE, DEFAULT_TS_BATCH_SIZE)) From 28a1b6bb30542c37b073a5b9e2bd7e075aea5132 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Tue, 28 Sep 2021 12:34:21 -0700 Subject: [PATCH 17/30] Fix flake8 --- src/sagemaker_pytorch_serving_container/ts_environment.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sagemaker_pytorch_serving_container/ts_environment.py b/src/sagemaker_pytorch_serving_container/ts_environment.py index c1a9eeb5..d83b7c51 100644 --- a/src/sagemaker_pytorch_serving_container/ts_environment.py +++ b/src/sagemaker_pytorch_serving_container/ts_environment.py @@ -42,9 +42,9 @@ class TorchServeEnvironment(): max_batch_delay (int): This is the maximum batch delay time TorchServe waits to receive batch_size number of requests. If TorchServe doesn’t receive batch_size number of requests before this timer time’s out, it sends what ever requests that were received to the model handler - min_workers (int): Minimum number of workers that torchserve is allowed to scale down to. - max_workers (int): Minimum number of workers that torchserve is allowed to scale up to. - response_timeout (int): Time delay after which inference will timeout in absence of a response + min_workers (int): Minimum number of workers that torchserve is allowed to scale down to + max_workers (int): Minimum number of workers that torchserve is allowed to scale up to + response_timeout (int): Time delay after which inference will timeout in absence of a response """ def __init__(self): self._batch_size = int(os.environ.get(ts_parameters.MODEL_SERVER_BATCH_SIZE, DEFAULT_TS_BATCH_SIZE)) From 837f9b8536983eb306d1c973b6ee3a5e5d443ce5 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Wed, 29 Sep 2021 20:11:46 -0700 Subject: [PATCH 18/30] Add a warning log for single model --- src/sagemaker_pytorch_serving_container/torchserve.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sagemaker_pytorch_serving_container/torchserve.py b/src/sagemaker_pytorch_serving_container/torchserve.py index a931d991..c62499ab 100644 --- a/src/sagemaker_pytorch_serving_container/torchserve.py +++ b/src/sagemaker_pytorch_serving_container/torchserve.py @@ -160,7 +160,7 @@ def _generate_ts_config_properties(): ts_env = ts_environment.TorchServeEnvironment() - if ts_env.is_env_set(): + if ts_env.is_env_set() and not ENABLE_MULTI_MODEL: models = { DEFAULT_TS_MODEL_NAME: { "1.0": { @@ -175,7 +175,8 @@ def _generate_ts_config_properties(): } } user_defined_configuration["models"] = json.dumps(models) - user_defined_configuration["load_models"] = f"{DEFAULT_TS_MODEL_NAME}.mar" + logger.warn("Sagemaker TS environment variables have been set and will be used \ + for single model endpoint.") custom_configuration = str() From f09317014067bc49c498e6bc44b2f9727459c1f5 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Wed, 29 Sep 2021 20:38:47 -0700 Subject: [PATCH 19/30] Fix extra spacing in log --- src/sagemaker_pytorch_serving_container/torchserve.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sagemaker_pytorch_serving_container/torchserve.py b/src/sagemaker_pytorch_serving_container/torchserve.py index c62499ab..404a7481 100644 --- a/src/sagemaker_pytorch_serving_container/torchserve.py +++ b/src/sagemaker_pytorch_serving_container/torchserve.py @@ -175,8 +175,8 @@ def _generate_ts_config_properties(): } } user_defined_configuration["models"] = json.dumps(models) - logger.warn("Sagemaker TS environment variables have been set and will be used \ - for single model endpoint.") + logger.warn("Sagemaker TS environment variables have been set and will be used " + "for single model endpoint.") custom_configuration = str() From 7a6a103d8897de764c992ada177e58cd91caee4a Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Thu, 30 Sep 2021 13:10:54 -0700 Subject: [PATCH 20/30] Use string instead of a dict --- .../torchserve.py | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/sagemaker_pytorch_serving_container/torchserve.py b/src/sagemaker_pytorch_serving_container/torchserve.py index 404a7481..9a39e667 100644 --- a/src/sagemaker_pytorch_serving_container/torchserve.py +++ b/src/sagemaker_pytorch_serving_container/torchserve.py @@ -13,7 +13,6 @@ """This module contains functionality to configure and start Torchserve.""" from __future__ import absolute_import -import json import os import signal import subprocess @@ -161,20 +160,20 @@ def _generate_ts_config_properties(): ts_env = ts_environment.TorchServeEnvironment() if ts_env.is_env_set() and not ENABLE_MULTI_MODEL: - models = { - DEFAULT_TS_MODEL_NAME: { - "1.0": { - "defaultVersion": True, - "marName": f"{DEFAULT_TS_MODEL_NAME}.mar", - "minWorkers": ts_env._min_workers, - "maxWorkers": ts_env._max_workers, - "batchSize": ts_env._batch_size, - "maxBatchDelay": ts_env._max_batch_delay, - "responseTimeout": ts_env._response_timeout - } - } - } - user_defined_configuration["models"] = json.dumps(models) + models_string = f'''{{\\\n' + '{DEFAULT_TS_MODEL_NAME}: {{\\\n' + '"1.0": {{\\\n' + '"defaultVersion": True,\\\n' + '"marName": {DEFAULT_TS_MODEL_NAME}.mar,\\\n' + '"minWorkers": ts_env._min_workers,\\\n' + '"maxWorkers": ts_env._max_workers,\\\n' + '"batchSize": ts_env._batch_size,\\\n' + '"maxBatchDelay": ts_env._max_batch_delay,\\\n' + '"responseTimeout": ts_env._response_timeout,\\\n' + '}}\\\n' + '}}\\\n' + '}}''' + user_defined_configuration["models"] = models_string logger.warn("Sagemaker TS environment variables have been set and will be used " "for single model endpoint.") From 6040a4adf8ccbf4f3e0501642c8d2ccde036265d Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Thu, 30 Sep 2021 13:20:35 -0700 Subject: [PATCH 21/30] Print config --- src/sagemaker_pytorch_serving_container/torchserve.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sagemaker_pytorch_serving_container/torchserve.py b/src/sagemaker_pytorch_serving_container/torchserve.py index 9a39e667..00b72aa4 100644 --- a/src/sagemaker_pytorch_serving_container/torchserve.py +++ b/src/sagemaker_pytorch_serving_container/torchserve.py @@ -145,6 +145,8 @@ def _set_python_path(): def _create_torchserve_config_file(): configuration_properties = _generate_ts_config_properties() + logger.warn(f"Complete configuration:\n {configuration_properties}") + utils.write_file(TS_CONFIG_FILE, configuration_properties) From f86c7d6ee7100e144b4d4dc075545d72c4a47945 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Thu, 30 Sep 2021 13:29:53 -0700 Subject: [PATCH 22/30] Fix string --- .../torchserve.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/sagemaker_pytorch_serving_container/torchserve.py b/src/sagemaker_pytorch_serving_container/torchserve.py index 00b72aa4..73284d5e 100644 --- a/src/sagemaker_pytorch_serving_container/torchserve.py +++ b/src/sagemaker_pytorch_serving_container/torchserve.py @@ -162,19 +162,19 @@ def _generate_ts_config_properties(): ts_env = ts_environment.TorchServeEnvironment() if ts_env.is_env_set() and not ENABLE_MULTI_MODEL: - models_string = f'''{{\\\n' - '{DEFAULT_TS_MODEL_NAME}: {{\\\n' - '"1.0": {{\\\n' - '"defaultVersion": True,\\\n' - '"marName": {DEFAULT_TS_MODEL_NAME}.mar,\\\n' - '"minWorkers": ts_env._min_workers,\\\n' - '"maxWorkers": ts_env._max_workers,\\\n' - '"batchSize": ts_env._batch_size,\\\n' - '"maxBatchDelay": ts_env._max_batch_delay,\\\n' - '"responseTimeout": ts_env._response_timeout,\\\n' - '}}\\\n' - '}}\\\n' - '}}''' + models_string = f'''{{\\\n + {DEFAULT_TS_MODEL_NAME}: {{\\\n + "1.0": {{\\\n + "defaultVersion": True,\\\n + "marName": {DEFAULT_TS_MODEL_NAME}.mar,\\\n + "minWorkers": ts_env._min_workers,\\\n + "maxWorkers": ts_env._max_workers,\\\n + "batchSize": ts_env._batch_size,\\\n + "maxBatchDelay": ts_env._max_batch_delay,\\\n + "responseTimeout": ts_env._response_timeout,\\\n + }}\\\n + }}\\\n + }}''' user_defined_configuration["models"] = models_string logger.warn("Sagemaker TS environment variables have been set and will be used " "for single model endpoint.") From 7fd9f4262ed828a6fcf9adafdd47ce7f9a97201c Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Thu, 30 Sep 2021 13:40:43 -0700 Subject: [PATCH 23/30] Fix f-string --- .../torchserve.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/sagemaker_pytorch_serving_container/torchserve.py b/src/sagemaker_pytorch_serving_container/torchserve.py index 73284d5e..372de051 100644 --- a/src/sagemaker_pytorch_serving_container/torchserve.py +++ b/src/sagemaker_pytorch_serving_container/torchserve.py @@ -163,15 +163,15 @@ def _generate_ts_config_properties(): if ts_env.is_env_set() and not ENABLE_MULTI_MODEL: models_string = f'''{{\\\n - {DEFAULT_TS_MODEL_NAME}: {{\\\n + "{DEFAULT_TS_MODEL_NAME}": {{\\\n "1.0": {{\\\n - "defaultVersion": True,\\\n + "defaultVersion": true,\\\n "marName": {DEFAULT_TS_MODEL_NAME}.mar,\\\n - "minWorkers": ts_env._min_workers,\\\n - "maxWorkers": ts_env._max_workers,\\\n - "batchSize": ts_env._batch_size,\\\n - "maxBatchDelay": ts_env._max_batch_delay,\\\n - "responseTimeout": ts_env._response_timeout,\\\n + "minWorkers": {ts_env._min_workers},\\\n + "maxWorkers": {ts_env._max_workers},\\\n + "batchSize": {ts_env._batch_size},\\\n + "maxBatchDelay": {ts_env._max_batch_delay},\\\n + "responseTimeout": {ts_env._response_timeout},\\\n }}\\\n }}\\\n }}''' From 63c916ef1a43e54d30eb3a551b25861d4494470e Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Thu, 30 Sep 2021 13:50:25 -0700 Subject: [PATCH 24/30] Remove newline --- .../torchserve.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/sagemaker_pytorch_serving_container/torchserve.py b/src/sagemaker_pytorch_serving_container/torchserve.py index 372de051..44db1ebb 100644 --- a/src/sagemaker_pytorch_serving_container/torchserve.py +++ b/src/sagemaker_pytorch_serving_container/torchserve.py @@ -162,18 +162,18 @@ def _generate_ts_config_properties(): ts_env = ts_environment.TorchServeEnvironment() if ts_env.is_env_set() and not ENABLE_MULTI_MODEL: - models_string = f'''{{\\\n - "{DEFAULT_TS_MODEL_NAME}": {{\\\n - "1.0": {{\\\n - "defaultVersion": true,\\\n - "marName": {DEFAULT_TS_MODEL_NAME}.mar,\\\n - "minWorkers": {ts_env._min_workers},\\\n - "maxWorkers": {ts_env._max_workers},\\\n - "batchSize": {ts_env._batch_size},\\\n - "maxBatchDelay": {ts_env._max_batch_delay},\\\n - "responseTimeout": {ts_env._response_timeout},\\\n - }}\\\n - }}\\\n + models_string = f'''{{\\ + "{DEFAULT_TS_MODEL_NAME}": {{\\ + "1.0": {{\\ + "defaultVersion": true,\\ + "marName": "{DEFAULT_TS_MODEL_NAME}.mar",\\ + "minWorkers": {ts_env._min_workers},\\ + "maxWorkers": {ts_env._max_workers},\\ + "batchSize": {ts_env._batch_size},\\ + "maxBatchDelay": {ts_env._max_batch_delay},\\ + "responseTimeout": {ts_env._response_timeout}\\ + }}\\ + }}\\ }}''' user_defined_configuration["models"] = models_string logger.warn("Sagemaker TS environment variables have been set and will be used " From 0138d3587be52582a5490a41ba0b05acabeac358 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Thu, 30 Sep 2021 14:26:49 -0700 Subject: [PATCH 25/30] Adjust f string --- .../etc/default-ts.properties | 2 +- src/sagemaker_pytorch_serving_container/torchserve.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sagemaker_pytorch_serving_container/etc/default-ts.properties b/src/sagemaker_pytorch_serving_container/etc/default-ts.properties index bc0f996a..14e1c1f9 100644 --- a/src/sagemaker_pytorch_serving_container/etc/default-ts.properties +++ b/src/sagemaker_pytorch_serving_container/etc/default-ts.properties @@ -1,4 +1,4 @@ # Based on https://github.com/pytorch/serve/blob/master/docs/configuration.md enable_envvars_config=true -decode_input_request=false +#decode_input_request=false load_models=ALL diff --git a/src/sagemaker_pytorch_serving_container/torchserve.py b/src/sagemaker_pytorch_serving_container/torchserve.py index 44db1ebb..555dc604 100644 --- a/src/sagemaker_pytorch_serving_container/torchserve.py +++ b/src/sagemaker_pytorch_serving_container/torchserve.py @@ -172,9 +172,9 @@ def _generate_ts_config_properties(): "batchSize": {ts_env._batch_size},\\ "maxBatchDelay": {ts_env._max_batch_delay},\\ "responseTimeout": {ts_env._response_timeout}\\ + }}\\ }}\\ - }}\\ - }}''' + }}''' user_defined_configuration["models"] = models_string logger.warn("Sagemaker TS environment variables have been set and will be used " "for single model endpoint.") From d42daa40d9006f90e74e724dcee4d993817f817b Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Thu, 30 Sep 2021 15:35:26 -0700 Subject: [PATCH 26/30] Fix flake8 --- .../etc/default-ts.properties | 2 +- src/sagemaker_pytorch_serving_container/torchserve.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/sagemaker_pytorch_serving_container/etc/default-ts.properties b/src/sagemaker_pytorch_serving_container/etc/default-ts.properties index 14e1c1f9..bc0f996a 100644 --- a/src/sagemaker_pytorch_serving_container/etc/default-ts.properties +++ b/src/sagemaker_pytorch_serving_container/etc/default-ts.properties @@ -1,4 +1,4 @@ # Based on https://github.com/pytorch/serve/blob/master/docs/configuration.md enable_envvars_config=true -#decode_input_request=false +decode_input_request=false load_models=ALL diff --git a/src/sagemaker_pytorch_serving_container/torchserve.py b/src/sagemaker_pytorch_serving_container/torchserve.py index 555dc604..62cd8cb1 100644 --- a/src/sagemaker_pytorch_serving_container/torchserve.py +++ b/src/sagemaker_pytorch_serving_container/torchserve.py @@ -145,8 +145,6 @@ def _set_python_path(): def _create_torchserve_config_file(): configuration_properties = _generate_ts_config_properties() - logger.warn(f"Complete configuration:\n {configuration_properties}") - utils.write_file(TS_CONFIG_FILE, configuration_properties) From 61a7d0bd6f2fb4a2e78082d4657720e964ad32f1 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Thu, 30 Sep 2021 17:21:09 -0700 Subject: [PATCH 27/30] Trigger build From 2ccf3b652370d34d1e8ad8767b66560d9d343c74 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Fri, 1 Oct 2021 01:27:05 -0700 Subject: [PATCH 28/30] Trigger build From 59db6296ba4263afb6df495162ec879ba5684171 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Fri, 1 Oct 2021 10:32:20 -0700 Subject: [PATCH 29/30] Trigger build From a14faf1092887c82a5e39f6ba28c6cade439b5e8 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Fri, 1 Oct 2021 11:06:14 -0700 Subject: [PATCH 30/30] Trigger build