From 587154e473c8088e14bb79d2f237147caa824ba3 Mon Sep 17 00:00:00 2001 From: Haoran Peng Date: Sat, 5 Nov 2022 21:20:10 +0000 Subject: [PATCH 1/8] adding redis cache decorator --- ts/utils/redis_cache.py | 57 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 ts/utils/redis_cache.py diff --git a/ts/utils/redis_cache.py b/ts/utils/redis_cache.py new file mode 100644 index 0000000000..04c08f89fa --- /dev/null +++ b/ts/utils/redis_cache.py @@ -0,0 +1,57 @@ +import logging +import pickle +from functools import wraps + +import redis + +from ts.context import Context + + +def _make_key(args, kwds): + key = args + if kwds: + key += (object(),) + for item in kwds.items(): + key += item + return pickle.dumps(key) + + +def _no_op_decorator(func): + @wraps(func) + def wrapper(*args, **kwds): + return func(*args, **kwds) + + return wrapper + + +def handler_cache(host, port, db, maxsize=128): + r = redis.Redis(host=host, port=port, db=db) + try: + r.ping() + except ConnectionError as e: + logging.info( + f"Cannot connect to a redis server, ensure a server is running on {host}:{port}" + ) + raise e + + def decorating_function(func): + @wraps(func) + def wrapper(*args, **kwds): + # Removing Context objects from key hashing + key = _make_key( + args=[arg for arg in args if not isinstance(arg, Context)], + kwds={k: v for (k, v) in kwds.items() if not isinstance(v, Context)}, + ) + value_str = r.get(key) + if value_str is not None: + return pickle.loads(value_str) # might need to decode + value = func(*args, **kwds) + # Randomly remove one entry if maxsize is reached + if r.dbsize() >= maxsize: + r.delete(r.randomkey()) + r.set(key, pickle.dumps(value)) + return value + + return wrapper + + return decorating_function From 26e81e555e0a9a188c1f4befaed111be3466373b Mon Sep 17 00:00:00 2001 From: Haoran Peng Date: Sat, 5 Nov 2022 21:30:19 +0000 Subject: [PATCH 2/8] remove comment --- ts/utils/redis_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ts/utils/redis_cache.py b/ts/utils/redis_cache.py index 04c08f89fa..fd034ee483 100644 --- a/ts/utils/redis_cache.py +++ b/ts/utils/redis_cache.py @@ -44,7 +44,7 @@ def wrapper(*args, **kwds): ) value_str = r.get(key) if value_str is not None: - return pickle.loads(value_str) # might need to decode + return pickle.loads(value_str) value = func(*args, **kwds) # Randomly remove one entry if maxsize is reached if r.dbsize() >= maxsize: From 54286c94cd80c8b773311406592f430cd06f93d0 Mon Sep 17 00:00:00 2001 From: Haoran Peng Date: Sun, 6 Nov 2022 20:36:46 +0000 Subject: [PATCH 3/8] do not raise --- ts/utils/redis_cache.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/ts/utils/redis_cache.py b/ts/utils/redis_cache.py index fd034ee483..bd697f04f1 100644 --- a/ts/utils/redis_cache.py +++ b/ts/utils/redis_cache.py @@ -2,7 +2,12 @@ import pickle from functools import wraps -import redis +try: + import redis + + _has_redis = True +except ImportError: + _has_redis = False from ts.context import Context @@ -25,14 +30,28 @@ def wrapper(*args, **kwds): def handler_cache(host, port, db, maxsize=128): + """Decorator for handler's handle() method that cache input/output to a redis database. + + A typical usage would be: + + class SomeHandler(BaseHandler): + def __init__(self): + ... + self.handle = handler_cache(host='localhost', port=6379, db=0, maxsize=128)(self.handle) + + The user should ensure that both the input and the output can be pickled. + """ + if not _has_redis: + logging.error(f"Cannot import redis, try pip install redis.") + return _no_op_decorator r = redis.Redis(host=host, port=port, db=db) try: r.ping() - except ConnectionError as e: - logging.info( - f"Cannot connect to a redis server, ensure a server is running on {host}:{port}" + except redis.exceptions.ConnectionError: + logging.error( + f"Cannot connect to a redis server, ensure a server is running on {host}:{port}." ) - raise e + return _no_op_decorator def decorating_function(func): @wraps(func) From e709e22a4bd0fa547fa79fbb7989de70bb24dd92 Mon Sep 17 00:00:00 2001 From: Haoran Peng Date: Sun, 6 Nov 2022 20:48:50 +0000 Subject: [PATCH 4/8] partial commit --- examples/redis_cache/README.md | 106 +++++++++++++++++++ examples/redis_cache/mnist_handler_cached.py | 10 ++ 2 files changed, 116 insertions(+) create mode 100644 examples/redis_cache/README.md create mode 100644 examples/redis_cache/mnist_handler_cached.py diff --git a/examples/redis_cache/README.md b/examples/redis_cache/README.md new file mode 100644 index 0000000000..d800dc72e5 --- /dev/null +++ b/examples/redis_cache/README.md @@ -0,0 +1,106 @@ +# Caching with redis database + +In this example, we show how to cache input/output of our handler to reduce resource utilization. + +The + +We used the following pytorch example to train the basic MNIST model for digit recognition : +https://github.com/pytorch/examples/tree/master/mnist + +# Objective +1. Demonstrate how to package a custom trained model with custom handler into torch model archive (.mar) file +2. Demonstrate how to create model handler code +3. Demonstrate how to load model archive (.mar) file into TorchServe and run inference. + +# Serve a custom model on TorchServe + +Run the commands given in following steps from the parent directory of the root of the repository. For example, if you cloned the repository into /home/my_path/serve, run the steps from /home/my_path + + * Step - 1: Create a new model architecture file which contains model class extended from torch.nn.modules. In this example we have created [mnist model file](mnist.py). + * Step - 2: Train a MNIST digit recognition model using https://github.com/pytorch/examples/blob/master/mnist/main.py and save the state dict of model. We have added the pre-created [state dict](mnist_cnn.pt) of this model. + * Step - 3: Write a custom handler to run the inference on your model. In this example, we have added a [custom_handler](mnist_handler.py) which runs the inference on the input grayscale images using the above model and recognizes the digit in the image. + * Step - 4: Create a torch model archive using the torch-model-archiver utility to archive the above files. + + ```bash + torch-model-archiver --model-name mnist --version 1.0 --model-file examples/image_classifier/mnist/mnist.py --serialized-file examples/image_classifier/mnist/mnist_cnn.pt --handler examples/image_classifier/mnist/mnist_handler.py + ``` + + Step 5 is optional. Perform this step to use pytorch profiler + + * Step - 5: To enable pytorch profiler, set the following environment variable. + + ``` + export ENABLE_TORCH_PROFILER=true + ``` + + * Step - 6: Register the model on TorchServe using the above model archive file and run digit recognition inference + + ```bash + mkdir model_store + mv mnist.mar model_store/ + torchserve --start --model-store model_store --models mnist=mnist.mar --ts-config config.properties + curl http://127.0.0.1:8080/predictions/mnist -T examples/image_classifier/mnist/test_data/0.png + ``` + +# Profiling inference output + +The profiler information is printed in the torchserve logs / console + +![Profiler Stats](screenshots/mnist_profiler_stats.png) + +By default the pytorch profiler trace files are generated under "/tmp/pytorch_profiler/" directory. + +The path can be overridden by setting `on_trace_ready` parameter in `profiler_args` - [Example here](../../../test/pytest/profiler_utils/resnet_profiler_override.py) + +And the trace files can be loaded in tensorboard using torch-tb-profiler. Check the following link for more information - https://github.com/pytorch/kineto/tree/main/tb_plugin + +Install torch-tb-profiler and run the following command to view the results in UI + +``` +tensorboard --logdir /tmp/pytorch_profiler/mnist/ +``` + +The pytorch profiler traces can be viewed as below + +![Pytorch Profiler UI](screenshots/mnist_trace.png) + +For captum Explanations on the Torchserve side, use the below curl request: +```bash +curl http://127.0.0.1:8080/explanations/mnist -T examples/image_classifier/mnist/test_data/0.png +``` + +In order to run Captum Explanations with the request input in a json file, follow the below steps: + +In the config.properties, specify `service_envelope=body` and make the curl request as below: +```bash +curl -H "Content-Type: application/json" --data @examples/image_classifier/mnist/mnist_ts.json http://127.0.0.1:8080/explanations/mnist_explain +``` +When a json file is passed as a request format to the curl, Torchserve unwraps the json file from the request body. This is the reason for specifying service_envelope=body in the config.properties file + +### Captum Explanations + +The explain is called with the following request api `http://127.0.0.1:8080/explanations/mnist_explain` + +Torchserve supports Captum Explanations for Eager models only. + +Captum/Explain doesn't support batching. + +#### The handler changes: + +1. The handlers should initialize. +```python +self.ig = IntegratedGradients(self.model) +``` +in the initialize function for the captum to work.(It is initialized in the base class-vision_handler) + +2. The Base handler handle uses the explain_handle method to perform captum insights based on whether user wants predictions or explanations. These methods can be overriden to make your changes in the handler. + +3. The get_insights method in the handler is called by the explain_handle method to calculate insights using captum. + +4. If the custom handler overrides handle function of base handler, the explain_handle function should be called to get captum insights. + +### Running KServe + +Refer the [MNIST Readme for KServe](https://github.com/pytorch/serve/blob/master/kubernetes/kserve/kserve_wrapper/README.md) to run it locally. + +Refer the [End to End KServe document](https://github.com/pytorch/serve/blob/master/kubernetes/kserve/README.md) to run it in the cluster. diff --git a/examples/redis_cache/mnist_handler_cached.py b/examples/redis_cache/mnist_handler_cached.py new file mode 100644 index 0000000000..5b0cb87025 --- /dev/null +++ b/examples/redis_cache/mnist_handler_cached.py @@ -0,0 +1,10 @@ +from examples.image_classifier.mnist.mnist_handler import MNISTDigitClassifier +from ts.utils.redis_cache import handler_cache + + +class MNISTDigitClassifierCached(MNISTDigitClassifier): + def __init__(self): + super(MNISTDigitClassifierCached, self).__init__() + self.handle = handler_cache(host="localhost", port=6379, db=0, maxsize=2)( + self.handle + ) From 24e9f92b3eebd49bca314b0c7c10434828f5917d Mon Sep 17 00:00:00 2001 From: Haoran Peng Date: Sun, 6 Nov 2022 22:39:05 +0000 Subject: [PATCH 5/8] update readme --- examples/redis_cache/README.md | 116 +++++++----------------- ts/utils/redis_cache.py | 4 +- ts_scripts/spellcheck_conf/wordlist.txt | 1 + 3 files changed, 35 insertions(+), 86 deletions(-) diff --git a/examples/redis_cache/README.md b/examples/redis_cache/README.md index d800dc72e5..8c068cb8b8 100644 --- a/examples/redis_cache/README.md +++ b/examples/redis_cache/README.md @@ -1,106 +1,54 @@ -# Caching with redis database +# Caching with Redis database -In this example, we show how to cache input/output of our handler to reduce resource utilization. +We will build a minimal working example that uses a Redis server to cache the input/output of a custom handler. -The +The example will be based on the [MNIST classifier example](https://github.com/pytorch/serve/tree/master/examples/image_classifier/mnist). -We used the following pytorch example to train the basic MNIST model for digit recognition : -https://github.com/pytorch/examples/tree/master/mnist +### Pre-requisites -# Objective -1. Demonstrate how to package a custom trained model with custom handler into torch model archive (.mar) file -2. Demonstrate how to create model handler code -3. Demonstrate how to load model archive (.mar) file into TorchServe and run inference. - -# Serve a custom model on TorchServe - -Run the commands given in following steps from the parent directory of the root of the repository. For example, if you cloned the repository into /home/my_path/serve, run the steps from /home/my_path - - * Step - 1: Create a new model architecture file which contains model class extended from torch.nn.modules. In this example we have created [mnist model file](mnist.py). - * Step - 2: Train a MNIST digit recognition model using https://github.com/pytorch/examples/blob/master/mnist/main.py and save the state dict of model. We have added the pre-created [state dict](mnist_cnn.pt) of this model. - * Step - 3: Write a custom handler to run the inference on your model. In this example, we have added a [custom_handler](mnist_handler.py) which runs the inference on the input grayscale images using the above model and recognizes the digit in the image. - * Step - 4: Create a torch model archive using the torch-model-archiver utility to archive the above files. +- Redis is installed on your system and a server running. Follow the [Redis getting started guide](https://redis.io/docs/getting-started/) to set up a Redis server. +- The [Python Redis interface](https://github.com/redis/redis-py) is installed: ```bash - torch-model-archiver --model-name mnist --version 1.0 --model-file examples/image_classifier/mnist/mnist.py --serialized-file examples/image_classifier/mnist/mnist_cnn.pt --handler examples/image_classifier/mnist/mnist_handler.py + pip install redis ``` - Step 5 is optional. Perform this step to use pytorch profiler - - * Step - 5: To enable pytorch profiler, set the following environment variable. +We will assume a Redis server is started on `localhost` at port `6379`. - ``` - export ENABLE_TORCH_PROFILER=true - ``` - - * Step - 6: Register the model on TorchServe using the above model archive file and run digit recognition inference - - ```bash - mkdir model_store - mv mnist.mar model_store/ - torchserve --start --model-store model_store --models mnist=mnist.mar --ts-config config.properties - curl http://127.0.0.1:8080/predictions/mnist -T examples/image_classifier/mnist/test_data/0.png - ``` +### Using the `ts.utils.redis_cache.handler_cache` decorator -# Profiling inference output +The decorator's usage is similar to that of the built-in `functools.lru_cache`. -The profiler information is printed in the torchserve logs / console - -![Profiler Stats](screenshots/mnist_profiler_stats.png) - -By default the pytorch profiler trace files are generated under "/tmp/pytorch_profiler/" directory. - -The path can be overridden by setting `on_trace_ready` parameter in `profiler_args` - [Example here](../../../test/pytest/profiler_utils/resnet_profiler_override.py) - -And the trace files can be loaded in tensorboard using torch-tb-profiler. Check the following link for more information - https://github.com/pytorch/kineto/tree/main/tb_plugin - -Install torch-tb-profiler and run the following command to view the results in UI +A typical usage would be: +```python +from ts.utils.redis_cache import handler_cache +class SomeHandler(BaseHandler): + def __init__(self): + ... + self.handle = handler_cache(host='localhost', port=6379, db=0, maxsize=128)(self.handle) ``` -tensorboard --logdir /tmp/pytorch_profiler/mnist/ -``` +See [mnist_handler_cached.py](https://github.com/pytorch/serve/tree/master/examples/redis_cache/mnist_handler_cached.py) for a minimal concrete example. -The pytorch profiler traces can be viewed as below +### Package and serve the model as usual -![Pytorch Profiler UI](screenshots/mnist_trace.png) - -For captum Explanations on the Torchserve side, use the below curl request: +Execute commands from the project root: ```bash -curl http://127.0.0.1:8080/explanations/mnist -T examples/image_classifier/mnist/test_data/0.png +torch-model-archiver --model-name mnist --version 1.0 --model-file examples/image_classifier/mnist/mnist.py --serialized-file examples/image_classifier/mnist/mnist_cnn.pt --handler examples/redis_cache/mnist_handler_cached.py +mkdir -p model_store +mv mnist.mar model_store/ +torchserve --start --model-store model_store --models mnist=mnist.mar --ts-config examples/image_classifier/mnist/config.properties ``` -In order to run Captum Explanations with the request input in a json file, follow the below steps: - -In the config.properties, specify `service_envelope=body` and make the curl request as below: +Run inference using: ```bash -curl -H "Content-Type: application/json" --data @examples/image_classifier/mnist/mnist_ts.json http://127.0.0.1:8080/explanations/mnist_explain -``` -When a json file is passed as a request format to the curl, Torchserve unwraps the json file from the request body. This is the reason for specifying service_envelope=body in the config.properties file - -### Captum Explanations - -The explain is called with the following request api `http://127.0.0.1:8080/explanations/mnist_explain` - -Torchserve supports Captum Explanations for Eager models only. - -Captum/Explain doesn't support batching. - -#### The handler changes: - -1. The handlers should initialize. -```python -self.ig = IntegratedGradients(self.model) +curl http://127.0.0.1:8080/predictions/mnist -T examples/image_classifier/mnist/test_data/0.png +# The second call will return the cached result +curl http://127.0.0.1:8080/predictions/mnist -T examples/image_classifier/mnist/test_data/0.png ``` -in the initialize function for the captum to work.(It is initialized in the base class-vision_handler) - -2. The Base handler handle uses the explain_handle method to perform captum insights based on whether user wants predictions or explanations. These methods can be overriden to make your changes in the handler. - -3. The get_insights method in the handler is called by the explain_handle method to calculate insights using captum. - -4. If the custom handler overrides handle function of base handler, the explain_handle function should be called to get captum insights. - -### Running KServe -Refer the [MNIST Readme for KServe](https://github.com/pytorch/serve/blob/master/kubernetes/kserve/kserve_wrapper/README.md) to run it locally. +### Breif note on performance +The input and output are both serialized (by pickle) before being put into the cache. +The output also needs to be retrieved and deserialized at a cache hit. -Refer the [End to End KServe document](https://github.com/pytorch/serve/blob/master/kubernetes/kserve/README.md) to run it in the cluster. +If the input and/or output are very large objects, these serialization process might take a while and longer keys take longer to compare. diff --git a/ts/utils/redis_cache.py b/ts/utils/redis_cache.py index bd697f04f1..bdbc8c7668 100644 --- a/ts/utils/redis_cache.py +++ b/ts/utils/redis_cache.py @@ -30,7 +30,7 @@ def wrapper(*args, **kwds): def handler_cache(host, port, db, maxsize=128): - """Decorator for handler's handle() method that cache input/output to a redis database. + """Decorator for handler's handle() method that cache input/output to a Redis database. A typical usage would be: @@ -49,7 +49,7 @@ def __init__(self): r.ping() except redis.exceptions.ConnectionError: logging.error( - f"Cannot connect to a redis server, ensure a server is running on {host}:{port}." + f"Cannot connect to a Redis server, ensure a server is running on {host}:{port}." ) return _no_op_decorator diff --git a/ts_scripts/spellcheck_conf/wordlist.txt b/ts_scripts/spellcheck_conf/wordlist.txt index 1848cef711..84cc8d89dc 100644 --- a/ts_scripts/spellcheck_conf/wordlist.txt +++ b/ts_scripts/spellcheck_conf/wordlist.txt @@ -985,3 +985,4 @@ minioadmin pythonic Diffusers diffusers +Redis From 7bd9c901c9b0800ff7f7740b30756b9180ba9518 Mon Sep 17 00:00:00 2001 From: Haoran Peng Date: Sun, 6 Nov 2022 23:20:59 +0000 Subject: [PATCH 6/8] update readme --- examples/redis_cache/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/redis_cache/README.md b/examples/redis_cache/README.md index 8c068cb8b8..db650d81ea 100644 --- a/examples/redis_cache/README.md +++ b/examples/redis_cache/README.md @@ -13,7 +13,9 @@ The example will be based on the [MNIST classifier example](https://github.com/p pip install redis ``` -We will assume a Redis server is started on `localhost` at port `6379`. +Note that if the pre-requisites are not met, a no op decorator will be used and no exceptions will be raised. + +We will now assume a Redis server is started on `localhost` at port `6379`. ### Using the `ts.utils.redis_cache.handler_cache` decorator From a60d6a4558b8860f2fde6dae375c97092ade5576 Mon Sep 17 00:00:00 2001 From: Haoran Peng Date: Mon, 7 Nov 2022 17:42:47 +0000 Subject: [PATCH 7/8] update readme --- examples/redis_cache/README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/redis_cache/README.md b/examples/redis_cache/README.md index 8c068cb8b8..4c12d7906f 100644 --- a/examples/redis_cache/README.md +++ b/examples/redis_cache/README.md @@ -6,15 +6,19 @@ The example will be based on the [MNIST classifier example](https://github.com/p ### Pre-requisites -- Redis is installed on your system and a server running. Follow the [Redis getting started guide](https://redis.io/docs/getting-started/) to set up a Redis server. - +- Redis is installed on your system. Follow the [Redis getting started guide](https://redis.io/docs/getting-started/) to install Redis. + + Start a Redis server using (the server will be started on `localhost:6379` by default): + ```bash + redis-server + # optionally specify the port: + # redis-server --port 6379 + ``` - The [Python Redis interface](https://github.com/redis/redis-py) is installed: ```bash pip install redis ``` -We will assume a Redis server is started on `localhost` at port `6379`. - ### Using the `ts.utils.redis_cache.handler_cache` decorator The decorator's usage is similar to that of the built-in `functools.lru_cache`. From c506b38921f0ff2736e9e1b28246545bbc3a4dec Mon Sep 17 00:00:00 2001 From: Haoran Peng Date: Fri, 11 Nov 2022 10:59:57 +0000 Subject: [PATCH 8/8] moving redis cache to examples folder --- examples/redis_cache/mnist_handler_cached.py | 2 +- {ts/utils => examples/redis_cache}/redis_cache.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename {ts/utils => examples/redis_cache}/redis_cache.py (100%) diff --git a/examples/redis_cache/mnist_handler_cached.py b/examples/redis_cache/mnist_handler_cached.py index 5b0cb87025..170047d6d7 100644 --- a/examples/redis_cache/mnist_handler_cached.py +++ b/examples/redis_cache/mnist_handler_cached.py @@ -1,5 +1,5 @@ from examples.image_classifier.mnist.mnist_handler import MNISTDigitClassifier -from ts.utils.redis_cache import handler_cache +from examples.redis_cache.redis_cache import handler_cache class MNISTDigitClassifierCached(MNISTDigitClassifier): diff --git a/ts/utils/redis_cache.py b/examples/redis_cache/redis_cache.py similarity index 100% rename from ts/utils/redis_cache.py rename to examples/redis_cache/redis_cache.py