Stage release

benchmarks/README.md

@@ -8,6 +8,20 @@ The benchmarks measure the performance of TorchServe on various models and bench
 
 The script is mainly intended to run on a Ubuntu EC2 instance.  For this reason, we have provided an `install_dependencies.sh` script to install everything needed to execute the benchmark on this environment.  All you need to do is run this file and clone the TorchServe repo.
 
+While installing JMeter through brew, the `install_depdendencies.sh` script asks for following command line input.  
+```bash
+Installing JMeter through Brew
++ yes ''
++ brew update
+==> Select the Linuxbrew installation directory
+- Enter your password to install to /home/linuxbrew/.linuxbrew (recommended)
+- Press Control-D to install to /home/ubuntu/.linuxbrew
+- Press Control-C to cancel installation
+[sudo] password for ubuntu: 
+```
+
+Here `Press Control-D to install to /home/ubuntu/.linuxbrew` as the `ubuntu` user on EC2 node has password-less sudo access.
+
 ### MacOS
 
 For mac, you should have python3 and java installed.  If you wish to run the default benchmarks featuring a docker-based instance of TorchServe, you will need to install docker as well.  Finally, you will need to install jmeter with plugins which can be accomplished by running `mac_install_dependencies.sh`.

docs/default_handlers.md

@@ -0,0 +1,36 @@
+# TorchServe default inference handlers
+
+TorchServe provides following inference handlers out of box :
+
+## image_classifier
+
+ * Description : Handles image classification models trained on imagenet dataset.
+ * Input : RGB image
+ * Output : Top 5 predictions and their respective probability of the image
+
+For more details refer [examples](../examples/image_classifier)
+
+## image_segmenter
+
+ * Description : Handles image segmentation models trained on imagenet dataset.
+ * Input : RGB image
+ * Output : Output shape as [CL H W], CL - number of classes, H - height and W - width.
+
+For more details refer [examples](../examples/image_segmenter)
+
+## object_detector
+
+ * Description : Handles object detection models.
+ * Input : RGB image
+ * Output : List of detected classes and bounding boxes respectively 
+
+For more details refer [examples](../examples/object_detector) 
+
+## text_classifier
+
+ * Description : Handles models trained on imagenet dataset.
+ * Input : text file
+ * Output : Class of input text
+
+For more details refer [examples](../examples/text_classification)
+

docs/metrics.md

@@ -28,9 +28,9 @@ The location of log files and metric files can be configured at [log4j.propertie
 |	MemoryAvailable	|	host	|	MB	|	memory available on host	|
 |	MemoryUsed	|	host	|	MB	|	memory used on host	|
 |	MemoryUtilization	|	host	|	percentage	|	memory used on host	|
-|	Requests2XX	|	host	|	count	|	total number of requests that responded in 200-300 range	|
-|	Requests4XX	|	host	|	count	|	total number of requests that responded in 400-500 range |
-|	Requests5XX	|	host	|	count	|	total number of requests that responded above 500 |
+|	Requests2XX	|	host	|	count	|	logged for every request responded in 200-300 status code range	|
+|	Requests4XX	|	host	|	count	|	logged for every request responded in 400-500 status code range |
+|	Requests5XX	|	host	|	count	|	logged for every request responded with status code above 500 |
 
 
 ## Formatting

frontend/server/src/test/java/org/pytorch/serve/ModelServerTest.java

@@ -213,15 +213,15 @@ public void testTS()
         Channel channel = null;
         Channel managementChannel = null;
         for (int i = 0; i < 5; ++i) {
-            channel = connect(false);
+            channel = connect(false, 300);
             if (channel != null) {
                 break;
             }
             Thread.sleep(100);
         }
 
         for (int i = 0; i < 5; ++i) {
-            managementChannel = connect(true);
+            managementChannel = connect(true, 300);
             if (managementChannel != null) {
                 break;
             }
@@ -1328,7 +1328,7 @@ private void testMetricManager() throws JsonParseException, InterruptedException
         }
     }
 
-    private Channel connect(boolean management) {
+    private Channel connect(boolean management, int readTimeOut) {
         Logger logger = LoggerFactory.getLogger(ModelServerTest.class);
 
         final Connector connector = configManager.getListener(management);
@@ -1349,7 +1349,7 @@ public void initChannel(Channel ch) {
                                     if (connector.isSsl()) {
                                         p.addLast(sslCtx.newHandler(ch.alloc()));
                                     }
-                                    p.addLast(new ReadTimeoutHandler(30));
+                                    p.addLast(new ReadTimeoutHandler(readTimeOut));
                                     p.addLast(new HttpClientCodec());
                                     p.addLast(new HttpContentDecompressor());
                                     p.addLast(new ChunkedWriteHandler());
@@ -1365,6 +1365,10 @@ public void initChannel(Channel ch) {
         return null;
     }
 
+    private Channel connect(boolean management) {
+        return connect(management, 120);
+    }
+
     @ChannelHandler.Sharable
     private class TestHandler extends SimpleChannelInboundHandler<FullHttpResponse> {
 

ts/torch_handler/object_detector.py

@@ -1,4 +1,5 @@
 import io
+import torch
 from PIL import Image
 from torchvision import transforms
 from torch.autograd import Variable
@@ -14,6 +15,12 @@ class ObjectDetector(VisionHandler):
     def __init__(self):
         super(ObjectDetector, self).__init__()
 
+    def initialize(self, ctx):
+        super(VisionHandler, self).initialize(ctx)
+        self.initialized = False
+        self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+        self.initialized = True
+
     def preprocess(self, data):
         """
          Scales, crops, and normalizes a image for a PyTorch model,
@@ -32,10 +39,12 @@ def inference(self, img, threshold=0.5):
         # Predict the classes and bounding boxes in an image using a trained deep learning model.
         img = Variable(img).to(self.device)
         pred = self.model([img])  # Pass the image to the model
-        pred_class = [i for i in list(pred[0]['labels'].numpy())]  # Get the Prediction Score
-        pred_boxes = [[(i[0], i[1]), (i[2], i[3])] for i in list(pred[0]['boxes'].detach().numpy())]  # Bounding boxes
-        pred_score = list(pred[0]['scores'].detach().numpy())
-        pred_t = [pred_score.index(x) for x in pred_score if x > threshold][-1]  # Get list of index with score greater than threshold.
+        pred_class = [i for i in list(pred[0]['labels'].cpu().numpy())]  # Get the Prediction Score
+        # Bounding boxes
+        pred_boxes = [[(i[0], i[1]), (i[2], i[3])] for i in list(pred[0]['boxes'].cpu().detach().numpy())]
+        pred_score = list(pred[0]['scores'].cpu().detach().numpy())
+        # Get list of index with score greater than threshold.
+        pred_t = [pred_score.index(x) for x in pred_score if x > threshold][-1]
         pred_boxes = pred_boxes[:pred_t + 1]
         pred_class = pred_class[:pred_t + 1]
         return [pred_class, pred_boxes]