E2E Tests

.gitignore

@@ -9,6 +9,7 @@ __pycache__/
 .python-version
 .env
 .venv
+*.egg-info
 
 # OSX
 .DS_Store

CONTRIBUTING.md

@@ -228,7 +228,11 @@ export CORTEX_DEV_DEFAULT_PREDICTOR_IMAGE_REGISTRY="cortexlabs"
 export CORTEX_TELEMETRY_SENTRY_DSN="https://c334df915c014ffa93f2076769e5b334@sentry.io/1848098"
 export CORTEX_TELEMETRY_SEGMENT_WRITE_KEY="0WvoJyCey9z1W2EW7rYTPJUMRYat46dl"
 
-alias cortex='$HOME/bin/cortex'  # your path may be different depending on where you cloned the repo
+# instruct the Python client to use your development CLI binary (update the path to point to your cortex repo)
+export CORTEX_CLI_PATH="<cortex_repo_path>/bin/cortex"
+
+# create a cortex alias which runs your development CLI
+alias cortex="$CORTEX_CLI_PATH"
 ```
 
 Refresh your bash profile:

test/README.md

@@ -1,67 +1,5 @@
-# Examples
+# Cortex Tests
 
-## TensorFlow
-
-- [Iris classification](tensorflow/iris-classifier): deploy a model to classify iris flowers.
-
-- [Text generation](tensorflow/text-generator): deploy OpenAI's GPT-2 to generate text.
-
-- [Sentiment analysis](tensorflow/sentiment-analyzer): deploy a BERT model for sentiment analysis.
-
-- [Image classification](tensorflow/image-classifier-inception): deploy an Inception model to classify images.
-
-- [Image classification](tensorflow/image-classifier-resnet50): deploy a ResNet50 model to classify images.
-
-- [License plate reader](tensorflow/license-plate-reader): deploy a YOLOv3 model (and others) to identify license plates in real time.
-
-- [Multi-model classification](tensorflow/multi-model-classifier): deploy 3 models (ResNet50, Iris, Inception) in a single API.
-
-## Keras
-
-- [Denoisify text documents](keras/document-denoiser): deploy an Autoencoder model to clean text document images of noise.
-
-## PyTorch
-
-- [Iris classification](pytorch/iris-classifier): deploy a model to classify iris flowers.
-
-- [Text generation](pytorch/text-generator): deploy Hugging Face's GPT-2 model to generate text.
-
-- [Sentiment analysis](pytorch/sentiment-analyzer): deploy a Hugging Face transformers model for sentiment analysis.
-
-- [Search completion](pytorch/search-completer): deploy a Facebook's RoBERTa model to complete search terms.
-
-- [Answer generation](pytorch/answer-generator): deploy Microsoft's DialoGPT model to answer questions.
-
-- [Text summarization](pytorch/text-summarizer): deploy a BART model (from Hugging Face's transformers library) to summarize text.
-
-- [Reading comprehension](pytorch/reading-comprehender): deploy an AllenNLP model for reading comprehension.
-
-- [Language identification](pytorch/language-identifier): deploy a fastText model to identify languages.
-
-- [Multi-model text analysis](pytorch/multi-model-text-analyzer): deploy 2 models (Sentiment and Summarization analyzers) in a single API.
-
-- [Image classification](pytorch/image-classifier-alexnet): deploy an AlexNet model from TorchVision to classify images.
-
-- [Image classification](pytorch/image-classifier-resnet50): deploy a ResNet50 model from TorchVision to classify images.
-
-- [Object detection](pytorch/object-detector): deploy a Faster R-CNN model from TorchVision to detect objects in images.
-
-- [Question generator](pytorch/question-generator): deploy a transformers model to generate questions given text and the correct answer.
-
-## ONNX
-
-- [Iris classification](onnx/iris-classifier): deploy an XGBoost model (exported in ONNX) to classify iris flowers.
-
-- [YOLOv5 YouTube detection](onnx/yolov5-youtube): deploy a YOLOv5 model trained on COCO val2017 dataset.
-
-- [Multi-model classification](onnx/multi-model-classifier): deploy 3 models (ResNet50, MobileNet, ShuffleNet) in a single API.
-
-## scikit-learn
-
-- [Iris classification](sklearn/iris-classifier): deploy a model to classify iris flowers.
-
-- [MPG estimation](sklearn/mpg-estimator): deploy a linear regression model to estimate MPG.
-
-## spacy
-
-- [Entity recognizer](spacy/entity-recognizer): deploy a spacy model for named entity recognition.
+- [Example APIs](apis)
+- [End-to-end Tests](e2e)
+- [Testing Utilities](utils)

test/apis/README.md

@@ -0,0 +1,67 @@
+# Examples
+
+## TensorFlow
+
+- [Iris classification](tensorflow/iris-classifier): deploy a model to classify iris flowers.
+
+- [Text generation](tensorflow/text-generator): deploy OpenAI's GPT-2 to generate text.
+
+- [Sentiment analysis](tensorflow/sentiment-analyzer): deploy a BERT model for sentiment analysis.
+
+- [Image classification](tensorflow/image-classifier-inception): deploy an Inception model to classify images.
+
+- [Image classification](tensorflow/image-classifier-resnet50): deploy a ResNet50 model to classify images.
+
+- [License plate reader](tensorflow/license-plate-reader): deploy a YOLOv3 model (and others) to identify license plates in real time.
+
+- [Multi-model classification](tensorflow/multi-model-classifier): deploy 3 models (ResNet50, Iris, Inception) in a single API.
+
+## Keras
+
+- [Denoisify text documents](keras/document-denoiser): deploy an Autoencoder model to clean text document images of noise.
+
+## PyTorch
+
+- [Iris classification](pytorch/iris-classifier): deploy a model to classify iris flowers.
+
+- [Text generation](pytorch/text-generator): deploy Hugging Face's GPT-2 model to generate text.
+
+- [Sentiment analysis](pytorch/sentiment-analyzer): deploy a Hugging Face transformers model for sentiment analysis.
+
+- [Search completion](pytorch/search-completer): deploy a Facebook's RoBERTa model to complete search terms.
+
+- [Answer generation](pytorch/answer-generator): deploy Microsoft's DialoGPT model to answer questions.
+
+- [Text summarization](pytorch/text-summarizer): deploy a BART model (from Hugging Face's transformers library) to summarize text.
+
+- [Reading comprehension](pytorch/reading-comprehender): deploy an AllenNLP model for reading comprehension.
+
+- [Language identification](pytorch/language-identifier): deploy a fastText model to identify languages.
+
+- [Multi-model text analysis](pytorch/multi-model-text-analyzer): deploy 2 models (Sentiment and Summarization analyzers) in a single API.
+
+- [Image classification](pytorch/image-classifier-alexnet): deploy an AlexNet model from TorchVision to classify images.
+
+- [Image classification](pytorch/image-classifier-resnet50): deploy a ResNet50 model from TorchVision to classify images.
+
+- [Object detection](pytorch/object-detector): deploy a Faster R-CNN model from TorchVision to detect objects in images.
+
+- [Question generator](pytorch/question-generator): deploy a transformers model to generate questions given text and the correct answer.
+
+## ONNX
+
+- [Iris classification](onnx/iris-classifier): deploy an XGBoost model (exported in ONNX) to classify iris flowers.
+
+- [YOLOv5 YouTube detection](onnx/yolov5-youtube): deploy a YOLOv5 model trained on COCO val2017 dataset.
+
+- [Multi-model classification](onnx/multi-model-classifier): deploy 3 models (ResNet50, MobileNet, ShuffleNet) in a single API.
+
+## scikit-learn
+
+- [Iris classification](sklearn/iris-classifier): deploy a model to classify iris flowers.
+
+- [MPG estimation](sklearn/mpg-estimator): deploy a linear regression model to estimate MPG.
+
+## spacy
+
+- [Entity recognizer](spacy/entity-recognizer): deploy a spacy model for named entity recognition.

test/apis/batch/onnx/sample.json

@@ -0,0 +1,3 @@
+[
+    "https://i.imgur.com/PzXprwl.jpg"
+]

test/apis/batch/tensorflow/sample.json

@@ -0,0 +1,3 @@
+[
+    "https://i.imgur.com/PzXprwl.jpg"
+]

test/live-reloading/python/mpg-estimator/predictor.py → test/apis/live-reloading/python/mpg-estimator/predictor.py

@@ -1,5 +1,4 @@
 import mlflow.sklearn
-import numpy as np
 
 
 class PythonPredictor:

test/model-caching/python/mpg-estimator/predictor.py → test/apis/model-caching/python/mpg-estimator/predictor.py

@@ -1,5 +1,4 @@
 import mlflow.sklearn
-import numpy as np
 
 
 class PythonPredictor:

test/apis/pytorch/iris-classifier/expectations.yaml

@@ -0,0 +1,5 @@
+# this file is used for testing purposes only
+
+response:
+  content_type: "text"
+  expected: "versicolor"

test/e2e/README.md

@@ -0,0 +1,75 @@
+# End-to-end Tests
+
+## Dependencies
+
+Install the `e2e` package, from the project directory:
+
+```shell
+pip install -e test/e2e
+```
+
+This only needs to be installed once (not on every code change).
+
+## Running the tests
+
+Before running tests, instruct the Python client to use your development CLI binary:
+
+```shell
+export CORTEX_CLI_PATH=<cortex_repo_path>/bin/cortex
+```
+
+### AWS
+
+From an existing cluster:
+
+```shell
+pytest test/e2e/tests -k aws --aws-env <cortex_aws_env>
+```
+
+Using a new cluster, created for testing only and deleted afterwards:
+
+```shell
+pytest test/e2e/tests -k aws --aws-config <cortex_aws_cluster_config.yaml>
+```
+
+**Note:** For the BatchAPI tests, the `--s3-bucket` option should be provided with an
+AWS S3 bucket for testing purposes. It is more convinient however to define
+this bucket through an environment variable, see [configuration](#configuration).
+
+### GCP
+
+From an existing cluster:
+
+```shell
+pytest test/e2e/tests -k gcp --gcp-env <cortex_gcp_env>
+```
+
+Using a new cluster, created for testing only and deleted afterwards:
+
+```shell
+pytest test/e2e/tests -k gcp --gcp-config <cortex_gcp_cluster_config.yaml>
+```
+
+### All Tests
+
+You can run all tests at once, however the provider specific options should be passed
+accordingly, or the test cases will be skipped.
+
+e.g.
+
+```shell
+pytest test/e2e/tests --aws-env <cortex_aws_env> --gcp-env <cortex_gcp_env>
+```
+
+## Configuration
+
+It is possible to configure the behaviour of the tests by defining
+environment variables or a `.env` file at the project directory.
+
+```dotenv
+# .env file
+CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT=60
+CORTEX_TEST_BATCH_DEPLOY_TIMEOUT=30
+CORTEX_TEST_BATCH_JOB_TIMEOUT=120
+CORTEX_TEST_BATCH_S3_BUCKET_DIR=s3://<s3_bucket>/test/jobs
+```

test/e2e/e2e/__init__.py

@@ -0,0 +1,17 @@
+# Copyright 2020 Cortex Labs, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .cluster import create_cluster, delete_cluster
+
+__all__ = ["create_cluster", "delete_cluster"]

test/e2e/e2e/cluster.py

@@ -0,0 +1,59 @@
+# Copyright 2020 Cortex Labs, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+import sys
+
+import yaml
+
+from e2e.exceptions import ClusterCreationException, ClusterDeletionException
+
+
+def create_cluster(cluster_config: str):
+    """Create a cortex cluster from a cluster config"""
+    with open(cluster_config) as f:
+        config = yaml.safe_load(f)
+
+    cluster_name = config["cluster_name"]
+    provider = config["provider"]
+
+    p = subprocess.run(
+        [
+            "cortex",
+            "cluster",
+            "up",
+            "-y",
+            "--config",
+            cluster_config,
+            "--configure-env",
+            f"{cluster_name}-{provider}",
+        ],
+        stdout=sys.stdout,
+        stderr=sys.stderr,
+    )
+
+    if p.returncode != 0:
+        raise ClusterCreationException(f"failed to create cluster with config: {cluster_config}")
+
+
+def delete_cluster(cluster_config: str):
+    """Delete a cortex cluster from a cluster config"""
+    p = subprocess.run(
+        ["cortex", "cluster", "down", "-y", "--config", cluster_config],
+        stdout=sys.stdout,
+        stderr=sys.stderr,
+    )
+
+    if p.returncode != 0:
+        raise ClusterDeletionException(f"failed to delete cluster with config: {cluster_config}")

test/e2e/e2e/exceptions.py

@@ -0,0 +1,10 @@
+class ClusterCreationException(Exception):
+    pass
+
+
+class ClusterDeletionException(Exception):
+    pass
+
+
+class ExpectationsValidationException(Exception):
+    pass